diff --git a/.gitignore b/.gitignore index 9c07d4a..97dd83d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,12 @@ -*.class + *.log + +*.class +.idea +target/ + +project/target +.bsp/ +.bloop/ +.vscode/ + diff --git a/.scalafmt.conf b/.scalafmt.conf new file mode 100644 index 0000000..15fff87 --- /dev/null +++ b/.scalafmt.conf @@ -0,0 +1,4 @@ +version = 2.5.0 + +align.preset = more // For pretty alignment. +maxColumn = 120 // For my wide 30" display. \ No newline at end of file diff --git a/build.sbt b/build.sbt new file mode 100644 index 0000000..b91c6b5 --- /dev/null +++ b/build.sbt @@ -0,0 +1,36 @@ + +lazy val akkaVersion = "2.6.15" +lazy val akkaHttpVersion = "10.2.5" +lazy val akkaHttpJsonVersion = "1.37.0" +lazy val akkaPersistenceJdbc = "5.0.1" +lazy val circeVersion = "0.14.1" +lazy val slickVersion = "3.3.3" +lazy val postgresqlVersion = "42.2.23" +lazy val tapirAkkaHttpVersion = "0.19.0-M4" +lazy val logbackClassicVersion = "1.2.5" + +lazy val scalaTestVersion = "3.2.9" + +lazy val root = (project in file(".")). + settings( + inThisBuild(List( + organization := "com.igobrilhante", + scalaVersion := "2.13.4" + )), + name := "github-challenge", + libraryDependencies ++= Seq( + "com.typesafe.akka" %% "akka-http" % akkaHttpVersion, + "de.heikoseeberger" %% "akka-http-circe" % akkaHttpJsonVersion, + "com.typesafe.akka" %% "akka-actor-typed" % akkaVersion, + "com.typesafe.akka" %% "akka-stream" % akkaVersion, + "ch.qos.logback" % "logback-classic" % logbackClassicVersion, + + "io.circe" %% "circe-core" % circeVersion, + "io.circe" %% "circe-generic" % circeVersion, + "io.circe" %% "circe-parser" % circeVersion, + + "com.typesafe.akka" %% "akka-http-testkit" % akkaHttpVersion % Test, + "com.typesafe.akka" %% "akka-actor-testkit-typed" % akkaVersion % Test, + "org.scalatest" %% "scalatest" % "3.1.4" % Test + ) + ) diff --git a/project/build.properties b/project/build.properties new file mode 100644 index 0000000..f0be67b --- /dev/null +++ b/project/build.properties @@ -0,0 +1 @@ +sbt.version=1.5.1 diff --git a/project/plugins.sbt b/project/plugins.sbt new file mode 100644 index 0000000..d4312f7 --- /dev/null +++ b/project/plugins.sbt @@ -0,0 +1 @@ +addSbtPlugin("io.spray" % "sbt-revolver" % "0.9.1") diff --git a/src/main/resources/application.conf b/src/main/resources/application.conf new file mode 100644 index 0000000..47c447c --- /dev/null +++ b/src/main/resources/application.conf @@ -0,0 +1,13 @@ +app { + routes { + # If ask takes more time than this to complete the request is failed + ask-timeout = 5s + } +} + +github { + username = "username" + username = ${?GH_USERNAME} + token = "" + token = ${?GH_TOKEN} +} \ No newline at end of file diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml new file mode 100644 index 0000000..b1fe9ae --- /dev/null +++ b/src/main/resources/logback.xml @@ -0,0 +1,20 @@ + + + + + [%date{ISO8601}] [%level] [%logger] [%thread] [%X{akkaSource}] - %msg%n + + + + + 1024 + true + + + + + + + + diff --git a/src/main/scala/com/igobrilhante/GHSystem.scala b/src/main/scala/com/igobrilhante/GHSystem.scala new file mode 100644 index 0000000..dcac7e8 --- /dev/null +++ b/src/main/scala/com/igobrilhante/GHSystem.scala @@ -0,0 +1,12 @@ +package com.igobrilhante + +import akka.actor.typed.Behavior +import akka.actor.typed.scaladsl.Behaviors + +object GHSystem { + + trait Command + + def apply(): Behavior[GHSystem.Command] = Behaviors.empty + +} diff --git a/src/main/scala/com/igobrilhante/github/api/GHService.scala b/src/main/scala/com/igobrilhante/github/api/GHService.scala new file mode 100644 index 0000000..c2cf661 --- /dev/null +++ b/src/main/scala/com/igobrilhante/github/api/GHService.scala @@ -0,0 +1,19 @@ +package com.igobrilhante.github.api + +import scala.concurrent.Future + +import com.igobrilhante.github.models.{GHContributor, GHOrganization, GHRepository} + +trait GHService { + + def getOrganization(organizationId: String): Future[GHOrganization] + + def getRepositories(organizationId: String, page: Int): Future[List[GHRepository]] + + def getAllRepositories(organizationId: String): Future[List[GHRepository]] + + def getContributors(organizationId: String, repositoryId: String): Future[List[GHContributor]] + + def getRankedContributors(organizationId: String): Future[List[GHContributor]] + +} diff --git a/src/main/scala/com/igobrilhante/github/commons/Logging.scala b/src/main/scala/com/igobrilhante/github/commons/Logging.scala new file mode 100644 index 0000000..6932b08 --- /dev/null +++ b/src/main/scala/com/igobrilhante/github/commons/Logging.scala @@ -0,0 +1,7 @@ +package com.igobrilhante.github.commons + +import org.slf4j.{Logger, LoggerFactory} + +trait Logging { + val logger: Logger = LoggerFactory.getLogger(this.getClass) +} diff --git a/src/main/scala/com/igobrilhante/github/impl/GHServiceImpl.scala b/src/main/scala/com/igobrilhante/github/impl/GHServiceImpl.scala new file mode 100644 index 0000000..4e49491 --- /dev/null +++ b/src/main/scala/com/igobrilhante/github/impl/GHServiceImpl.scala @@ -0,0 +1,139 @@ +package com.igobrilhante.github.impl + +import scala.concurrent.{ExecutionContext, Future} + +import akka.actor.typed.ActorSystem +import akka.http.javadsl.unmarshalling.Unmarshaller +import akka.http.scaladsl.Http +import akka.http.scaladsl.model.{HttpHeader, HttpRequest, HttpResponse} +import akka.http.scaladsl.unmarshalling.{Unmarshal, Unmarshaller} +import akka.stream.Materializer +import akka.http.scaladsl.model._ +import HttpMethods._ + +import com.igobrilhante.GHSystem +import com.igobrilhante.github.api.GHService +import com.igobrilhante.github.models.{GHCommit, GHContributor, GHOrganization, GHRepository} +import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport._ +import HttpProtocols._ +import MediaTypes._ +import HttpCharsets._ +import akka.http.scaladsl.model.headers.BasicHttpCredentials + +import com.igobrilhante.github.commons.Logging + +class GHServiceImpl()(implicit val system: ActorSystem[_], val ec: ExecutionContext) extends GHService with Logging { + + private val mat = Materializer.matFromSystem + private val BaseApi = "https://api.github.com" + private val http = Http(system) + private val MaxReposPerPage = 100 + private val config = system.settings.config + private val githubConfig = config.getConfig("github") + private val GitHubUser = githubConfig.getString("username") + private val GitHubToken = githubConfig.getString("token") + private val GitHubCredentials = BasicHttpCredentials(GitHubUser, GitHubToken) + + override def getOrganization(organizationId: String): Future[GHOrganization] = { + val uri = s"$BaseApi/orgs/${organizationId}" + for { + response <- request(uri) + list <- Unmarshal(response).to[GHOrganization] + } yield list + } + + override def getRepositories(organizationId: String, page: Int = 1): Future[List[GHRepository]] = { + val uri = s"$BaseApi/orgs/${organizationId}/repos?page=$page&per_page=$MaxReposPerPage" + for { + response <- request(uri) + list <- Unmarshal(response).to[List[GHRepository]] + } yield list + + } + + override def getContributors(organizationId: String, repositoryId: String): Future[List[GHContributor]] = { + val uri = s"$BaseApi/repos/${organizationId}/${repositoryId}/contributors" + for { + response <- request(uri) + list <- Unmarshal(response).to[List[GHContributor]] + } yield list + } + + def getRankedContributors(organizationId: String): Future[List[GHContributor]] = { + + def rankContributors(list: List[(GHRepository, List[GHContributor])]) = { + + list + .flatMap { case (repo, contributors) => contributors.map((_, repo)) } + .groupBy(_._1.login) + .map { + case (_, contributorRepositories) => + val totalContributions = contributorRepositories.map(_._1.contributions).sum + val contributor = contributorRepositories.head._1 + contributor.copy(contributions = totalContributions) + } + .toList + .sortBy(_.contributions)(Ordering[Int].reverse) + + } + + for { + allRepositories <- getAllRepositories(organizationId) + reposAndContributors <- + Future.sequence(allRepositories.map(repo => getContributors(organizationId, repo.name).map(c => (repo, c)))) + result = rankContributors(reposAndContributors) + } yield result + } + + def getAllRepositories(organizationId: String): Future[List[GHRepository]] = { + getAllRepositoriesStrategy(organizationId) + } + + private def getAllRepositoriesStrategy(organizationId: String): Future[List[GHRepository]] = { + + def test(buckets: Int) = { + (1 to buckets) + .map(page => getRepositories(organizationId, page)) + .foldLeft(Future.successful(List.empty[GHRepository])) { + case (result, future) => + for { + currentList <- result + next <- future + } yield (currentList ++ next) + } + } + + for { + org <- getOrganization(organizationId) + total = org.total + buckets = math.ceil(total / MaxReposPerPage.toDouble).toInt + result <- test(buckets) + } yield result + } + + private def getAllRepositoriesRecursive(organizationId: String): Future[List[GHRepository]] = { + def getRepos(page: Int): Future[List[GHRepository]] = { + for { + repos <- getRepositories(organizationId, page) + result <- { + if (repos.length < MaxReposPerPage) Future.successful(repos) + else getRepos(page + 1).map(_ ++ repos) + } + } yield result + } + getRepos(page = 1) + } + + private def request(uri: String) = + http.singleRequest( + HttpRequest(uri = uri).withHeaders( + headers.Authorization(GitHubCredentials) + ) + ) + +// private def request2[A](uri: String)(implicit um: Unmarshaller[HttpResponse, A]) = +// http +// .singleRequest(HttpRequest(uri = uri)) +// .flatMap(response => Unmarshal(response).to[A](um, ec, mat)) + +} diff --git a/src/main/scala/com/igobrilhante/github/models/GHCommit.scala b/src/main/scala/com/igobrilhante/github/models/GHCommit.scala new file mode 100644 index 0000000..2688bc0 --- /dev/null +++ b/src/main/scala/com/igobrilhante/github/models/GHCommit.scala @@ -0,0 +1,11 @@ +package com.igobrilhante.github.models + +import io.circe.{Decoder, Encoder} +import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} + +case class GHCommit() + +object GHCommit { + implicit val decoder: Decoder[GHCommit] = deriveDecoder + implicit val encoder: Encoder[GHCommit] = deriveEncoder +} diff --git a/src/main/scala/com/igobrilhante/github/models/GHContributor.scala b/src/main/scala/com/igobrilhante/github/models/GHContributor.scala new file mode 100644 index 0000000..bf98779 --- /dev/null +++ b/src/main/scala/com/igobrilhante/github/models/GHContributor.scala @@ -0,0 +1,13 @@ +package com.igobrilhante.github.models + +import io.circe.{Decoder, Encoder} +import io.circe.generic.semiauto._ + +case class GHContributor(id: Long, login: String, contributions: Int) + +object GHContributor { + + implicit val decoder: Decoder[GHContributor] = deriveDecoder + implicit val encoder: Encoder[GHContributor] = deriveEncoder + +} diff --git a/src/main/scala/com/igobrilhante/github/models/GHOrganization.scala b/src/main/scala/com/igobrilhante/github/models/GHOrganization.scala new file mode 100644 index 0000000..d048759 --- /dev/null +++ b/src/main/scala/com/igobrilhante/github/models/GHOrganization.scala @@ -0,0 +1,15 @@ +package com.igobrilhante.github.models + +import io.circe.{Decoder, Encoder} +import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} + +case class GHOrganization(id: Long, login: String, name: String, public_repos: Int, total_private_repos: Option[Int]) { + + def total: Int = public_repos + total_private_repos.getOrElse(0) + +} + +object GHOrganization { + implicit val decoder: Decoder[GHOrganization] = deriveDecoder + implicit val encoder: Encoder[GHOrganization] = deriveEncoder +} diff --git a/src/main/scala/com/igobrilhante/github/models/GHRepository.scala b/src/main/scala/com/igobrilhante/github/models/GHRepository.scala new file mode 100644 index 0000000..6babffe --- /dev/null +++ b/src/main/scala/com/igobrilhante/github/models/GHRepository.scala @@ -0,0 +1,13 @@ +package com.igobrilhante.github.models + +import io.circe.{Decoder, Encoder} +import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} + +case class GHRepository(id: Long, name: String, `private`: Boolean) + +object GHRepository { + + implicit val decoder: Decoder[GHRepository] = deriveDecoder + implicit val encoder: Encoder[GHRepository] = deriveEncoder + +} diff --git a/src/test/resources/application-test.conf b/src/test/resources/application-test.conf new file mode 100644 index 0000000..7f76332 --- /dev/null +++ b/src/test/resources/application-test.conf @@ -0,0 +1,3 @@ +include "application" + +# default config for tests, we just import the regular conf \ No newline at end of file diff --git a/src/test/scala/com/igobrilhante/github/tests/GHServicePublicSpec.scala b/src/test/scala/com/igobrilhante/github/tests/GHServicePublicSpec.scala new file mode 100644 index 0000000..0da2d1d --- /dev/null +++ b/src/test/scala/com/igobrilhante/github/tests/GHServicePublicSpec.scala @@ -0,0 +1,43 @@ +package com.igobrilhante.github.tests + +import com.igobrilhante.github.api.GHService +import com.igobrilhante.github.impl.GHServiceImpl + +class GHServicePublicSpec extends GHSpec { + + val service: GHService = new GHServiceImpl() + + val organizationId = "ScalaConsultants" + val publicRepo = "lift-rest-demo" + + "GHService Public Spec" should "get public repositories for ScalaConsultants" in { + + val list = service.getRepositories("ScalaConsultants", page = 1).futureValue(timeout) + + list should not be empty + + } + + it should s"get the organization '$organizationId'" in { + val org = service.getOrganization(organizationId).futureValue(timeout) + + org.login shouldEqual organizationId + + } + + it should "get a list of contributors for public repo 'lift-rest-demo'" in { + val list = service.getContributors(organizationId, publicRepo).futureValue(timeout) + + list should not be empty + } + + it should s"get all repositories for $organizationId" in { + val org = service.getOrganization(organizationId).futureValue(timeout) + val list = service.getAllRepositories(organizationId).futureValue(timeout) + + list should have size org.total + list.filter(_.`private`) should have size org.total_private_repos.getOrElse(0).toLong + list.filterNot(_.`private`) should have size org.public_repos + } + +} diff --git a/src/test/scala/com/igobrilhante/github/tests/GHSpec.scala b/src/test/scala/com/igobrilhante/github/tests/GHSpec.scala new file mode 100644 index 0000000..0c17c26 --- /dev/null +++ b/src/test/scala/com/igobrilhante/github/tests/GHSpec.scala @@ -0,0 +1,35 @@ +package com.igobrilhante.github.tests + +import scala.concurrent.duration._ +import scala.language.postfixOps + +import akka.actor.testkit.typed.scaladsl.ActorTestKit +import akka.actor.typed.ActorSystem +import akka.http.scaladsl.testkit.ScalatestRouteTest + +import com.igobrilhante.GHSystem +import org.scalatest.concurrent.PatienceConfiguration.Timeout +import org.scalatest.concurrent.ScalaFutures +import org.scalatest.flatspec.AsyncFlatSpec +import org.scalatest.matchers.should.Matchers +import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach} + +trait GHSpec + extends AsyncFlatSpec + with Matchers + with BeforeAndAfterEach + with BeforeAndAfterAll + with ScalaFutures + with ScalatestRouteTest { + + val timeout = Timeout(30 seconds) + val testKit = ActorTestKit() + implicit val ec = testKit.system.executionContext + implicit val actorSystem = ActorSystem.create(GHSystem(), "main") + + override def afterAll(): Unit = { + actorSystem.terminate() + testKit.shutdownTestKit() + } + +}