akka-loader: Added left_minute and starting_lineup fields to player_s…

…tats

akka-loader/src/main/scala/LoaderApp.scala

@@ -1,7 +1,7 @@
 import akka.actor.{ActorRef, ActorSystem}
 import chpp.OauthTokens
 import chpp.worlddetails.models.WorldDetails
-import cli.{CommandLine, LoadConfig, LoadScheduledConfig, ScheduleConfig, TeamRankingsConfig}
+import cli._
 import clickhouse.TeamRankJoiner
 import com.google.inject.Guice
 import com.typesafe.config.ConfigFactory

akka-loader/src/main/scala/clickhouse/PlayerStatsJoiner.scala

@@ -40,7 +40,9 @@ object PlayerStatsJoiner {
        |player_events.red_cards,
        |player_events.goals,
        |player_info.nationality,
-       |player_info.match_duration
+       |player_info.match_duration,
+       |player_events.left_minute,
+       |player_events.starting_lineup
        |FROM $databaseName.player_info
        |LEFT JOIN
        |(

akka-loader/src/main/scala/loadergraph/playerevents/PlayerEventsAccumulator.scala

@@ -9,7 +9,8 @@ case class PlayerEventsAccumulator(var season: Int,
                                    var redCards: Int = 0,
                                    var goals: Int = 0,
                                    var injury: Int = 0,
-                                   var leftFieldMinute: Int = - 1) {
+                                   var leftFieldMinute: Int = - 1,
+                                   var startingLineup: Boolean = false) {
   def build: PlayerEventsModelCH =
     PlayerEventsModelCH(season = season,
       round = round,
@@ -18,5 +19,6 @@ case class PlayerEventsAccumulator(var season: Int,
       redCards = redCards,
       goals = goals,
       injury = injury,
-      leftFieldMinute = leftFieldMinute)
+      leftFieldMinute = leftFieldMinute,
+      startingLineup = if (startingLineup) 1 else 0)
 }

akka-loader/src/main/scala/loadergraph/playerevents/PlayerEventsFlow.scala

@@ -1,27 +1,24 @@
 package loadergraph.playerevents
 
-import akka.actor.ActorSystem
 import akka.stream.scaladsl.{Flow, Source}
-import chpp.OauthTokens
 import chpp.matchdetails.models.{BookingType, InjuryType}
 import com.crobox.clickhouse.stream.Insert
 import loadergraph.ClickhouseFlow
 import loadergraph.matchlineup.StreamMatchDetailsWithLineup
 import models.clickhouse.PlayerEventsModelCH
 
 import scala.collection.mutable
-import scala.concurrent.ExecutionContext
 
 object PlayerEventsFlow {
-    def apply(databaseName: String)(implicit oauthTokens: OauthTokens, system: ActorSystem,
-              executionContext: ExecutionContext): Flow[StreamMatchDetailsWithLineup, Insert, _] = {
+    def apply(databaseName: String): Flow[StreamMatchDetailsWithLineup, Insert, _] = {
     Flow[StreamMatchDetailsWithLineup].flatMapConcat(streamMatchDetails => {
       val playersMap = mutable.Map[Long, PlayerEventsAccumulator]()
 
       updateGoalsForPlayers(streamMatchDetails, playersMap)
       updateInjuriesForPlayers(streamMatchDetails, playersMap)
       updateCardsForPlayers(streamMatchDetails, playersMap)
       updateSubstitutionTime(streamMatchDetails, playersMap)
+      updateStartingLineup(streamMatchDetails, playersMap)
 
       Source(playersMap.values.map(_.build).toList)
     })
@@ -70,16 +67,27 @@ object PlayerEventsFlow {
       })
   }
 
-  private def updateSubstitutionTime(streamMatchDetailsWithLineup: StreamMatchDetailsWithLineup, playersMap: mutable.Map[Long, PlayerEventsAccumulator]): Unit = {
-    streamMatchDetailsWithLineup.matchLineup.team.substitutions
+  private def updateSubstitutionTime(streamMatchDetails: StreamMatchDetailsWithLineup, playersMap: mutable.Map[Long, PlayerEventsAccumulator]): Unit = {
+    streamMatchDetails.matchLineup.team.substitutions
       .foreach(substitution => {
         val playerAccumulator = playersMap.getOrElseUpdate(substitution.subjectPlayerId,
-          createPlayerAccumulator(streamMatchDetailsWithLineup, substitution.subjectPlayerId))
+          createPlayerAccumulator(streamMatchDetails, substitution.subjectPlayerId))
 
         playerAccumulator.leftFieldMinute = substitution.matchMinute
       })
   }
 
+  private def updateStartingLineup(streamMatchDetailsWithLineup: StreamMatchDetailsWithLineup, playersMap: mutable.Map[Long, PlayerEventsAccumulator]): Unit = {
+    streamMatchDetailsWithLineup.matchLineup.team.startingLineup
+      .filter(s => s.roleId.id >= 100 && s.roleId.id <= 113) // Without captains, subs and set pieces
+      .foreach{ startingLineupPlayer =>
+        val playerAccumulator = playersMap.getOrElseUpdate(startingLineupPlayer.playerId,
+          createPlayerAccumulator(streamMatchDetailsWithLineup, startingLineupPlayer.playerId))
+
+        playerAccumulator.startingLineup = true
+      }
+  }
+
   private def createPlayerAccumulator(streamMatchDetails: StreamMatchDetailsWithLineup, playerId: Long): PlayerEventsAccumulator =
     PlayerEventsAccumulator(season = streamMatchDetails.matc.season,
       round = streamMatchDetails.matc.round,

akka-loader/src/main/scala/loadergraph/playerinfos/PlayerInfoFlow.scala

@@ -21,18 +21,18 @@ object PlayerInfoFlow {
       .via(PlayersHttpFlow())
       .async
       .via(LogProgressFlow("Players of teams", Some(_._2.matc.team.leagueUnit.league.activeTeams)))
-      .flatMapConcat{case(players, matchDetails) =>
+      .flatMapConcat{case(players, matchDetailsWithLineup) =>
         val matchDuration = players.team.playerList
           .flatMap(_.lastMatch)
-          .filter(lastMatch => lastMatch.matchId == matchDetails.matc.id)
+          .filter(lastMatch => lastMatch.matchId == matchDetailsWithLineup.matc.id)
           .map(_.playedMinutes)
           .maxOption
           .getOrElse(0)
 
         val playerInfos = players.team.playerList
           .map(player =>
             PlayerInfoModelCH.convert(player = player,
-              matchDetails = matchDetails,
+              matchDetails = matchDetailsWithLineup,
               matchDuration = matchDuration,
               countryMap = countryMap)
           )

akka-loader/src/main/scala/models/clickhouse/PlayerEventsModelCH.scala

@@ -9,7 +9,8 @@ case class PlayerEventsModelCH(season: Int,
                                redCards: Int,
                                goals: Int,
                                injury: Int,
-                               leftFieldMinute: Int)
+                               leftFieldMinute: Int,
+                               startingLineup: Int)
 
 object PlayerEventsModelCH {
   implicit val format: JsonFormat[PlayerEventsModelCH] = new JsonFormat[PlayerEventsModelCH] {
@@ -25,6 +26,7 @@ object PlayerEventsModelCH {
         ("goals", JsNumber(obj.goals)),
         ("injury", JsNumber(obj.injury)),
         ("left_minute", JsNumber(obj.leftFieldMinute)),
+        ("starting_lineup", JsNumber(obj.startingLineup))
       )
     }
   }

chpp/src/main/scala/chpp/ChppRequestExecutor.scala

@@ -34,8 +34,8 @@ object ChppRequestExecutor {
     akka.pattern.retry(
       attempt = () => execute(request),
       attempts = retries,
-      minBackoff = 300.millisecond,
-      maxBackoff = 2.seconds,
+      minBackoff = 800.millisecond,
+      maxBackoff = 10.seconds,
       randomFactor = 0.3
     ) transform {
       case Success(model) => Try(Right(model))

chpp/src/main/scala/chpp/RequestCreator.scala

@@ -1,22 +1,23 @@
 package chpp
 
+
+import akka.http.scaladsl.model.headers.RawHeader
+import akka.http.scaladsl.model.{ContentTypes, HttpEntity, HttpRequest}
+import akka.util.ByteString
+import org.apache.commons.codec.binary.Base64
+
 import java.net.URLEncoder
 import java.text.SimpleDateFormat
 import java.util.Date
-
-import akka.http.scaladsl.model.HttpRequest
-import akka.http.scaladsl.model.headers.RawHeader
 import javax.crypto.Mac
 import javax.crypto.spec.SecretKeySpec
-import org.apache.commons.codec.binary.Base64
-
 import scala.util.Random
 
 case class  RequestCreator()
 
 object RequestCreator {
-  val URL = "chpp.hattrick.org"
-  val API_ENDPOINT = "/chppxml.ashx"
+  private val URL = "chpp.hattrick.org"
+  private val API_ENDPOINT = "/chppxml.ashx"
 
   def create(requestParams: Map[String, String])
             (implicit oauthTokens: OauthTokens): HttpRequest = {
@@ -55,8 +56,11 @@ object RequestCreator {
     val header = "OAuth " +
       (oauthParameters + ("oauth_signature" -> URLEncoder.encode(result, "UTF-8"))).map{case(key, value) => s"""$key="$value""""}.mkString(", ");
 
-    HttpRequest(uri = s"https://$URL" + url)
-      .withHeaders(RawHeader("Authorization", header))
+    HttpRequest(uri = s"https://$URL" + url,
+      entity = HttpEntity.Strict(ContentTypes.`text/xml(UTF-8)`, data = ByteString.empty))
+      .withHeaders(
+        RawHeader("Authorization", header),
+      )
   }
 
   def params(file: String, version: String, params: (String, Option[Any])*): Map[String, String] = {

docker-compose.yml

@@ -6,7 +6,7 @@ networks:
 
 services:
   hattrick-clickhouse:
-    image: yandex/clickhouse-server:20.4.4.18
+    image: clickhouse/clickhouse-server:24.1.2.5
     container_name: hattrick-clickhouse
     expose:
      - "8123"

sql/init_scripts/02-player-events.sql

@@ -7,7 +7,8 @@
     `red_cards` UInt8, 
     `goals` UInt8, 
     `injury` UInt8, 
-    `left_minute` Int8
+    `left_minute` Int8,
+    `starting_lineup` UInt8
 )
 ENGINE = MergeTree()
 ORDER BY (season, round, cityHash64(player_id), yellow_cards, red_cards, goals, injury, left_minute)

sql/init_scripts/04-player-stats.sql

@@ -30,6 +30,8 @@
     `goals` UInt8,
     `nationality` UInt8,
     `match_duration` UInt8,
+    `left_minute` Int8,
+    `starting_lineup` UInt8,
     PROJECTION by_player_id (SELECT * ORDER BY (player_id, season, round))
 )
 ENGINE = MergeTree()