Better logger config (#2170)

* wip: better logger config

* wip: better logger config

* wip: better logger config

* wip: better logger config

* wip: better printing for plus-concatenated args

* wip: best-match prefix search

distage/distage-framework/.jvm/src/main/scala/izumi/distage/roles/RoleAppBootLoggerModule.scala

@@ -13,7 +13,7 @@ class RoleAppBootLoggerModule[F[_]: TagK: DefaultModule]() extends ModuleDef {
   make[EarlyLoggerFactory].from[EarlyLoggerFactory.EarlyLoggerFactoryImpl]
 
   make[LogConfigLoader].from[LogConfigLoader.LogConfigLoaderImpl]
-  make[RouterFactory].from[RouterFactory.RouterFactoryImpl]
+  make[RouterFactory].from[RouterFactory.RouterFactoryConsoleSinkImpl]
   make[LateLoggerFactory].from[LateLoggerFactory.LateLoggerFactoryImpl]
   make[LogQueue].fromResource(ThreadingLogQueue.resource())
 

distage/distage-framework/.jvm/src/main/scala/izumi/distage/roles/launcher/LogConfigLoader.scala

@@ -6,7 +6,6 @@ import izumi.distage.roles.launcher.LogConfigLoader.DeclarativeLoggerConfig
 import izumi.logstage.api.Log
 import izumi.logstage.api.Log.Level.Warn
 import izumi.logstage.api.Log.Message
-import izumi.logstage.api.config.{LoggerPath, LoggerPathConfig, LoggerPathForLines}
 import izumi.logstage.api.rendering.RenderingOptions
 import logstage.IzLogger
 
@@ -28,7 +27,7 @@ object LogConfigLoader {
   case class DeclarativeLoggerConfig(
     format: LoggerFormat,
     rendering: RenderingOptions,
-    levels: Map[LoggerPath, LoggerPathConfig],
+    levels: Map[String, Log.Level],
     rootLevel: Log.Level,
     interceptJUL: Boolean,
   )
@@ -51,23 +50,22 @@ object LogConfigLoader {
       val options = logconf.options.getOrElse(RenderingOptions.default)
       val jul = logconf.jul.getOrElse(true)
 
-      val levels = logconf.levels.flatMap {
-        case (stringLevel, packageList) =>
-          val level = Log.Level.parseLetter(stringLevel)
-          packageList.flatMap {
-            pkg =>
-              val p = LoggerPathForLines.parse(pkg)
-              if (p.lines.nonEmpty) {
-                p.lines.map {
-                  l =>
-                    (LoggerPath(p.id, Some(l)), LoggerPathConfig(level))
-                }
-              } else {
-                Seq((LoggerPath(p.id, None), LoggerPathConfig(level)))
-              }
-
-          }
-      }
+      import izumi.fundamentals.collections.IzCollections.*
+      val levels = logconf.levels.iterator
+        .flatMap {
+          case (stringLevel, packageList) =>
+            val level = Log.Level.parseLetter(stringLevel)
+            packageList.map {
+              pkg =>
+                (pkg, level)
+            }
+        }
+        .toMultimapView
+        .map {
+          case (path, levels) =>
+            (path, levels.min)
+        }
+        .toMap
 
       val format = if (isJson) {
         LoggerFormat.Json

distage/distage-framework/.jvm/src/main/scala/izumi/distage/roles/launcher/RouterFactory.scala

@@ -1,20 +1,24 @@
 package izumi.distage.roles.launcher
 
 import izumi.distage.roles.launcher.LogConfigLoader.{DeclarativeLoggerConfig, LoggerFormat}
-import izumi.logstage.api.config.{LoggerConfig, LoggerPathConfig, LoggerPathRule}
+import izumi.logstage.api.config.LoggingTarget
 import izumi.logstage.api.logger.LogQueue
 import izumi.logstage.api.rendering.StringRenderingPolicy
-import izumi.logstage.api.routing.LogConfigServiceImpl
 import logstage.circe.LogstageCirceRenderingPolicy
 import logstage.{ConfigurableLogRouter, ConsoleSink}
 
+import scala.annotation.nowarn
+
 trait RouterFactory {
   def createRouter(config: DeclarativeLoggerConfig, buffer: LogQueue): ConfigurableLogRouter
 }
 
 object RouterFactory {
-  class RouterFactoryImpl extends RouterFactory {
+  class RouterFactoryConsoleSinkImpl extends RouterFactory {
+    @nowarn("msg=Unused import")
     override def createRouter(config: DeclarativeLoggerConfig, buffer: LogQueue): ConfigurableLogRouter = {
+      import scala.collection.compat.*
+
       val policy = config.format match {
         case LoggerFormat.Json =>
           new LogstageCirceRenderingPolicy()
@@ -23,16 +27,11 @@ object RouterFactory {
       }
       val sink = new ConsoleSink(policy)
       val sinks = List(sink)
-      val levelConfigs = config.levels.map {
-        case (pkg, level) =>
-          (pkg, LoggerPathRule(level, sinks))
-      }
 
-      // TODO: here we may read log configuration from config file
-      val router = new ConfigurableLogRouter(
-        new LogConfigServiceImpl(
-          LoggerConfig(LoggerPathRule(LoggerPathConfig(config.rootLevel), sinks), levelConfigs)
-        ),
+      val router = ConfigurableLogRouter(
+        config.rootLevel,
+        sinks,
+        config.levels.view.mapValues(level => LoggingTarget.Level(level)).toMap,
         buffer,
       )
 

distage/distage-testkit-core/src/main/scala/izumi/distage/testkit/runner/impl/TestPlanner.scala

@@ -123,7 +123,7 @@ class TestPlanner[F[_]: TagK: DefaultModule](
                   // test loggers will not create polling threads and will log immediately
                   val logConfigLoader = new LogConfigLoaderImpl(CLILoggerOptions(envExec.logLevel, json = false), configLoadLogger)
                   val logConfig = logConfigLoader.loadLoggingConfig(config)
-                  val router = new RouterFactory.RouterFactoryImpl().createRouter(logConfig, logBuffer)
+                  val router = new RouterFactory.RouterFactoryConsoleSinkImpl().createRouter(logConfig, logBuffer)
 
                   prepareGroupPlans(envExec, config, env, tests, router).left.map(bad => (tests, bad))
                 }

fundamentals/fundamentals-collections/src/main/scala/izumi/fundamentals/collections/WildcardPrefixTree.scala

@@ -1,33 +1,76 @@
 package izumi.fundamentals.collections
 
 import izumi.fundamentals.collections
-import izumi.fundamentals.collections.WildcardPrefixTree.PathElement
-
-case class WildcardPrefixTree[K, V](values: Seq[V], children: Map[PathElement[K], WildcardPrefixTree[K, V]]) {
-  def findSubtrees(prefix: List[K]): Seq[WildcardPrefixTree[K, V]] = {
-    prefix match {
-      case Nil =>
-        Seq(this)
-      case head :: tail =>
-        val exact = children.get(PathElement.Value(head))
-        val wildcard = children.get(PathElement.Wildcard)
-        (exact.toSeq ++ wildcard.toSeq).flatMap(_.findSubtrees(tail))
+import izumi.fundamentals.collections.WildcardPrefixTree.{BestMatch, PathElement}
+
+import scala.annotation.tailrec
+
+final case class WildcardPrefixTree[K, V](values: Seq[V], children: Map[PathElement[K], WildcardPrefixTree[K, V]]) {
+  @tailrec
+  def findExactMatch(prefix: Iterable[K]): Option[WildcardPrefixTree[K, V]] = {
+    if (prefix.isEmpty) {
+      Some(this)
+    } else {
+      val exact = children.get(PathElement.Value(prefix.head))
+      val wildcard = children.get(PathElement.Wildcard)
+      exact.orElse(wildcard) match {
+        case Some(value) =>
+          value.findExactMatch(prefix.tail)
+        case None =>
+          None
+      }
+    }
+  }
+
+  @tailrec
+  def findBestMatch(prefix: Iterable[K]): BestMatch[K, V] = {
+    if (prefix.isEmpty) {
+      BestMatch(this, prefix)
+    } else {
+      val exact = children.get(PathElement.Value(prefix.head))
+      val wildcard = children.get(PathElement.Wildcard)
+      exact.orElse(wildcard) match {
+        case Some(value) =>
+          value.findBestMatch(prefix.tail)
+        case None =>
+          BestMatch(this, prefix)
+      }
+    }
+  }
+
+  def findAllMatchingSubtrees(prefix: Iterable[K]): Seq[WildcardPrefixTree[K, V]] = {
+    if (prefix.isEmpty) {
+      Seq(this)
+    } else {
+      val exact = children.get(PathElement.Value(prefix.head))
+      val wildcard = children.get(PathElement.Wildcard)
+      (exact.toSeq ++ wildcard.toSeq).flatMap(_.findAllMatchingSubtrees(prefix.tail))
     }
   }
 
-  def subtreeValues: Seq[V] = {
-    values ++ children.values.flatMap(_.subtreeValues)
+  def allValuesFromSubtrees: Seq[V] = {
+    values ++ children.values.flatMap(_.allValuesFromSubtrees)
+  }
+
+  def maxValues: Int = {
+    (Seq(values.size) ++ children.values.map(_.maxValues)).max
   }
 }
 
 object WildcardPrefixTree {
+  case class BestMatch[K, V](found: WildcardPrefixTree[K, V], unmatched: Iterable[K])
+
   sealed trait PathElement[+V]
   object PathElement {
-    case class Value[V](value: V) extends PathElement[V]
+    final case class Value[V](value: V) extends PathElement[V]
     case object Wildcard extends PathElement[Nothing]
   }
 
   def build[P, V](pairs: Seq[(Seq[Option[P]], V)]): WildcardPrefixTree[P, V] = {
+    build(pairs, identity)
+  }
+
+  def build[P, V](pairs: Seq[(Seq[Option[P]], V)], map: (Seq[V] => Seq[V])): WildcardPrefixTree[P, V] = {
     val (currentValues, subValues) = pairs.partition(_._1.isEmpty)
 
     val next = subValues
@@ -45,10 +88,12 @@ object WildcardPrefixTree {
             case None =>
               PathElement.Wildcard
           }
-          wk -> build(group.map(_._2))
+          wk -> build(group.map(_._2), map)
       }
       .toMap
 
-    collections.WildcardPrefixTree(currentValues.map(_._2), next)
+    val values = map(currentValues.map(_._2))
+
+    collections.WildcardPrefixTree(values, next)
   }
 }

fundamentals/fundamentals-collections/src/test/scala/izumi/fundamentals/collections/WildcardPrefixTreeTest.scala

@@ -2,12 +2,14 @@ package izumi.fundamentals.collections
 
 import org.scalatest.wordspec.AnyWordSpec
 
+import scala.collection.immutable.Seq
+
 class WildcardPrefixTreeTest extends AnyWordSpec {
   def call[K, V](tree: WildcardPrefixTree[K, V], path: K*): Set[V] = {
-    tree.findSubtrees(path.toList).flatMap(_.subtreeValues).toSet
+    tree.findAllMatchingSubtrees(path.toList).flatMap(_.allValuesFromSubtrees).toSet
   }
 
-  "prefix tree" should {
+  "prefix tree1" should {
     "support prefix search" in {
       val tree = WildcardPrefixTree.build(
         Seq(
@@ -41,6 +43,74 @@ class WildcardPrefixTreeTest extends AnyWordSpec {
       assert(call(tree, "a", "b") == Set(1, 2, 3))
       assert(call(tree, "a", "x") == Set(1, 3))
     }
+
+    "support root wildcard" in {
+      val tree = WildcardPrefixTree.build(
+        Seq(
+          (Seq(Some("a"), None, Some("c")), 1),
+          (Seq(Some("a"), None, Some("d")), 3),
+          (Seq(None), 2),
+        )
+      )
+
+      assert(call(tree, "a") == Set(1, 2, 3))
+      assert(call(tree, "b") == Set(2))
+
+      assert(tree.findAllMatchingSubtrees(Seq("a")).size == 2)
+
+      assert(tree.findExactMatch(Seq("a")).exists(_.values.isEmpty))
+    }
+
+    "merge values in identical rows" in {
+      val tree = WildcardPrefixTree.build(
+        Seq(
+          (Seq(Some("b"), None, Some("c")), 5),
+          (Seq(Some("b"), None, Some("c")), 6),
+          (Seq(None), 1),
+          (Seq(None), 2),
+        )
+      )
+
+      assert(call(tree, "a") == Set(1, 2))
+      assert(call(tree, "b", "x", "c") == Set(5, 6))
+    }
+
+    "find best matches" in {
+      val tree1 = WildcardPrefixTree.build(
+        Seq(
+          (Seq(Some("b"), Some("c")), 1),
+          (Seq(Some("b"), Some("c"), Some("c")), 10),
+          (Seq(Some("b"), None), 2),
+          (Seq(Some("b"), None, Some("c")), 11),
+          (Seq(None), 100),
+          (Seq(None), 101),
+        )
+      )
+
+      assert(tree1.findBestMatch(Seq("b", "c", "missing")).found.values.toSet == Set(1))
+      assert(tree1.findBestMatch(Seq("b", "c", "missing")).unmatched == Seq("missing"))
+
+      assert(tree1.findBestMatch(Seq("b", "missing", "c")).found.values.toSet == Set(11))
+      assert(tree1.findBestMatch(Seq("b", "missing", "c")).unmatched.isEmpty)
+
+      assert(tree1.findBestMatch(Seq("b", "missing", "missing")).found.values.toSet == Set(2))
+      assert(tree1.findBestMatch(Seq("b", "missing", "missing2")).unmatched == Seq("missing2"))
+      assert(tree1.findBestMatch(Seq("b", "x", "missing")).found.values.toSet == Set(2))
+      assert(tree1.findBestMatch(Seq("b", "x", "missing")).unmatched == Seq("missing"))
+
+      assert(tree1.findBestMatch(Seq("missing1", "missing2")).found.values.toSet == Set(100, 101))
+      assert(tree1.findBestMatch(Seq("missing1", "missing2")).unmatched == Seq("missing2"))
+
+      val tree2 = WildcardPrefixTree.build(
+        Seq(
+          (Seq.empty, 100),
+          (Seq(Some("x")), 101),
+        )
+      )
+
+      assert(tree2.findBestMatch(Seq("b", "c", "missing")).found.values.toSet == Set(100))
+
+    }
   }
 
 }

fundamentals/fundamentals-language/src/main/scala/izumi/fundamentals/platform/language/SourceFilePosition.scala

@@ -5,5 +5,5 @@ final case class SourceFilePosition(file: String, line: Int) {
 }
 
 object SourceFilePosition {
-  def unknown = SourceFilePosition("?", 0)
+  def unknown: SourceFilePosition = SourceFilePosition("?", 0)
 }

fundamentals/fundamentals-platform/src/main/scala/izumi/fundamentals/platform/strings/WildcardPrefixTreeTools.scala

@@ -0,0 +1,39 @@
+package izumi.fundamentals.platform.strings
+
+import izumi.fundamentals.collections.WildcardPrefixTree
+import izumi.fundamentals.collections.WildcardPrefixTree.PathElement
+
+object WildcardPrefixTreeTools {
+  implicit class WildcardPrefixTreeExt[K, V](tree: WildcardPrefixTree[K, V]) {
+    def print: String = {
+      print(Seq.empty, tree)
+    }
+
+    private def print(path: Seq[PathElement[K]], tree: WildcardPrefixTree[K, V]): String = {
+      import izumi.fundamentals.preamble.*
+
+      val pathRepr = if (path.isEmpty) {
+        "/"
+      } else {
+        path.map {
+          case PathElement.Value(value) => value.toString
+          case PathElement.Wildcard => "*"
+        }.last
+      }
+
+      val head = if (tree.values.isEmpty) {
+        s"$pathRepr"
+      } else {
+        s"$pathRepr => ${tree.values.mkString(", ")}"
+      }
+
+      if (tree.children.isEmpty) {
+        head
+      } else {
+        val sub = tree.children.map { case (p, t) => print(path :+ p, t) }.niceList(shift = "").shift(2)
+        s"$head $sub"
+      }
+
+    }
+  }
+}

logstage/logstage-adapter-slf4j/src/main/scala/izumi/logstage/adapter/slf4j/LogstageSlf4jLogger.scala

@@ -1,7 +1,7 @@
 package izumi.logstage.adapter.slf4j
 
-import izumi.fundamentals.platform.language.SourceFilePosition
-import izumi.logstage.api.Log._
+import izumi.fundamentals.platform.language.{CodePosition, SourceFilePosition}
+import izumi.logstage.api.Log.*
 import izumi.logstage.api.logger.LogRouter
 import org.slf4j.{Logger, Marker}
 
@@ -32,10 +32,11 @@ class LogstageSlf4jLogger(name: String, router: LogRouter) extends Logger {
 
     val ctx = caller match {
       case Some(frame) =>
-        StaticExtendedContext(id, SourceFilePosition(frame.getFileName, frame.getLineNumber))
+        val fname = Option(frame.getFileName).getOrElse("?")
+        StaticExtendedContext(CodePosition(SourceFilePosition(fname, frame.getLineNumber), name))
 
       case None =>
-        StaticExtendedContext(id, SourceFilePosition.unknown)
+        StaticExtendedContext(CodePosition(SourceFilePosition.unknown, name))
     }
 
     val customContext = marker match {