Fixes #26235: License update needs a button and we need all license end dates

webapp/sources/rudder/rudder-core/src/main/scala/com/normation/plugins/PluginSettings.scala

@@ -96,65 +96,140 @@ object JsonPluginsDetails {
   implicit val encoderJsonPluginsDetails: JsonEncoder[JsonPluginsDetails] = DeriveJsonEncoder.gen
 
   def buildDetails(plugins: Seq[JsonPluginDetails]): JsonPluginsDetails = {
-    val limits = JsonGlobalPluginLimits.getGlobalLimits(plugins.flatMap(_.license))
-    JsonPluginsDetails(limits, plugins)
+    val global = GlobalPluginsLicense.from[ZonedDateTime](plugins.flatMap(_.license))
+    JsonPluginsDetails(global.map(JsonGlobalPluginLimits.fromGlobalLicense), plugins)
   }
 }
 
-/*
- * Global limit information about plugins (the most restrictive)
- */
-final case class JsonGlobalPluginLimits(
-    licensees: Option[NonEmptyChunk[String]],
+/**
+  * Base structure for an aggregated view of licenses for multiple plugins.
+  * In this file implementations need different types for the endDate field, and need them to be serializable.
+  * 
+  * We should pay attention to the JsonEncoder of this base structure : we want an encoder for it,
+  * but it should dispatch the encoding to case class structures that are implementing the values.
+  */
+sealed abstract class GlobalPluginsLicense[EndDate](
+    val licensees: Option[NonEmptyChunk[String]],
     // for now, min/max version is not used and is always 00-99
-    startDate: Option[ZonedDateTime],
-    endDate:   Option[ZonedDateTime],
-    maxNodes:  Option[Int]
+    val startDate: Option[ZonedDateTime],
+    val endDate:   Option[EndDate],
+    val maxNodes:  Option[Int]
 ) {
-  import JsonGlobalPluginLimits.*
-  def combine(that: JsonGlobalPluginLimits): JsonGlobalPluginLimits = {
-    // for efficiency : check equality and hash first before field comparison,
-    // as it will mostly be the case because license information should be similar
-    if (this == that) this
-    else {
-      JsonGlobalPluginLimits(
-        comp[NonEmptyChunk[String]](this.licensees, that.licensees, _ ++ _),
-        comp[ZonedDateTime](this.startDate, that.startDate, (x, y) => if (x.isAfter(y)) x else y),
-        comp[ZonedDateTime](this.endDate, that.endDate, (x, y) => if (x.isBefore(y)) x else y),
-        comp[Int](this.maxNodes, that.maxNodes, (x, y) => if (x < y) x else y)
-      )
-    }
+  import GlobalPluginsLicense.*
+
+  private[GlobalPluginsLicense] def sortDistinctLicensees: GlobalPluginsLicense[EndDate] = {
+    withLicensees(licensees.map(_.sorted.distinct).flatMap(NonEmptyChunk.fromChunk))
+  }
+
+  protected def withLicensees(licensees: Option[NonEmptyChunk[String]]): GlobalPluginsLicense[EndDate] = {
+    new GlobalPluginsLicense[EndDate](
+      licensees,
+      startDate,
+      endDate,
+      maxNodes
+    ) {}
   }
 }
-object JsonGlobalPluginLimits {
+
+object GlobalPluginsLicense {
   import DateFormaterService.JodaTimeToJava
-  import DateFormaterService.json.encoderZonedDateTime
-  implicit val encoderGlobalPluginLimits: JsonEncoder[JsonGlobalPluginLimits] = DeriveJsonEncoder.gen
 
-  def fromLicenseInfo(info: PluginLicenseInfo): JsonGlobalPluginLimits = {
-    JsonGlobalPluginLimits(
+  final case class DateCount(date: ZonedDateTime, count: Int)
+  // Opaque type for counts by date. Encoded as json list but using a Map for unicity when grouping by date
+  final case class DateCounts(value: Map[ZonedDateTime, DateCount]) {
+    def values: Iterable[DateCount] = value.values
+  }
+
+  /**
+    * Typeclass for helping do aggregation on the fields.
+    * This is a semigroup, limited to supported operations on implemented types,
+    * and to the base type.
+    */
+  sealed private trait Aggregate[T] {
+    def aggregate(a: T, b: T): T
+  }
+  private object Aggregate          {
+    def apply[T](implicit ev: Aggregate[T]) = ev
+
+    implicit val zonedDateTime: Aggregate[ZonedDateTime] = new Aggregate[ZonedDateTime] {
+      override def aggregate(a: ZonedDateTime, b: ZonedDateTime): ZonedDateTime = if (a.isBefore(b)) a else b
+    }
+
+    implicit val dateCounts: Aggregate[DateCounts] = new Aggregate[DateCounts] {
+      override def aggregate(a: DateCounts, b: DateCounts): DateCounts = {
+        DateCounts((a.value.toList ::: b.value.toList).groupBy { case (date, _) => date }.map {
+          case (k, v) => k -> DateCount(k, v.map { case (_, dc) => dc.count }.sum)
+        }.toMap)
+      }
+    }
+
+    // the main aggregation : merge two plugin licenses into a global one
+    implicit def base[EndDate: Aggregate]: Aggregate[GlobalPluginsLicense[EndDate]] = {
+      new Aggregate[GlobalPluginsLicense[EndDate]] {
+        // for efficiency : check equality and hash first before field comparison,
+        // as it will mostly be the case because license information should be similar
+        override def aggregate(a: GlobalPluginsLicense[EndDate], b: GlobalPluginsLicense[EndDate]) = {
+          if (a == b) a
+          else {
+            new GlobalPluginsLicense[EndDate](
+              comp[NonEmptyChunk[String]](a.licensees, b.licensees, _ ++ _),
+              comp[ZonedDateTime](a.startDate, b.startDate, (x, y) => if (x.isAfter(y)) x else y),
+              comp[EndDate](a.endDate, b.endDate, Aggregate[EndDate].aggregate),
+              comp[Int](a.maxNodes, b.maxNodes, (x, y) => if (x < y) x else y)
+            ) {}
+          }
+        }
+      }
+    }
+  }
+  implicit private class AggregateOps[T](val t: T) extends AnyVal {
+    def aggregate(other: T)(implicit agg: Aggregate[T]): T = agg.aggregate(t, other)
+  }
+
+  /**
+    * Typeclass for proving that a type can be constructed from a Java ZonedDateTime.
+    * This should be a functor to allow building wrapping datastructures but is limited to supported ones for now.
+    */
+  sealed private trait ToEndDate[T] {
+    def fromZonedDateTime(date: ZonedDateTime): T
+  }
+  private object ToEndDate          {
+    def apply[T](implicit ev: ToEndDate[T]) = ev
+
+    implicit val id:         ToEndDate[ZonedDateTime] = new ToEndDate[ZonedDateTime] {
+      def fromZonedDateTime(date: ZonedDateTime): ZonedDateTime = date
+    }
+    implicit def dateCounts: ToEndDate[DateCounts]    = new ToEndDate[DateCounts] {
+      // single date count is has a count of 1, upon aggregation counts will be added
+      def fromZonedDateTime(date: ZonedDateTime): DateCounts = DateCounts(Map(date -> DateCount(date, 1)))
+    }
+  }
+
+  def fromLicenseInfo[T: ToEndDate: Aggregate](info: PluginLicenseInfo): GlobalPluginsLicense[T] = {
+    new GlobalPluginsLicense[T](
       Some(NonEmptyChunk(info.licensee)),
       Some(info.startDate.toJava),
-      Some(info.endDate.toJava),
+      Some(ToEndDate[T].fromZonedDateTime(info.endDate.toJava)),
       info.maxNodes
-    )
+    ) {}
   }
 
-  def empty = JsonGlobalPluginLimits(None, None, None, None)
-  // from a list of plugins, create the global limits
-  def getGlobalLimits(licenses: Seq[PluginLicenseInfo]): Option[JsonGlobalPluginLimits] = {
+  def empty = new GlobalPluginsLicense[Unit](None, None, None, None) {}
+
+  def from[T: ToEndDate: Aggregate](licenses: Seq[PluginLicenseInfo]): Option[GlobalPluginsLicense[T]] = {
     NonEmptyChunk
       .fromIterableOption(licenses)
-      .map(getGlobalLimits(_))
+      .map(from(_))
       .flatMap(r => Option.when(r != empty)(r))
   }
 
-  def getGlobalLimits(licenses: NonEmptyChunk[PluginLicenseInfo]): JsonGlobalPluginLimits = {
-    val res                     = licenses.reduceMapLeft(fromLicenseInfo) { case (lim, lic) => lim.combine(fromLicenseInfo(lic)) }
-    val sortedDistinctLicensees = res.licensees.map(_.sorted.distinct).flatMap(NonEmptyChunk.fromChunk)
-    res.copy(licensees = sortedDistinctLicensees)
+  private def from[T: ToEndDate: Aggregate](licenses: NonEmptyChunk[PluginLicenseInfo]): GlobalPluginsLicense[T] = {
+    licenses
+      .reduceMapLeft(fromLicenseInfo(_)) { case (lim, lic) => lim.aggregate(fromLicenseInfo(lic)) }
+      .sortDistinctLicensees
   }
 
+  // this is map2
   private def comp[A](a: Option[A], b: Option[A], compare: (A, A) => A): Option[A] = (a, b) match {
     case (None, None)       => None
     case (Some(x), None)    => Some(x)
@@ -163,6 +238,28 @@ object JsonGlobalPluginLimits {
   }
 }
 
+/*
+ * Global limit information about plugins (the most restrictive, with the minimum end date )
+ */
+final case class JsonGlobalPluginLimits(
+    override val licensees: Option[NonEmptyChunk[String]],
+    override val startDate: Option[ZonedDateTime],
+    override val endDate:   Option[ZonedDateTime],
+    override val maxNodes:  Option[Int]
+) extends GlobalPluginsLicense[ZonedDateTime](licensees, startDate, endDate, maxNodes)
+
+object JsonGlobalPluginLimits {
+  import DateFormaterService.json.encoderZonedDateTime
+
+  implicit val encoderGlobalPluginLimits: JsonEncoder[JsonGlobalPluginLimits] = DeriveJsonEncoder.gen
+
+  // upcast the global licenses after aggregation
+  def fromGlobalLicense(license: GlobalPluginsLicense[ZonedDateTime]): JsonGlobalPluginLimits = {
+    import license.*
+    JsonGlobalPluginLimits(licensees, startDate, endDate, maxNodes)
+  }
+}
+
 sealed trait PluginSystemStatus {
   def value: String
 }
@@ -415,7 +512,7 @@ object PluginManagementError       {
 }
 
 case class PluginId(value: String) extends AnyVal
-object PluginId                  {
+object PluginId           {
   implicit val decoder:     JsonDecoder[PluginId]         = JsonDecoder[String].mapOrFail(parse)
   implicit val encoder:     JsonEncoder[PluginId]         = JsonEncoder[String].contramap(_.value)
   implicit val transformer: Transformer[PluginId, String] = Transformer.derive[PluginId, String]
@@ -433,18 +530,64 @@ object PluginId                  {
   }
 }
 
+final case class JsonPluginsLicense(
+    licensees: Option[NonEmptyChunk[String]],
+    startDate: Option[ZonedDateTime],
+    endDates:  Option[GlobalPluginsLicense.DateCounts],
+    maxNodes:  Option[Int]
+)
+object JsonPluginsLicense {
+
+  import DateFormaterService.JodaTimeToJava
+  import GlobalPluginsLicense.*
+
+  implicit val dateFieldEncoder: JsonFieldEncoder[ZonedDateTime] =
+    JsonFieldEncoder[String].contramap(DateFormaterService.serializeZDT)
+
+  implicit val dateCountEncoder:  JsonEncoder[DateCount]          = DeriveJsonEncoder.gen[DateCount]
+  implicit val dateCountsEncoder: JsonEncoder[DateCounts]         = JsonEncoder[Chunk[DateCount]].contramap(m => Chunk.from(m.values))
+  implicit val encoder:           JsonEncoder[JsonPluginsLicense] = DeriveJsonEncoder.gen[JsonPluginsLicense]
+
+  // copy the endDate to endDates
+  def from(global: GlobalPluginsLicenseCounts): JsonPluginsLicense = {
+    import global.*
+    JsonPluginsLicense(licensees, startDate, endDate, maxNodes)
+  }
+}
+
+/**
+  * Global license information about many plugins, aggregated such that :
+  *  - more than 1 distinct licensees can exist for the collection of all plugins
+  *  - end date of plugins licenses are aggregated with counts, so that we know how many plugin license expire at a given date
+  **/
+final case class GlobalPluginsLicenseCounts(
+    override val licensees: Option[NonEmptyChunk[String]],
+    override val startDate: Option[ZonedDateTime],
+    // serves for the aggregation, but the json field is "endDates"
+    override val endDate:   Option[GlobalPluginsLicense.DateCounts],
+    override val maxNodes:  Option[Int]
+) extends GlobalPluginsLicense(licensees, startDate, endDate, maxNodes)
+
+object GlobalPluginsLicenseCounts {
+  // upcast the global licenses after aggregation
+  def fromGlobalLicense(license: GlobalPluginsLicense[GlobalPluginsLicense.DateCounts]): GlobalPluginsLicenseCounts = {
+    import license.*
+    GlobalPluginsLicenseCounts(licensees, startDate, endDate, maxNodes)
+  }
+}
+
 final case class JsonPluginsSystemDetails(
-    license: Option[JsonGlobalPluginLimits],
+    license: Option[JsonPluginsLicense],
     plugins: Chunk[JsonPluginSystemDetails]
 )
-object JsonPluginsSystemDetails  {
+object JsonPluginsSystemDetails   {
   import JsonPluginSystemDetails.*
 
   implicit val encoder: JsonEncoder[JsonPluginsSystemDetails] = DeriveJsonEncoder.gen[JsonPluginsSystemDetails]
 
   def buildDetails(plugins: Chunk[JsonPluginSystemDetails]): JsonPluginsSystemDetails = {
-    val limits = JsonGlobalPluginLimits.getGlobalLimits(plugins.flatMap(_.license))
-    JsonPluginsSystemDetails(limits, plugins)
+    val global = GlobalPluginsLicense.from[GlobalPluginsLicense.DateCounts](plugins.flatMap(_.license))
+    JsonPluginsSystemDetails(global.map(g => JsonPluginsLicense.from(GlobalPluginsLicenseCounts.fromGlobalLicense(g))), plugins)
   }
 }
 final case class JsonPluginSystemDetails(
@@ -465,7 +608,7 @@ final case class JsonPluginManagementError(
     error:   String,
     message: String
 )
-object JsonPluginManagementError {
+object JsonPluginManagementError  {
   implicit val transformer: Transformer[PluginManagementError, JsonPluginManagementError] = {
     Transformer
       .define[PluginManagementError, JsonPluginManagementError]
@@ -579,7 +722,7 @@ object RudderPackagePlugin     {
 trait RudderPackageService {
   import RudderPackageService.*
 
-  def updateBase(): IOResult[Option[CredentialError]]
+  def update(): IOResult[Option[CredentialError]]
 
   def listAllPlugins(): IOResult[Chunk[RudderPackagePlugin]]
 
@@ -619,7 +762,7 @@ class RudderPackageCmdService(configCmdLine: String) extends RudderPackageServic
     case h :: tail => Right((h, tail))
   }
 
-  override def updateBase(): IOResult[Option[CredentialError]] = {
+  override def update(): IOResult[Option[CredentialError]] = {
     // In case of error we need to check the result
     for {
       res          <- runCmd("update" :: Nil)
@@ -699,8 +842,8 @@ class RudderPackageCmdService(configCmdLine: String) extends RudderPackageServic
   */
 trait PluginSystemService {
 
-  def list(): IOResult[Either[CredentialError, Chunk[JsonPluginSystemDetails]]]
-
+  def updateIndex(): IOResult[Option[CredentialError]]
+  def list():        IOResult[Chunk[JsonPluginSystemDetails]]
   def install(plugins:     Chunk[PluginId]): IOResult[Unit]
   def remove(plugins:      Chunk[PluginId]): IOResult[Unit]
   def updateStatus(status: PluginSystemStatus, plugins: Chunk[PluginId]): IOResult[Unit]
@@ -711,8 +854,12 @@ trait PluginSystemService {
   * Implementation for tests, will do any operation without any error
   */
 class InMemoryPluginSystemService(ref: Ref[Map[PluginId, JsonPluginSystemDetails]]) extends PluginSystemService {
-  override def list(): UIO[Either[CredentialError, Chunk[JsonPluginSystemDetails]]] = {
-    ref.get.map(m => Right(Chunk.fromIterable(m.values)))
+  override def updateIndex(): IOResult[Option[CredentialError]] = {
+    ZIO.none
+  }
+
+  override def list(): UIO[Chunk[JsonPluginSystemDetails]] = {
+    ref.get.map(m => Chunk.fromIterable(m.values))
   }
 
   override def install(plugins: Chunk[PluginId]): UIO[Unit] = {

webapp/sources/rudder/rudder-rest/src/main/scala/com/normation/rudder/rest/EndpointsDefinition.scala

@@ -649,6 +649,12 @@ sealed trait PluginInternalApi extends EnumEntry with EndpointSchema with Intern
 }
 object PluginInternalApi       extends Enum[PluginInternalApi] with ApiModuleProvider[PluginInternalApi] {
 
+  case object UpdatePluginsIndex  extends PluginInternalApi with ZeroParam with StartsAtVersion21 with SortIndex {
+    val z: Int = implicitly[Line].value
+    val description    = "Update plugins index and licenses"
+    val (action, path) = POST / "pluginsinternal" / "update"
+    override def dataContainer: Option[String] = None
+  }
   case object ListPlugins         extends PluginInternalApi with ZeroParam with StartsAtVersion21 with SortIndex {
     val z: Int = implicitly[Line].value
     val description    = "List all plugins"