Docker cleanups (#2174)

* container state

* docker exceptions

* wip

distage/distage-core-api/src/main/scala/izumi/distage/model/exceptions/runtime/IntegrationCheckException.scala

@@ -14,3 +14,4 @@ class IntegrationCheckException(val failures: NEList[ResourceCheck.Failure], cap
   def this(failures: NEList[ResourceCheck.Failure]) = this(failures, true)
   def this(failure: ResourceCheck.Failure) = this(NEList(failure))
 }
+

distage/distage-core-api/src/main/scala/izumi/distage/model/exceptions/runtime/NonCriticalIntegrationFailure.scala

@@ -0,0 +1,5 @@
+package izumi.distage.model.exceptions.runtime
+
+trait NonCriticalIntegrationFailure { this: Throwable =>
+  def asThrowable: Throwable = this
+}

distage/distage-framework-docker/src/main/scala/izumi/distage/docker/healthcheck/ContainerHealthCheck.scala

@@ -59,7 +59,7 @@ object ContainerHealthCheck {
 
     final case class Failed(failure: String = "No diagnostics available") extends BadHealthcheck
 
-    final case class Terminated(failure: String) extends BadHealthcheck
+    final case class Terminated(failure: String, cause: ContainerState) extends BadHealthcheck
 
     final case class UnavailableWithMeta(
       unavailablePorts: UnavailablePorts,
@@ -82,11 +82,11 @@ object ContainerHealthCheck {
         case ContainerState.Running =>
           performCheck
 
-        case ContainerState.Exited(status) =>
-          HealthCheckResult.Terminated(s"Container unexpectedly exited with exit code=$status.")
+        case s: ContainerState.Exited =>
+          HealthCheckResult.Terminated(s"Container unexpectedly exited with exit code=${s.status}.", s)
 
-        case ContainerState.NotFound =>
-          HealthCheckResult.Terminated("Container not found, expected to find a running container.")
+        case s: ContainerState.NotFound.type =>
+          HealthCheckResult.Terminated("Container not found, expected to find a running container.", s)
       }
     }
   }

distage/distage-framework-docker/src/main/scala/izumi/distage/docker/healthcheck/ExitSuccessCheck.scala

@@ -13,15 +13,15 @@ final class ExitSuccessCheck(exitCode: Int) extends ContainerHealthCheck {
         logger.info(s"$container successfully exited, health check passed.")
         HealthCheckResult.Passed
 
-      case ContainerState.Exited(status) =>
-        HealthCheckResult.Terminated(s"Container terminated with unexpected code. Code=$status, expected=$exitCode")
-
       case ContainerState.Running =>
         logger.info(s"$container is still running, expected container to exit, health check failed.")
         HealthCheckResult.Failed("Container is still running, expected container to exit")
 
+      case ContainerState.Exited(status) =>
+        HealthCheckResult.Terminated(s"Container terminated with unexpected code. Code=$status, expected=$exitCode", state)
+
       case ContainerState.NotFound =>
-        HealthCheckResult.Terminated("Container not found, expected container to exit with status code 0.")
+        HealthCheckResult.Terminated("Container not found, expected container to exit with status code 0.", state)
 
     }
   }

distage/distage-framework-docker/src/main/scala/izumi/distage/docker/impl/ContainerResource.scala

@@ -7,7 +7,7 @@ import izumi.distage.docker.healthcheck.ContainerHealthCheck.HealthCheckResult.G
 import izumi.distage.docker.healthcheck.ContainerHealthCheck.{HealthCheckResult, VerifiedContainerConnectivity}
 import izumi.distage.docker.impl.ContainerResource.PortDecl
 import izumi.distage.docker.impl.DockerClientWrapper.{ContainerDestroyMeta, RemovalReason}
-import izumi.distage.docker.model.Docker
+import izumi.distage.docker.model.{Docker, DockerFailureCause, DockerFailureException, DockerTimeoutException}
 import izumi.distage.docker.model.Docker.*
 import izumi.distage.docker.{DockerConst, DockerContainer}
 import izumi.distage.model.definition.Lifecycle
@@ -129,8 +129,8 @@ open class ContainerResource[F[_], Tag](
               Right(out)
             }
 
-          case Right(HealthCheckResult.Terminated(failure)) =>
-            F.fail(new RuntimeException(s"$container terminated with failure: $failure"))
+          case Right(HealthCheckResult.Terminated(failure, state)) =>
+            F.fail(DockerFailureException(s"$container terminated with failure: $failure", DockerFailureCause.Terminated(state)))
 
           case Right(last) =>
             val maxAttempts = config.healthCheckMaxAttempts
@@ -141,7 +141,7 @@ open class ContainerResource[F[_], Tag](
             } else {
               last match {
                 case HealthCheckResult.Failed(failure) =>
-                  F.fail(new TimeoutException(s"Health checks failed after $maxAttempts attempts for $container: $failure"))
+                  F.fail(DockerTimeoutException(s"Health checks failed after $maxAttempts attempts for $container: $failure"))
 
                 case HealthCheckResult.UnavailableWithMeta(unavailablePorts, unverifiedPorts) =>
                   val sb = new StringBuilder()
@@ -170,21 +170,30 @@ open class ContainerResource[F[_], Tag](
                   }
                   F.fail(new TimeoutException(sb.toString()))
 
-                case HealthCheckResult.Terminated(failure) =>
-                  F.fail(new RuntimeException(s"Unexpected condition: $container terminated with failure: $failure"))
+                case HealthCheckResult.Terminated(failure, state) =>
+                  F.fail(DockerFailureException(s"Unexpected condition: $container terminated with failure: $failure", DockerFailureCause.Terminated(state)))
 
                 case impossible: GoodHealthcheck =>
-                  F.fail(new TimeoutException(s"BUG: good healthcheck $impossible while health checks failed after $maxAttempts attempts: $container"))
+                  F.fail(
+                    DockerFailureException(
+                      s"BUG: good healthcheck $impossible while health checks failed after $maxAttempts attempts: $container",
+                      DockerFailureCause.Bug,
+                    )
+                  )
               }
             }
 
           case Left(t) =>
-            F.fail(new RuntimeException(s"$container failed due to exception: ${t.stacktraceString}", t))
+            F.fail(DockerFailureException(s"$container failed due to exception: ${t.stacktraceString}", DockerFailureCause.Throwed(t), t))
         }
   }
 
   private def lostDependencies(inspection: InspectContainerResponse): Boolean = {
-    Option(inspection.getConfig.getLabels.get(DockerConst.Labels.dependencies)).filterNot(_.isEmpty).map(_.split(';')) match {
+    Option(inspection.getConfig)
+      .map(_.getLabels)
+      .map(_.get(DockerConst.Labels.dependencies))
+      .filterNot(_.isEmpty)
+      .map(_.split(';')) match {
       case Some(value) =>
         !value.forall {
           id =>
@@ -374,7 +383,10 @@ open class ContainerResource[F[_], Tag](
 
         maybeMappedPorts match {
           case Left(value) =>
-            throw new RuntimeException(s"Created container from `$imageName` with ${res.getId -> "id"}, but ports are missing: $value!")
+            throw DockerFailureException(
+              s"Created container from `$imageName` with ${res.getId -> "id"}, but ports are missing: $value!",
+              DockerFailureCause.MissingPorts(value),
+            )
 
           case Right(mappedPorts) =>
             val container = DockerContainer[Tag](

distage/distage-framework-docker/src/main/scala/izumi/distage/docker/model/Docker.scala

@@ -308,8 +308,9 @@ object Docker {
 
   sealed trait ContainerState
   object ContainerState {
+    sealed trait ContainerFailure extends ContainerState
     case object Running extends ContainerState
-    case object NotFound extends ContainerState
-    final case class Exited(status: Long) extends ContainerState
+    case object NotFound extends ContainerFailure
+    final case class Exited(status: Long) extends ContainerFailure
   }
 }

distage/distage-framework-docker/src/main/scala/izumi/distage/docker/model/DockerException.scala

@@ -0,0 +1,24 @@
+package izumi.distage.docker.model
+
+import izumi.distage.docker.model.Docker.UnmappedPorts
+import izumi.distage.model.exceptions.runtime.NonCriticalIntegrationFailure
+
+sealed abstract class DockerException(message: String, cause: Throwable) extends RuntimeException(message, cause) with NonCriticalIntegrationFailure {
+  def this(message: String) = this(message, null)
+}
+
+case class DockerFailureException(message: String, explanation: DockerFailureCause, cause: Throwable) extends DockerException(message)
+object DockerFailureException {
+  def apply(message: String, explanation: DockerFailureCause): DockerFailureException = DockerFailureException(message, explanation, null)
+}
+
+case class DockerTimeoutException(message: String) extends DockerException(message)
+
+sealed trait DockerFailureCause
+
+object DockerFailureCause {
+  case class Terminated(state: Docker.ContainerState) extends DockerFailureCause
+  case class Throwed(cause: Throwable) extends DockerFailureCause
+  case class MissingPorts(ports: UnmappedPorts) extends DockerFailureCause
+  case object Bug extends DockerFailureCause
+}

distage/distage-framework-docker/src/test/scala/izumi/distage/testkit/docker/ContainerDependenciesTest.scala

@@ -6,11 +6,12 @@ import izumi.distage.docker.impl.DockerClientWrapper
 import izumi.distage.docker.impl.DockerClientWrapper.{ContainerDestroyMeta, DockerIntegrationCheck, RemovalReason}
 import izumi.distage.docker.model.Docker.ContainerId
 import izumi.distage.docker.modules.DockerSupportModule
-import izumi.distage.model.exceptions.runtime.ProvisioningIntegrationException
 import izumi.fundamentals.platform.functional.Identity
 import logstage.IzLogger
 import org.scalatest.wordspec.AnyWordSpec
 
+import scala.util.Try
+
 final class ContainerDependenciesTest extends AnyWordSpec {
   "distage-docker should re-create containers with failed dependencies, https://github.com/7mind/izumi/issues/1366" in {
     val module = new ModuleDef {
@@ -20,14 +21,13 @@ final class ContainerDependenciesTest extends AnyWordSpec {
       make[IzLogger].fromValue(IzLogger())
     }
 
-    try {
-      Injector()
-        .produceGet[DockerIntegrationCheck[Identity]](module)
-        .use(_ => ())
-    } catch {
-      case ProvisioningIntegrationException(_) =>
-        assume(false)
-    }
+    assert(
+      Try(
+        Injector()
+          .produceGet[DockerIntegrationCheck[Identity]](module)
+          .use(_ => ())
+      ).isSuccess
+    )
 
     def runContainers(): (ContainerId, ContainerId) = {
       Injector()

distage/distage-testkit-core/src/main/scala/izumi/distage/testkit/runner/impl/services/TestStatusConverter.scala

@@ -1,26 +1,46 @@
 package izumi.distage.testkit.runner.impl.services
 
-import izumi.distage.model.exceptions.runtime.ProvisioningIntegrationException
+import izumi.distage.model.exceptions.runtime.{IntegrationCheckException, NonCriticalIntegrationFailure, ProvisioningException}
 import izumi.distage.testkit.model.{EnvResult, GroupResult, IndividualTestResult, TestStatus}
 import izumi.functional.bio.Exit
+import izumi.fundamentals.collections.nonempty.NEList
+import izumi.fundamentals.platform.integration.ResourceCheck
 
 class TestStatusConverter(
   isTestSkipException: Throwable => Boolean
 ) {
+  object ProvisioningIntegrationException {
+    def unapply(arg: ProvisioningException): Option[NEList[ResourceCheck.Failure]] = {
+      val suppressed = arg.getSuppressed.toVector
 
-  def failLevelInstantiation(cause: GroupResult.EnvLevelFailure): TestStatus.Setup = {
-    cause.failure.toThrowable match {
-      case s if isTestSkipException(s) => // can't match, s is always a ProvisioningException
-        TestStatus.EarlyCancelled(cause, s) // this never happens right now
-      case ProvisioningIntegrationException(failures) =>
-        TestStatus.EarlyIgnoredByPrecondition(cause, failures)
-      case t =>
-        TestStatus.EarlyFailed(cause, t)
+      NEList
+        .from(suppressed.collect { case i: IntegrationCheckException => i.failures.toList }.flatten.toList)
+        .filter(_.size == suppressed.size) // only if all the underlying exceptions are of this type
     }
   }
 
-  def failRuntimePlanning(result: EnvResult.RuntimePlanningFailure): TestStatus.FailedRuntimePlanning = {
-    TestStatus.FailedRuntimePlanning(result)
+  object ProvisioningNonCritIntegrationException {
+    def unapply(arg: ProvisioningException): Option[NEList[Throwable]] = {
+      val suppressed = arg.getSuppressed.toVector
+
+      NEList
+        .from(suppressed.collect { case i: NonCriticalIntegrationFailure => i.asThrowable })
+        .filter(_.size == suppressed.size) // only if all the underlying exceptions are of this type
+    }
+  }
+
+  object CancellationException {
+    def unapply(arg: ProvisioningException): Option[NEList[Throwable]] = {
+      val suppressed = arg.getSuppressed.toVector
+
+      NEList
+        .from(suppressed.collect { case i if isTestSkipException(i) => i })
+        .filter(_.size == suppressed.size) // only if all the underlying exceptions are of this type
+    }
+  }
+
+  def success(s: IndividualTestResult.TestSuccess): TestStatus.Succeed = {
+    TestStatus.Succeed(s)
   }
 
   def failExecution(t: IndividualTestResult.ExecutionFailure): TestStatus.Done = {
@@ -32,18 +52,45 @@ class TestStatusConverter(
     fail(t, throwable, Exit.Trace.ThrowableTrace(throwable))
   }
 
+  def failRuntimePlanning(result: EnvResult.RuntimePlanningFailure): TestStatus.FailedRuntimePlanning = {
+    TestStatus.FailedRuntimePlanning(result)
+  }
+
+  def failLevelInstantiation(cause: GroupResult.EnvLevelFailure): TestStatus.Setup = {
+    val asThrowable = cause.failure.toThrowable
+    asThrowable match {
+      case CancellationException(_) => // can't match, s is always a ProvisioningException
+        TestStatus.EarlyCancelled(cause, asThrowable) // this never happens right now
+
+      case ProvisioningIntegrationException(failures) =>
+        TestStatus.EarlyIgnoredByPrecondition(cause, failures)
+
+      case ProvisioningNonCritIntegrationException(_) =>
+        TestStatus.EarlyCancelled(cause, asThrowable)
+
+      case t =>
+        TestStatus.EarlyFailed(cause, t)
+    }
+  }
+
   private def fail(cause: IndividualTestResult.IndividualTestFailure, exception: Throwable, trace: Exit.Trace[Any]): TestStatus.Done = {
     exception match {
       case s if isTestSkipException(s) =>
         TestStatus.Cancelled(cause, s, trace)
-      case ProvisioningIntegrationException(failures) =>
+
+      case i: IntegrationCheckException =>
+        TestStatus.IgnoredByPrecondition(cause, i.failures)
+      case ProvisioningIntegrationException(failures) => // TODO: can this match here?..
         TestStatus.IgnoredByPrecondition(cause, failures)
+
+      case _: NonCriticalIntegrationFailure =>
+        TestStatus.Cancelled(cause, exception, trace)
+      case ProvisioningNonCritIntegrationException(_) => // TODO: can this match here?..
+        TestStatus.Cancelled(cause, exception, trace)
+
       case e =>
         TestStatus.Failed(cause, e, trace)
     }
   }
 
-  def success(s: IndividualTestResult.TestSuccess): TestStatus.Succeed = {
-    TestStatus.Succeed(s)
-  }
 }