feat(ios): add dataContainerClear (#839)

configuration/src/main/kotlin/com/malinskiy/marathon/config/vendor/VendorConfiguration.kt

@@ -162,6 +162,7 @@ sealed class VendorConfiguration {
         @JsonProperty("compactOutput") val compactOutput: Boolean = false,
         @JsonProperty("rsync") val rsync: RsyncConfiguration = RsyncConfiguration(),
         @JsonProperty("xcodebuildTestArgs") val xcodebuildTestArgs: Map<String, String> = emptyMap(),
+        @JsonProperty("dataContainerClear") val dataContainerClear: Boolean = false,
         @JsonProperty("testParserConfiguration") val testParserConfiguration: com.malinskiy.marathon.config.vendor.ios.TestParserConfiguration = com.malinskiy.marathon.config.vendor.ios.TestParserConfiguration.NmTestParserConfiguration(),
 
         @JsonProperty("signing") val signing: SigningConfiguration = SigningConfiguration(),

docs/docs/ios/configure.md

@@ -357,7 +357,7 @@ timeoutConfiguration:
   screenshot: PT10S
   video: PT300S
   erase: PT30S
-  shutdown: PT30S 
+  shutdown: PT30S
   delete: PT30S
   create: PT30S
   boot: PT30S
@@ -454,6 +454,15 @@ rsync:
   remotePath: "/usr/bin/rsync-custom"
 ```
 
+### Clear state between test batch executions
+By default, marathon does not clear state between test batch executions. To mitigate potential test side effects, one could add an option to
+clear the container data between test runs. Keep in mind that test side effects might still be present.
+If you want to isolate tests even further, then you should consider reducing the batch size.
+
+```yaml
+dataContainerClear: true
+```
+
 ### Test parser
 
 :::tip

vendor/vendor-ios/src/main/kotlin/com/malinskiy/marathon/ios/AppleApplicationInstaller.kt

@@ -6,6 +6,7 @@ import com.malinskiy.marathon.config.vendor.VendorConfiguration
 import com.malinskiy.marathon.config.vendor.ios.TestType
 import com.malinskiy.marathon.exceptions.DeviceSetupException
 import com.malinskiy.marathon.execution.withRetry
+import com.malinskiy.marathon.ios.extensions.testBundle
 import com.malinskiy.marathon.ios.model.Sdk
 import com.malinskiy.marathon.ios.xctestrun.TestRootFactory
 import com.malinskiy.marathon.log.MarathonLogging
@@ -18,9 +19,9 @@ class AppleApplicationInstaller(
     private val logger = MarathonLogging.logger {}
 
     suspend fun prepareInstallation(device: AppleSimulatorDevice, useXctestParser: Boolean = false) {
-        val bundle = vendorConfiguration.bundle ?: throw ConfigurationException("no xctest found for configuration")
+        val bundle = vendorConfiguration.testBundle() ?: throw ConfigurationException("no xctest found for configuration")
 
-        val xctest = bundle.xctest
+        val xctest = bundle.testApplication
         logger.debug { "Moving xctest to ${device.serialNumber}" }
         val remoteXctest = device.remoteFileManager.remoteXctestFile()
         withRetry(3, 1000L) {
@@ -46,7 +47,7 @@ class AppleApplicationInstaller(
         TestRootFactory(device, vendorConfiguration).generate(testType, bundle, useXctestParser)
         grantPermissions(device)
 
-        bundle.extraApplications?.forEach {
+        vendorConfiguration.bundle?.extraApplications?.forEach {
             if (it.isDirectory && it.extension == "app") {
                 logger.debug { "Installing extra application $it to ${device.serialNumber}" }
                 val remoteExtraApplication = device.remoteFileManager.remoteExtraApplication(it.name)

vendor/vendor-ios/src/main/kotlin/com/malinskiy/marathon/ios/AppleDeviceTestRunner.kt

@@ -18,7 +18,7 @@ import com.malinskiy.marathon.test.toTestName
 import kotlinx.coroutines.CancellationException
 import kotlinx.coroutines.channels.ReceiveChannel
 
-class AppleDeviceTestRunner(private val device: AppleSimulatorDevice) {
+class AppleDeviceTestRunner(private val device: AppleSimulatorDevice, private val bundleIdentifier: AppleTestBundleIdentifier) {
     private val logger = MarathonLogging.logger {}
 
     suspend fun execute(
@@ -43,7 +43,15 @@ class AppleDeviceTestRunner(private val device: AppleSimulatorDevice) {
         )
         var channel: ReceiveChannel<List<TestEvent>>? = null
         try {
-            clearData(vendorConfiguration)
+            if (vendorConfiguration.dataContainerClear) {
+                val bundleIds = rawTestBatch.tests.map {
+                    bundleIdentifier.identify(it).appId
+                }.toSet()
+                bundleIds.forEach {
+                    device.clearAppContainer(it)
+                }
+            }
+
             listener.beforeTestRun()
 
             val localChannel = device.executeTestRequest(runnerRequest)
@@ -77,11 +85,4 @@ class AppleDeviceTestRunner(private val device: AppleSimulatorDevice) {
             }
         }
     }
-
-    private suspend fun clearData(vendorConfiguration: VendorConfiguration.IOSConfiguration) {
-//        if (vendorConfiguration.eraseSimulatorOnStart) {
-//            device.shutdown()
-//            device.erase()
-//        }
-    }
 }

vendor/vendor-ios/src/main/kotlin/com/malinskiy/marathon/ios/AppleSimulatorDevice.kt

@@ -22,6 +22,7 @@ import com.malinskiy.marathon.execution.listener.LineListener
 import com.malinskiy.marathon.execution.listener.LogListener
 import com.malinskiy.marathon.io.FileManager
 import com.malinskiy.marathon.ios.bin.AppleBinaryEnvironment
+import com.malinskiy.marathon.ios.bin.xcrun.simctl.service.ApplicationService
 import com.malinskiy.marathon.ios.cmd.CommandExecutor
 import com.malinskiy.marathon.ios.cmd.CommandResult
 import com.malinskiy.marathon.ios.cmd.FileBridge
@@ -105,12 +106,13 @@ class AppleSimulatorDevice(
     val arch: Arch
         get() = when {
             sdk == Sdk.IPHONESIMULATOR -> {
-                when(abi) {
+                when (abi) {
                     "x86_64" -> Arch.x86_64
                     "arm64" -> Arch.arm64
                     else -> Arch.arm64
                 }
             }
+
             udid.contains('-') -> Arch.arm64e
             else -> Arch.arm64
         }
@@ -126,7 +128,10 @@ class AppleSimulatorDevice(
     private lateinit var devicePlistPath: String
     private var deviceDescriptor: Map<*, *>? = null
     private val dispatcher by lazy {
-        newFixedThreadPoolContext(vendorConfiguration.threadingConfiguration.deviceThreads, "AppleSimulatorDevice - execution - ${commandExecutor.host.id}")
+        newFixedThreadPoolContext(
+            vendorConfiguration.threadingConfiguration.deviceThreads,
+            "AppleSimulatorDevice - execution - ${commandExecutor.host.id}"
+        )
     }
     override val coroutineContext: CoroutineContext = dispatcher
     override val remoteFileManager: RemoteFileManager = RemoteFileManager(this)
@@ -245,7 +250,7 @@ class AppleSimulatorDevice(
                     try {
                         val (listener, lineListeners) = createExecutionListeners(devicePoolId, testBatch, deferred)
                         executionLineListeners = lineListeners.onEach { addLineListener(it) }
-                        AppleDeviceTestRunner(this@AppleSimulatorDevice).execute(configuration, vendorConfiguration, testBatch, listener)
+                        AppleDeviceTestRunner(this@AppleSimulatorDevice, testBundleIdentifier).execute(configuration, vendorConfiguration, testBatch, listener)
                     } finally {
                         executionLineListeners.forEach { removeLineListener(it) }
                     }
@@ -752,6 +757,17 @@ class AppleSimulatorDevice(
         return binaryEnvironment.xcrun.simctl.privacy.grant(udid, permission, bundleId).successful
     }
 
+    suspend fun clearAppContainer(bundleId: String) {
+        binaryEnvironment.xcrun.simctl.application.terminateApplication(udid, bundleId)
+
+        val containerPath = binaryEnvironment.xcrun.simctl.application.containerPath(udid, bundleId, ApplicationService.ContainerType.DATA)
+        if (containerPath.successful) {
+            remoteFileManager.removeRemotePath(containerPath.combinedStdout.trim())
+        } else {
+            logger.warn { "Failed to clear app container:\nstdout: ${containerPath.combinedStdout}\nstderr: ${containerPath.combinedStderr}" }
+        }
+    }
+
     companion object {
         const val SHARED_PATH = "/tmp/marathon"
     }

vendor/vendor-ios/src/main/kotlin/com/malinskiy/marathon/ios/NmTestParser.kt

@@ -8,6 +8,7 @@ import com.malinskiy.marathon.device.Device
 import com.malinskiy.marathon.exceptions.TestParsingException
 import com.malinskiy.marathon.execution.RemoteTestParser
 import com.malinskiy.marathon.execution.withRetry
+import com.malinskiy.marathon.ios.extensions.testBundle
 import com.malinskiy.marathon.ios.model.AppleTestBundle
 import com.malinskiy.marathon.log.MarathonLogging
 import com.malinskiy.marathon.test.Test
@@ -23,23 +24,11 @@ class NmTestParser(
     private val logger = MarathonLogging.logger(NmTestParser::class.java.simpleName)
 
     override suspend fun extract(device: Device): List<Test> {
-        val app = vendorConfiguration.bundle?.app ?: throw IllegalArgumentException("No application bundle provided")
-        val xctest = vendorConfiguration.bundle?.xctest ?: throw IllegalArgumentException("No test bundle provided")
-        val possibleTestBinaries = xctest.listFiles()?.filter { it.isFile && it.extension == "" }
-            ?: throw ConfigurationException("missing test binaries in xctest folder at $xctest")
-        val testBinary = when (possibleTestBinaries.size) {
-            0 -> throw ConfigurationException("missing test binaries in xctest folder at $xctest")
-            1 -> possibleTestBinaries[0]
-            else -> {
-                logger.warn { "Multiple test binaries present in xctest folder" }
-                possibleTestBinaries.find { it.name == xctest.nameWithoutExtension } ?: possibleTestBinaries.first()
-            }
-        }
-
+        val bundle = vendorConfiguration.testBundle()
         return withRetry(3, 0) {
             try {
                 val device = device as? AppleSimulatorDevice ?: throw ConfigurationException("Unexpected device type for remote test parsing")
-                return@withRetry parseTests(device, xctest, testBinary)
+                return@withRetry parseTests(device, bundle)
             } catch (e: CancellationException) {
                 throw e
             } catch (e: Exception) {
@@ -51,9 +40,10 @@ class NmTestParser(
 
     private suspend fun parseTests(
         device: AppleSimulatorDevice,
-        xctest: File,
-        testBinary: File
+        bundle: AppleTestBundle,
     ): List<Test> {
+        val testBinary = bundle.testBinary
+        val xctest = bundle.testApplication
 
         logger.debug { "Found test binary $testBinary for xctest $xctest" }
 
@@ -88,7 +78,7 @@ class NmTestParser(
             }
 
 
-        val testBundle = AppleTestBundle(vendorConfiguration.bundle?.application, xctest, testBinary)
+        val testBundle = AppleTestBundle(vendorConfiguration.bundle?.application, xctest)
         swiftTests.forEach { testBundleIdentifier.put(it, testBundle) }
         objectiveCTests.forEach { testBundleIdentifier.put(it, testBundle) }
 

vendor/vendor-ios/src/main/kotlin/com/malinskiy/marathon/ios/XCTestParser.kt

@@ -105,7 +105,7 @@ class XCTestParser(
             }
         }
 
-        val testBundle = AppleTestBundle(vendorConfiguration.bundle?.application, xctest, testBinary)
+        val testBundle = AppleTestBundle(vendorConfiguration.bundle?.application, xctest)
         val result = tests.toList()
         result.forEach { testBundleIdentifier.put(it, testBundle) }
 

vendor/vendor-ios/src/main/kotlin/com/malinskiy/marathon/ios/bin/xcrun/simctl/service/ApplicationService.kt

@@ -16,11 +16,24 @@ class ApplicationService (commandExecutor: CommandExecutor,
         )
     }
 
+    suspend fun containerPath(udid: String, bundleId: String, containerType: ContainerType): CommandResult {
+        return criticalExec(
+            timeout = timeoutConfiguration.shell,
+            "get_app_container", udid, bundleId, containerType.value
+        )
+    }
+
+    enum class ContainerType(val value: String) {
+        APPLICATION("app"),
+        DATA("data"),
+        GROUPS("groups")
+    }
+
     /**
      * Terminates a running application with the given bundle ID on this device
      */
-    suspend fun terminateApplication(udid: String, bundleId: String): CommandResult {
-        return criticalExec(
+    suspend fun terminateApplication(udid: String, bundleId: String): CommandResult? {
+        return safeExecute(
             timeout = timeoutConfiguration.shell,
             "terminate", udid, bundleId
         )

vendor/vendor-ios/src/main/kotlin/com/malinskiy/marathon/ios/extensions/ConfigurationExtensions.kt

@@ -0,0 +1,11 @@
+package com.malinskiy.marathon.ios.extensions
+
+import com.malinskiy.marathon.config.vendor.VendorConfiguration
+import com.malinskiy.marathon.ios.model.AppleTestBundle
+
+fun VendorConfiguration.IOSConfiguration.testBundle(): AppleTestBundle {
+    val xctest = bundle?.xctest ?: throw IllegalArgumentException("No test bundle provided")
+    val app = bundle?.app ?: throw IllegalArgumentException("No application bundle provided")
+
+    return AppleTestBundle(app, xctest)
+}

vendor/vendor-ios/src/main/kotlin/com/malinskiy/marathon/ios/model/AppleTestBundle.kt

@@ -1,13 +1,47 @@
 package com.malinskiy.marathon.ios.model
 
+import com.dd.plist.NSDictionary
+import com.malinskiy.marathon.config.exceptions.ConfigurationException
 import com.malinskiy.marathon.execution.bundle.TestBundle
+import com.malinskiy.marathon.ios.plist.PropertyList
+import com.malinskiy.marathon.ios.plist.bundle.BundleInfo
+import com.malinskiy.marathon.log.MarathonLogging
 import java.io.File
 
 class AppleTestBundle(
     val application: File?,
     val testApplication: File,
-    val testBinary: File,
 ) : TestBundle() {
+    private val logger = MarathonLogging.logger {}
     override val id: String
         get() = testApplication.absolutePath
+
+    val applicationBundleInfo: BundleInfo? by lazy {
+        application?.let {
+            PropertyList.from<NSDictionary, BundleInfo>(
+                File(
+                    it,
+                    "Info.plist"
+                )
+            )
+        }
+    }
+    val appId =
+        applicationBundleInfo?.identification?.bundleIdentifier ?: throw ConfigurationException("No bundle identifier specified in $application")
+
+    val testBundleInfo: BundleInfo by lazy { PropertyList.from(File(testApplication, "Info.plist")) }
+    val testBundleId = (testBundleInfo.naming.bundleName ?: testApplication.nameWithoutExtension).replace('-', '_')
+
+    val testBinary: File by lazy {
+        val possibleTestBinaries = testApplication.listFiles()?.filter { it.isFile && it.extension == "" }
+            ?: throw ConfigurationException("missing test binaries in xctest folder at $testApplication")
+        when (possibleTestBinaries.size) {
+            0 -> throw ConfigurationException("missing test binaries in xctest folder at $testApplication")
+            1 -> possibleTestBinaries[0]
+            else -> {
+                logger.warn { "Multiple test binaries present in xctest folder" }
+                possibleTestBinaries.find { it.name == testApplication.nameWithoutExtension } ?: possibleTestBinaries.first()
+            }
+        }
+    }
 }