* attempt to improve docker isolation in tests

* minor refactor of global environments

Directory.Build.props

@@ -9,7 +9,7 @@
         <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
         <LangVersion>preview</LangVersion>
 
-        <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+        <Configuration Condition=" '$(Configuration)' == '' ">full</Configuration>
         <DebugType Condition=" '$(Configuration)' == 'Debug' ">full</DebugType>
         <DebugType Condition=" '$(Configuration)' == 'Release' ">pdbonly</DebugType>
 

test/EventStore.Client.Operations.Tests/ShutdownNodeTests.cs

@@ -1,3 +1,5 @@
+using FlakyTest.XUnit.Attributes;
+
 namespace EventStore.Client.Operations.Tests;
 
 public class ShutdownNodeTests : IClassFixture<InsecureClientTestFixture> {
@@ -9,8 +11,8 @@ public ShutdownNodeTests(ITestOutputHelper output, InsecureClientTestFixture fix
 	[Fact]
 	public async Task shutdown_does_not_throw() => 
 		await Fixture.Operations.ShutdownAsync(userCredentials: TestCredentials.Root).ShouldNotThrowAsync();
-
-	[Fact]
+	
+	[MaybeFixedFact(1)]
 	public async Task shutdown_without_credentials_throws() =>
 		await Fixture.Operations.ShutdownAsync().ShouldThrowAsync<AccessDeniedException>();
 }

test/EventStore.Client.Tests.Common/ApplicationInfo.cs

@@ -11,6 +11,9 @@
 
 namespace EventStore.Client;
 
+/// <summary>
+/// Loads configuration and provides information about the application environment.
+/// </summary>
 public static class Application {
 	static Application() {
 		ForegroundColor = ConsoleColor.Magenta;
@@ -27,7 +30,9 @@ static Application() {
 
 		Configuration = builder.Build();
 
-		WriteLine($"CONSOLE: {Environment} configuration loaded with {Configuration.AsEnumerable().Count()} entries from {builder.Sources.Count} sources.");
+		WriteLine($"CONSOLE: {Environment} configuration loaded "
+		        + $"with {Configuration.AsEnumerable().Count()} entries "
+		        + $"from {builder.Sources.Count} sources.");
 
 		IsDevelopment = IsEnvironment(Environments.Development);
 		IsStaging     = IsEnvironment(Environments.Staging);

test/EventStore.Client.Tests.Common/EventStore.Client.Tests.Common.csproj

@@ -13,6 +13,7 @@
     </ItemGroup>
 
     <ItemGroup>
+        <PackageReference Include="FlakyTest.XUnit" Version="2.4.10" />
         <PackageReference Include="JetBrains.Annotations" Version="2023.2.0"/>
         <PackageReference Include="Ductus.FluentDocker" Version="2.10.59"/>
         <PackageReference Include="Polly" Version="8.0.0"/>

test/EventStore.Client.Tests.Common/Extensions/ConfigurationExtensions.cs

@@ -0,0 +1,12 @@
+using Microsoft.Extensions.Configuration;
+
+namespace EventStore.Client.Tests;
+
+public static class ConfigurationExtensions {
+	public static void EnsureValue(this IConfiguration configuration, string key, string defaultValue) {
+		var value = configuration.GetValue<string?>(key);
+
+		if (string.IsNullOrEmpty(value))
+			configuration[key] = defaultValue;
+	}
+}

test/EventStore.Client.Tests.Common/Fixtures/DatabaseWarmup.cs

@@ -13,18 +13,22 @@ static class DatabaseWarmup<T> where T : EventStoreClientBase {
     static readonly SemaphoreSlim      Semaphore = new(1, 1);
     static readonly ILogger            Logger    = Log.ForContext(SourceContextPropertyName, typeof(T).Name);
 
+    static DatabaseWarmup() {
+		AppDomain.CurrentDomain.DomainUnload += (_, _) => Completed.Set(false);
+	}
+
     public static async Task TryExecuteOnce(T client, Func<CancellationToken, Task> action, CancellationToken cancellationToken = default) {
 	    await Semaphore.WaitAsync(cancellationToken);
 
 	    try {
-		    if (!Completed.EnsureCalledOnce()) {
+		    // if (!Completed.EnsureCalledOnce()) {
 			    Logger.Warning("*** Warmup started ***");
 			    await TryExecute(client, action, cancellationToken);
 			    Logger.Warning("*** Warmup completed ***");
-		    }
-		    else {
-			    Logger.Information("*** Warmup skipped ***");
-		    }
+		    // }
+		    // else {
+			   //  Logger.Information("*** Warmup skipped ***");
+		    // }
 	    }
 	    catch (Exception ex) {
 		    if (Application.DebuggerIsAttached) {

test/EventStore.Client.Tests.Common/Fixtures/EventStoreFixture.cs

@@ -4,7 +4,7 @@
 
 namespace EventStore.Client.Tests;
 
-public record EventStoreFixtureOptions(EventStoreClientSettings ClientSettings, IDictionary<string, string> Environment) {
+public record EventStoreFixtureOptions(EventStoreClientSettings ClientSettings, IDictionary<string, string?> Environment) {
 	public EventStoreFixtureOptions RunInMemory(bool runInMemory = true) =>
 		this with { Environment = Environment.With(x => x["EVENTSTORE_MEM_DB"] = runInMemory.ToString()) };
 
@@ -39,7 +39,7 @@ public EventStoreFixture() : this(options => options) { }
 	protected EventStoreFixture(ConfigureFixture configure) {
 		// TODO SS: should I verify the certificates exist here?
 		if (GlobalEnvironment.UseExternalServer) {
-			Options = new(new(), new Dictionary<string, string>());
+			Options = new(new(), new Dictionary<string, string?>());
 			Service = new TestBypassService();
 		}
 
@@ -87,11 +87,10 @@ public void CaptureTestRun(ITestOutputHelper outputHelper) {
 		var testRunId = Logging.CaptureLogs(outputHelper);
 		TestRuns.Add(testRunId);
 		Logger.Information(">>> Test Run {testRunId} {Operation} <<<", testRunId, "starting");
+		Service.ReportStatus();
 	}
 
-	public async Task InitializeAsync() {
-		await Service.Start().ShouldNotThrowAsync();
-
+	async Task WarmUp() {
 		Logger.Information("*** !!! Warming up database !!! ***");
 
 		Users = new(ClientSettings);
@@ -110,11 +109,16 @@ public async Task InitializeAsync() {
 
 		Operations = new(ClientSettings);
 		await Operations.WarmUp().ShouldNotThrowAsync();
+	}
 
-		Logger.Information("Setup completed");
+	public async Task InitializeAsync() {
+		await Service.Start().ShouldNotThrowAsync();
+
+		await WarmUp();
 
 		await OnSetup().ShouldNotThrowAsync("Failed to run OnSetup!");
 	}
+
 	public async Task DisposeAsync() {
 		try {
 			await OnTearDown();

test/EventStore.Client.Tests.Common/Fixtures/EventStoreTestCluster.cs

@@ -18,7 +18,7 @@ public static EventStoreFixtureOptions DefaultOptions() {
 			.With(x => x.ConnectivitySettings.MaxDiscoverAttempts = 30)
 			.With(x => x.ConnectivitySettings.DiscoveryInterval = TimeSpan.FromSeconds(1));
 
-		var defaultEnvironment = new Dictionary<string, string>(GlobalEnvironment.Variables) {
+		var defaultEnvironment = new Dictionary<string, string?>(GlobalEnvironment.Variables) {
 			["ES_CERTS_CLUSTER"]                        = Path.Combine(Environment.CurrentDirectory, "certs-cluster"),
 			["EVENTSTORE_CLUSTER_SIZE"]                 = "3",
 			["EVENTSTORE_INT_TCP_PORT"]                 = "1112",

test/EventStore.Client.Tests.Common/Fixtures/EventStoreTestNode.cs

@@ -1,3 +1,4 @@
+using System.Net;
 using Ductus.FluentDocker.Builders;
 using Ductus.FluentDocker.Model.Builders;
 using EventStore.Client.Tests.FluentDocker;
@@ -9,28 +10,37 @@ namespace EventStore.Client.Tests;
 public class EventStoreTestNode(EventStoreFixtureOptions? options = null) : TestContainerService {
 	EventStoreFixtureOptions Options { get; } = options ?? DefaultOptions();
 
-	static int _port = 2213;
+	static int _port = 2212;
 	static int NextPort() => Interlocked.Increment(ref _port);
 
 	public static EventStoreFixtureOptions DefaultOptions() {
 		const string connString = "esdb://admin:changeit@localhost:{port}/?tlsVerifyCert=false";
+
+		var port = NextPort().ToString();
 
 		var defaultSettings = EventStoreClientSettings
-			.Create(connString.Replace("{port}", NextPort().ToString()))
+			.Create(connString.Replace("{port}", port))
 			.With(x => x.LoggerFactory = new SerilogLoggerFactory(Log.Logger))
 			.With(x => x.DefaultDeadline = Application.DebuggerIsAttached ? new TimeSpan?() : TimeSpan.FromSeconds(180))
 			.With(x => x.ConnectivitySettings.MaxDiscoverAttempts = 30)
 			.With(x => x.ConnectivitySettings.DiscoveryInterval = TimeSpan.FromSeconds(1));
 
-        var defaultEnvironment = new Dictionary<string, string>(GlobalEnvironment.Variables) {
-	        ["EVENTSTORE_MEM_DB"]                       = "true",
-	        ["EVENTSTORE_CHUNK_SIZE"]                   = (1024 * 1024).ToString(),
-	        ["EVENTSTORE_CERTIFICATE_FILE"]             = "/etc/eventstore/certs/node/node.crt",
-	        ["EVENTSTORE_CERTIFICATE_PRIVATE_KEY_FILE"] = "/etc/eventstore/certs/node/node.key",
-	        ["EVENTSTORE_STREAM_EXISTENCE_FILTER_SIZE"] = "10000",
-	        ["EVENTSTORE_STREAM_INFO_CACHE_CAPACITY"]   = "10000",
-	        ["EVENTSTORE_ENABLE_ATOM_PUB_OVER_HTTP"]    = "True"
-        }; 
+		var defaultEnvironment = new Dictionary<string, string?>(GlobalEnvironment.Variables) {
+			// ["EVENTSTORE_HTTP_PORT"]                        = port,
+			// ["EVENTSTORE_ADVERTISE_HTTP_PORT_TO_CLIENT_AS"] = port,
+			["EVENTSTORE_NODE_PORT"]                        = port,
+			["EVENTSTORE_NODE_PORT_ADVERTISE_AS"]           = port,
+			["EVENTSTORE_ADVERTISE_NODE_PORT_TO_CLIENT_AS"] = port,
+			["EVENTSTORE_ENABLE_EXTERNAL_TCP"]              = "false",
+			["EVENTSTORE_MEM_DB"]                           = "true",
+			["EVENTSTORE_CHUNK_SIZE"]                       = (1024 * 1024).ToString(),
+			["EVENTSTORE_CERTIFICATE_FILE"]                 = "/etc/eventstore/certs/node/node.crt",
+			["EVENTSTORE_CERTIFICATE_PRIVATE_KEY_FILE"]     = "/etc/eventstore/certs/node/node.key",
+			["EVENTSTORE_STREAM_EXISTENCE_FILTER_SIZE"]     = "10000",
+			["EVENTSTORE_STREAM_INFO_CACHE_CAPACITY"]       = "10000",
+			["EVENTSTORE_ENABLE_ATOM_PUB_OVER_HTTP"]        = "false",
+			["EVENTSTORE_DISABLE_LOG_FILE"]                 = "true"
+		}; 
 
 		return new(defaultSettings, defaultEnvironment);
 	}
@@ -40,17 +50,22 @@ protected override ContainerBuilder Configure() {
 
 		var port          = Options.ClientSettings.ConnectivitySettings.Address.Port;
 		var certsPath     = Path.Combine(Environment.CurrentDirectory, "certs");
-        var containerName = $"esbd-dotnet-test-{Guid.NewGuid().ToString()[30..]}-{port}";
+        var containerName = $"esbd-dotnet-test-{port}-{Guid.NewGuid().ToString()[30..]}";
 
         CertificatesManager.VerifyCertificatesExist(certsPath); // its ok... no really...
-        
+
         return new Builder()
-            .UseContainer()
-            .UseImage(GlobalEnvironment.DockerImage)
-            .WithName(containerName)
-            .WithEnvironment(env)
-            .MountVolume(certsPath, "/etc/eventstore/certs", MountType.ReadOnly)
-            .ExposePort(port, 2113)
-            .WaitForHealthy(TimeSpan.FromSeconds(60));
-    }
+	        .UseContainer()
+	        .UseImage(Options.Environment["ES_DOCKER_IMAGE"])
+	        .WithName(containerName)
+	        .WithEnvironment(env)
+	        .MountVolume(certsPath, "/etc/eventstore/certs", MountType.ReadOnly)
+	        .ExposePort(port, port) // 2113 
+	        //.WaitForMessageInLog($"========== [\"0.0.0.0:{port}\"] IS LEADER... SPARTA!")
+	        //.WaitForMessageInLog("'ops' user added to $users.")
+	        .WaitForMessageInLog("'admin' user added to $users.");
+			//.WaitForHealthy(TimeSpan.FromSeconds(60));
+			//HEALTHCHECK &{["CMD-SHELL" "curl --fail --insecure https://localhost:2113/health/live || curl --fail http://localhost:2113/health/live || exit 1"] "5s" "5s" "0s" '\x18'}
+
+	}
 }

test/EventStore.Client.Tests.Common/FluentDocker/TestService.cs

@@ -9,6 +9,8 @@ namespace EventStore.Client.Tests.FluentDocker;
 public interface ITestService : IAsyncDisposable {
     Task Start();
     Task Stop();
+
+    void ReportStatus();
 }
 
 /// <summary>
@@ -93,6 +95,27 @@ public virtual async Task Stop() {
         }
     }
 
+    public void ReportStatus() {
+	    if (Service is IContainerService containerService) {
+		    ReportContainerStatus(containerService);
+	    }
+
+	    if (Service is ICompositeService compose) {
+			foreach (var container in compose.Containers) {
+				ReportContainerStatus(container);
+			}
+		}
+
+	    return;
+
+	    void ReportContainerStatus(IContainerService service) {
+		    var cfg = service.GetConfiguration(true);
+		    Logger.Information("Container {Name} {State} Ports: {Ports}", service.Name, service.State, cfg.Config.ExposedPorts.Keys);
+	    }
+
+	    // var docker = Fd.Hosts().Discover().FirstOrDefault(x => x.IsNative || x.Name == "default")!;
+    }
+
     public virtual ValueTask DisposeAsync() {
         try {
             Network?.Dispose();

test/EventStore.Client.Tests.Common/GlobalEnvironment.cs

@@ -5,48 +5,44 @@ namespace EventStore.Client.Tests;
 
 public static class GlobalEnvironment {
 	static GlobalEnvironment() {
-		Variables = FromConfiguration(Application.Configuration);
+		EnsureDefaults(Application.Configuration);
 
-		UseCluster        = false; // Application.Configuration.GetValue("ES_USE_CLUSTER", false);
-		UseExternalServer = Application.Configuration.GetValue("ES_USE_EXTERNAL_SERVER", false);
-		DockerImage       = Variables["ES_DOCKER_IMAGE"];
-		DbLogFormat       = Variables["EVENTSTORE_DB_LOG_FORMAT"];
+		UseCluster        = Application.Configuration.GetValue<bool>("ES_USE_CLUSTER");
+		UseExternalServer = Application.Configuration.GetValue<bool>("ES_USE_EXTERNAL_SERVER");
+		DockerImage       = Application.Configuration.GetValue<string>("ES_DOCKER_IMAGE")!;
+		DbLogFormat       = Application.Configuration.GetValue<string>("EVENTSTORE_DB_LOG_FORMAT")!;
+
+		Variables = Application.Configuration.AsEnumerable()
+			.Where(x => x.Key.StartsWith("ES_") || x.Key.StartsWith("EVENTSTORE_"))
+			.OrderBy(x => x.Key)
+			.ToImmutableDictionary(x => x.Key, x => x.Value ?? string.Empty)!;
+
+		return;
+
+		static void EnsureDefaults(IConfiguration configuration) {
+			configuration.EnsureValue("ES_USE_CLUSTER", "false");
+			configuration.EnsureValue("ES_USE_EXTERNAL_SERVER", "false");
+
+			configuration.EnsureValue("ES_DOCKER_REGISTRY", "ghcr.io/eventstore/eventstore");
+			configuration.EnsureValue("ES_DOCKER_TAG", "ci");
+			configuration.EnsureValue("ES_DOCKER_IMAGE", $"{configuration["ES_DOCKER_REGISTRY"]}:{configuration["ES_DOCKER_TAG"]}");
+
+			configuration.EnsureValue("EVENTSTORE_MEM_DB", "false");
+			configuration.EnsureValue("EVENTSTORE_RUN_PROJECTIONS", "None");
+			configuration.EnsureValue("EVENTSTORE_START_STANDARD_PROJECTIONS", "false");
+			configuration.EnsureValue("EVENTSTORE_DB_LOG_FORMAT", "V2");
+			configuration.EnsureValue("EVENTSTORE_LOG_LEVEL", "Verbose");
+			configuration.EnsureValue("EVENTSTORE_TRUSTED_ROOT_CERTIFICATES_PATH", "/etc/eventstore/certs/ca");
+		}
 	}
 
-	public static ImmutableDictionary<string, string> Variables { get; }
+	public static ImmutableDictionary<string, string?> Variables { get; }
 
 	public static bool   UseCluster        { get; }
 	public static bool   UseExternalServer { get; }
 	public static string DockerImage       { get; }
 	public static string DbLogFormat       { get; }
 
-	public static ImmutableDictionary<string, string> FromConfiguration(IConfiguration configuration) {
-		var env = configuration.AsEnumerable()
-			.Where(x => x.Key.StartsWith("ES_") || x.Key.StartsWith("EVENTSTORE_"))
-			.ToDictionary(x => x.Key, x => x.Value ?? string.Empty);
-
-		EnsureSet(env, "ES_USE_CLUSTER", "false");
-		EnsureSet(env, "ES_USE_EXTERNAL_SERVER", "false");
-
-		EnsureSet(env, "ES_DOCKER_REGISTRY", "ghcr.io/eventstore/eventstore");
-		EnsureSet(env, "ES_DOCKER_TAG", "ci");
-		EnsureSet(env, "ES_DOCKER_IMAGE", $"{env["ES_DOCKER_REGISTRY"]}:{env["ES_DOCKER_TAG"]}");
-
-		EnsureSet(env, "EVENTSTORE_MEM_DB", "false");
-		EnsureSet(env, "EVENTSTORE_RUN_PROJECTIONS", "None");
-		EnsureSet(env, "EVENTSTORE_START_STANDARD_PROJECTIONS", "false");
-		EnsureSet(env, "EVENTSTORE_DB_LOG_FORMAT", "V2");
-		EnsureSet(env, "EVENTSTORE_LOG_LEVEL", "Verbose");
-		EnsureSet(env, "EVENTSTORE_TRUSTED_ROOT_CERTIFICATES_PATH", "/etc/eventstore/certs/ca");
-
-		return env.ToImmutableDictionary();
-
-		static void EnsureSet(IDictionary<string, string> dic, string key, string value) {
-			if (!dic.TryGetValue(key, out var actualValue) || string.IsNullOrEmpty(actualValue))
-				dic[key] = value;
-		}
-	}
-
 	#region . Obsolete .
 
 	//[Obsolete("Use the EventStoreFixture instead so you don't have to use this method.", false)]

test/EventStore.Client.Tests.Common/Fixtures/Logging.cs → test/EventStore.Client.Tests.Common/Logging.cs

@@ -4,7 +4,6 @@
 using Serilog;
 using Serilog.Events;
 using Serilog.Formatting.Display;
-using Serilog.Sinks.SystemConsole.Themes;
 using Xunit.Sdk;
 
 namespace EventStore.Client.Tests;