Chore: Fixed SqLite docker tests (#521)

* Added Grate-in-Docker tests to mid-weight integration tests
* Made 'SetLogger' public, to be able to set from tests
* Some fixes to be able to run publish on the whole solution
* Added UnitTests solution filter file
* Build .slnf in integration test build step
* Disable Docker tests for now in pipeline, as we have to introduce a "build docker image" step before enabling it

.github/workflows/integration-tests.yml

@@ -7,6 +7,11 @@ on:
       - main
     paths-ignore:
       - docs/
+  pull_request:
+    branches:
+      - main
+    paths-ignore:
+      - docs/
   workflow_dispatch:
 
 permissions:
@@ -129,17 +134,8 @@ jobs:
 
     - name: Build Test DLLs ${{ matrix.arch }}
       run: |
-        dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release  ./unittests/CommandLine/CommandLine.MariaDB -o ./integration-tests/${{ matrix.arch }}
-        dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release  ./unittests/CommandLine/CommandLine.SqlServer -o ./integration-tests/${{ matrix.arch }}
-        dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release  ./unittests/CommandLine/CommandLine.Oracle -o ./integration-tests/${{ matrix.arch }}
-        dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release  ./unittests/CommandLine/CommandLine.PostgreSQL -o ./integration-tests/${{ matrix.arch }}
-        dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release  ./unittests/CommandLine/CommandLine.Sqlite -o ./integration-tests/${{ matrix.arch }}
-
-        dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release  ./unittests/MariaDB -o ./integration-tests/${{ matrix.arch }}
-        dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release  ./unittests/SqlServer -o ./integration-tests/${{ matrix.arch }}
-        dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release  ./unittests/Oracle -o ./integration-tests/${{ matrix.arch }}
-        dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release  ./unittests/PostgreSQL -o ./integration-tests/${{ matrix.arch }}
-        dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release  ./unittests/Sqlite -o ./integration-tests/${{ matrix.arch }}
+        dotnet restore -r ${{ matrix.arch }} unittests/UnitTests.slnf
+        dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release --no-restore ./unittests/UnitTests.slnf -o ./integration-tests/${{ matrix.arch }}
 
       env:
         VERSION: ${{ needs.set-version-number.outputs.nuGetVersion }}
@@ -169,17 +165,8 @@ jobs:
 
     - name: Build Test DLLs ${{ matrix.arch }}
       run: |
-        dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release  ./unittests/CommandLine/CommandLine.MariaDB -o ./integration-tests/${{ matrix.arch }}
-        dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release  ./unittests/CommandLine/CommandLine.SqlServer -o ./integration-tests/${{ matrix.arch }}
-        dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release  ./unittests/CommandLine/CommandLine.Oracle -o ./integration-tests/${{ matrix.arch }}
-        dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release  ./unittests/CommandLine/CommandLine.PostgreSQL -o ./integration-tests/${{ matrix.arch }}
-        dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release  ./unittests/CommandLine/CommandLine.Sqlite -o ./integration-tests/${{ matrix.arch }}
-
-        dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release  ./unittests/MariaDB -o ./integration-tests/${{ matrix.arch }}
-        dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release  ./unittests/SqlServer -o ./integration-tests/${{ matrix.arch }}
-        dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release  ./unittests/Oracle -o ./integration-tests/${{ matrix.arch }}
-        dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release  ./unittests/PostgreSQL -o ./integration-tests/${{ matrix.arch }}
-        dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release  ./unittests/Sqlite -o ./integration-tests/${{ matrix.arch }}
+        dotnet restore -r ${{ matrix.arch }} unittests/UnitTests.slnf
+        dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release --no-restore ./unittests/UnitTests.slnf -o ./integration-tests/${{ matrix.arch }}
 
       env:
         VERSION: ${{ needs.set-version-number.outputs.nuGetVersion }}
@@ -360,20 +347,34 @@ jobs:
       run: |
         dotnet test integration-tests/linux-x64/CommandLine.${{ matrix.database.type }}.dll `
         --logger:"xunit;Environment=${{ matrix.database.name }} (docker);LogFilePath=/tmp/test-results/docker/CommandLine.${{ matrix.database.name }}.xml"
-
       env:
         LogLevel: Information
         GrateExecutablePath: ${{ github.workspace }}/executables/linux-x64/grate
         TZ: UTC
         GrateTestConfig__DockerImage: ${{ matrix.database.image }} 
 
+    - name: Docker Test Docker
+      # TODO: We need to integrate build of a docker image to test into the pipeline, and push it to the local registry,
+      # and use that one, to run the integration tests. If not, we are testing the latest published released.
+      #if: ${{ !cancelled() }}
+      if: false
+      shell: pwsh
+      run: |
+        dotnet test integration-tests/linux-x64/Docker.${{ matrix.database.type }}.dll `
+        --logger:"xunit;Environment=${{ matrix.database.name }} (docker);LogFilePath=/tmp/test-results/docker/Docker.${{ matrix.database.name }}.xml"
+      env:
+        LogLevel: Information
+        TZ: UTC
+        GrateTestConfig__DockerImage: ${{ matrix.database.image }} 
+
     - name: Upload Unit Test Results
       if: always()
       uses: actions/upload-artifact@v4
       with:
         name: "docker ${{ matrix.database.name }} XML test results"
         path: |
           /tmp/test-results/docker/CommandLine.${{ matrix.database.name }}.xml
+          /tmp/test-results/docker/Docker.${{ matrix.database.name }}.xml
         retention-days: 1
 
   integration-test:
@@ -513,7 +514,8 @@ jobs:
   report:
     runs-on: ubuntu-latest
     name: Create test report
-    needs: integration-test
+    #needs: integration-test
+    needs: docker-integration-tests
     if: always()
 
     steps:

src/grate.core/Migration/AnsiSqlDatabase.cs

@@ -21,11 +21,13 @@ public abstract record AnsiSqlDatabase : IDatabase
 
     protected GrateConfiguration? Config { get; private set; }
 
-    protected ILogger Logger { get; }
+    protected ILogger Logger { get; private set; }
     private DbConnection? _connection;
     private DbConnection? _adminConnection;
     private readonly ISyntax _syntax;
 
+    public void SetLogger(ILogger logger) => Logger = logger;
+
     private IDictionary<string, string>? _scriptsRunCache;
 
     private readonly List<Func<DbConnection, Task>> _deferredWrites = new();

src/grate.core/Migration/DbMigrator.cs

@@ -9,12 +9,21 @@ namespace grate.Migration;
 
 internal record DbMigrator : IDbMigrator
 {
-    public ILogger Logger { get; set; }
+    public ILogger Logger
+    {
+        get => _logger;
+        set
+        {
+            _logger = value;
+            Database.SetLogger(value);
+        }
+    }
+
     private readonly IHashGenerator _hashGenerator;
 
     public DbMigrator(IDatabase database, ILogger<DbMigrator> logger, IHashGenerator hashGenerator, GrateConfiguration? configuration = null)
     {
-        Logger = logger;
+        _logger = logger;
         _hashGenerator = hashGenerator;
         Configuration = configuration ?? throw new ArgumentException("No configuration passed to DbMigrator.  Container setup error?", nameof(configuration));
         Database = database;
@@ -247,6 +256,9 @@ private string GetHash(string sql)
     /// Lazily initialised only if needed.
     /// </summary>
     private Dictionary<string, string?>? _tokens;
+
+    private ILogger _logger;
+
     private string ReplaceTokensIn(string sql)
     {
         _tokens ??= new TokenProvider(Configuration, Database).GetTokens();

src/grate.core/Migration/IDatabase.cs

@@ -1,5 +1,6 @@
 using System.Data.Common;
 using grate.Configuration;
+using Microsoft.Extensions.Logging;
 
 namespace grate.Migration;
 
@@ -26,6 +27,8 @@ public interface IDatabase : IAsyncDisposable, ICloneable
     Task CreateDatabase();
     Task RestoreDatabase(string backupPath);
 
+    void SetLogger(ILogger logger);
+
     /// <summary>
     /// Drops the database if it exists, and does nothing if it doesn't.
     /// </summary>

unittests/Basic_tests/Basic_tests.csproj

@@ -6,8 +6,9 @@
     <Nullable>enable</Nullable>
 
     <IsPackable>false</IsPackable>
-    <IsPublishable>false</IsPublishable>
+    <IsPublishable>true</IsPublishable>
     <IsTestProject>true</IsTestProject>
+
     <RootNamespace>Basic_tests</RootNamespace>
   </PropertyGroup>
 

unittests/Basic_tests/GrateMigrator_.cs

@@ -12,7 +12,12 @@ namespace Basic_tests;
 public class GrateMigrator_
 {
     private readonly IDatabase _database = Substitute.For<IDatabase>();
-    private readonly GrateConfiguration? _config = new GrateConfiguration();
+    private readonly GrateConfiguration? _config = new();
+
+    public GrateMigrator_()
+    {
+        _database.Clone().Returns(_database);
+    }
 
     [Fact]
     public void Setting_the_config_does_not_change_the_original()

unittests/CommandLine/CommandLine.Common/CommandLine.Common.csproj

@@ -5,10 +5,11 @@
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
 
-    <IsPublishable>false</IsPublishable>
+    <IsPublishable>true</IsPublishable>
     <IsPackable>false</IsPackable>
     <IsTestProject>false</IsTestProject>
-
+    <NoWarn>IL2091;IL2026</NoWarn>
+
     <XunitStartupFullName>CommandLine.Common.Startup</XunitStartupFullName>
   </PropertyGroup>
 

unittests/CommandLine/CommandLine.MariaDB/CommandLine.MariaDB.csproj

@@ -5,10 +5,10 @@
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
 
-    <IsPublishable>false</IsPublishable>
+    <IsPublishable>true</IsPublishable>
     <IsPackable>false</IsPackable>
     <IsTestProject>true</IsTestProject>
-
+    
   </PropertyGroup>
 
   <ItemGroup>

unittests/CommandLine/CommandLine.Oracle/CommandLine.Oracle.csproj

@@ -5,10 +5,10 @@
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
 
-    <IsPublishable>false</IsPublishable>
+    <IsPublishable>true</IsPublishable>
     <IsPackable>false</IsPackable>
     <IsTestProject>true</IsTestProject>
-
+    
   </PropertyGroup>
 
   <ItemGroup>

unittests/CommandLine/CommandLine.PostgreSQL/CommandLine.PostgreSQL.csproj

@@ -5,10 +5,10 @@
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
 
-    <IsPublishable>false</IsPublishable>
+    <IsPublishable>true</IsPublishable>
     <IsPackable>false</IsPackable>
     <IsTestProject>true</IsTestProject>
-
+    
   </PropertyGroup>
 
   <ItemGroup>

unittests/CommandLine/CommandLine.SqlServer/CommandLine.SqlServer.csproj

@@ -5,10 +5,10 @@
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
 
-    <IsPublishable>false</IsPublishable>
+    <IsPublishable>true</IsPublishable>
     <IsPackable>false</IsPackable>
     <IsTestProject>true</IsTestProject>
-
+    
   </PropertyGroup>
 
   <ItemGroup>

unittests/CommandLine/CommandLine.Sqlite/CommandLine.Sqlite.csproj

@@ -5,10 +5,10 @@
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
 
-    <IsPublishable>false</IsPublishable>
+    <IsPublishable>true</IsPublishable>
     <IsPackable>false</IsPackable>
     <IsTestProject>true</IsTestProject>
-
+    
   </PropertyGroup>
 
   <ItemGroup>

unittests/Docker/Docker.Common/Docker.Common.csproj

@@ -5,10 +5,11 @@
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
 
-    <IsPublishable>false</IsPublishable>
+    <IsPublishable>true</IsPublishable>
     <IsPackable>false</IsPackable>
     <IsTestProject>false</IsTestProject>
-
+    <NoWarn>IL2091;IL2026</NoWarn>
+
     <XunitStartupFullName>Docker.Common.Startup</XunitStartupFullName>
   </PropertyGroup>
 

unittests/Docker/Docker.Common/TestInfrastructure/DockerGrateMigrator.cs

@@ -25,34 +25,65 @@ INetwork Network
 
     public async Task Migrate()
     {
+        // Sqlite doesn't like to be extracted to a tmpfs mount, so we need to use a bind mount to a temporary folder
+        // if not, we get an error like this:
+        //  /home/app/.net/grate/fdcA3gxdjBiIcVt0mGoBJ5IgxSbD0kE=/libe_sqlite3.so: failed to map segment from shared object
+        // (similarly) /tmp/dotnet-bundle-extract/grate/fdcA3gxdjBiIcVt0mGoBJ5IgxSbD0kE=/libe_sqlite3.so: failed to map segment from shared object
+        var tmpFolder = Directory.CreateTempSubdirectory().ToString();
+
+        // Need to map the SQL files directory to the container
+        var sqlFilesDirectory = Configuration.SqlFilesDirectory.ToString();
+
+        Dictionary<string, string> bindMounts = new Dictionary<string, string>
+        {
+            { tmpFolder, "/home/app" },
+            { sqlFilesDirectory, sqlFilesDirectory }
+        };
+
+        // For Sqlite, the database file needs to be mapped as bind mounts, to be able to access the files
+        // both from the host and the container.
+        if (DatabaseType == DatabaseType.SQLite)
+        {
+            foreach (var connectionString in  new [] {Configuration.ConnectionString, Configuration.AdminConnectionString})
+            {
+                var dbFile = connectionString?.Split("=", 2)[1];
+                var folder = Path.GetDirectoryName(dbFile)!;
+                bindMounts[folder] = folder;
+            }
+        }
+
         // Convert configuration to command-line arguments
         var convertToDockerArguments = ConvertToDockerArguments(Configuration);
         var dockerArguments = convertToDockerArguments.ToList();
 
         // Add the database type
         dockerArguments.Add("--databasetype=" + DatabaseType.ToString().ToLowerInvariant());
 
+        //dockerArguments.Add("--verbosity=debug");
+
         // Needed when overriding the entrypoint, not the command
         dockerArguments.Insert(0, "./grate");
-
-        // Need to map the SQL files directory to the container
-        var sqlFilesDirectory = Configuration.SqlFilesDirectory.ToString();
 
         var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(20));
         var token = cancellationTokenSource.Token;
 
-        await using var container = new ContainerBuilder()
+        var containerBuilder = new ContainerBuilder()
             .WithImage(DockerImage)
-            .WithEnvironment("DOTNET_BUNDLE_EXTRACT_BASE_DIR", "/tmp/dotnet-bundle-extract")
             .WithEntrypoint(dockerArguments.ToArray())
+            .WithEnvironment("DOTNET_BUNDLE_EXTRACT_BASE_DIR", "/home/app")
             // This is dependent on changing the image to only use grate as ENTRYPOINT, and the rest as CMD
             //.WithCommand(dockerArguments.ToArray())
-            .WithBindMount(sqlFilesDirectory, sqlFilesDirectory)
             .WithCreateParameterModifier(param => param.HostConfig.ReadonlyRootfs = true)
             .WithTmpfsMount("/tmp")
             .WithNetwork(Network)
-            .WithLogger(Logger)
-            .Build();
+            .WithLogger(Logger);
+
+        foreach (var bindMount in bindMounts)
+        {
+            containerBuilder = containerBuilder.WithBindMount(bindMount.Key, bindMount.Value);
+        }
+
+        await using var container = containerBuilder.Build();
 
         try
         {

unittests/Docker/Docker.MariaDB/Docker.MariaDB.csproj

@@ -5,10 +5,10 @@
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
 
-    <IsPublishable>false</IsPublishable>
+    <IsPublishable>true</IsPublishable>
     <IsPackable>false</IsPackable>
     <IsTestProject>true</IsTestProject>
-
+    
   </PropertyGroup>
 
   <ItemGroup>

unittests/Docker/Docker.Oracle/Docker.Oracle.csproj

@@ -5,10 +5,10 @@
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
 
-    <IsPublishable>false</IsPublishable>
+    <IsPublishable>true</IsPublishable>
     <IsPackable>false</IsPackable>
     <IsTestProject>true</IsTestProject>
-
+    
   </PropertyGroup>
 
   <ItemGroup>

unittests/Docker/Docker.PostgreSQL/Docker.PostgreSQL.csproj

@@ -5,10 +5,10 @@
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
 
-    <IsPublishable>false</IsPublishable>
+    <IsPublishable>true</IsPublishable>
     <IsPackable>false</IsPackable>
     <IsTestProject>true</IsTestProject>
-
+    
   </PropertyGroup>
 
   <ItemGroup>