Feat/252 custom create database (#289)

Allow for custom CREATE DATABASE and DROP DATABASE scripts in folders, similar to up, beforeMigration, afterMigration, etc. If you have the need for any custom create database logic, put it in this folder, and it will be used instead of the standard.

docs/ConfigurationOptions/FolderConfiguration.md

@@ -131,6 +131,8 @@ grate processes the files in a standard set of directories in a fixed order for
 
 | Folder | Script type | Explanation |
 | ------ | ------- |------- |
+| <nobr> (-1. dropDatabase)</nobr> | Anytime scripts | If you have the need for a custom `DROP DATABASE` script (used with the `--drop` command-line flag) |
+| <nobr> (0. createDatabase)</nobr> | Anytime scripts | If you have the need for a custom `CREATE DATABASE` script, put it here, and it will be used instead of the default. |
 | <nobr> 1. beforeMigration</nobr> | Everytime scripts | If you have particular tasks you want to perform prior to any database migrations (custom logging? database backups? disable replication?) you can do it here. |
 | <nobr>2. alterDatabase</nobr> | Anytime scripts | If you have scripts that need to alter the database config itself (rather than the _contents_ of the database) thjis is the place to do it.  For example setting recovery modes, enabling query stores, etc etc |
 | <nobr>3. runAfterCreateDatabase</nobr> | Anytime scripts | This directory is only processed if the database was created from scratch by grate.  Maybe you need to add user accounts or similar?

grate.unittests/Basic/Infrastructure/FolderConfiguration/KnownFolders_CustomNames.cs

@@ -69,6 +69,7 @@ TransactionHandling transactionHandling
     private static readonly IKnownFolderNames OverriddenFolderNames = new KnownFolderNames()
     {
         BeforeMigration = "beforeMigration" + Random.GetString(8),
+        CreateDatabase = "createDatabase" + Random.GetString(8),
         AlterDatabase = "alterDatabase" + Random.GetString(8),
         RunAfterCreateDatabase = "runAfterCreateDatabase" + Random.GetString(8),
         RunBeforeUp = "runBeforeUp" + Random.GetString(8),

grate.unittests/Basic/Infrastructure/FolderConfiguration/KnownFolders_Default.cs

@@ -45,12 +45,12 @@ public void Returns_folders_in_current_order()
     [Test]
     [TestCaseSource(nameof(ExpectedKnownFolderNames))]
     public void Has_expected_folder_configuration(
-            MigrationsFolder folder, 
-            string expectedName, 
-            MigrationType expectedType,
-            ConnectionType expectedConnectionType,
-            TransactionHandling transactionHandling
-        )
+        MigrationsFolder folder, 
+        string expectedName, 
+        MigrationType expectedType,
+        ConnectionType expectedConnectionType,
+        TransactionHandling transactionHandling
+    )
     {
         var root = Root.ToString();
 
@@ -94,11 +94,11 @@ private static TestCaseData GetTestCase(
     ) =>
         new TestCaseData(folder, expectedName, expectedType, expectedConnectionType, transactionHandling)
             .SetArgDisplayNames(
-                    migrationsFolderDefinitionName, 
-                    expectedName,
-                    expectedType.ToString(),
-                    "conn: " + expectedConnectionType,
-                    "tran: " + transactionHandling
-                );
+                migrationsFolderDefinitionName, 
+                expectedName,
+                expectedType.ToString(),
+                "conn: " + expectedConnectionType,
+                "tran: " + transactionHandling
+            );
 
 }

grate.unittests/Generic/GenericDatabase.cs

@@ -1,5 +1,6 @@
 using System.Collections.Generic;
 using System.Data.Common;
+using System.IO;
 using System.Linq;
 using System.Threading.Tasks;
 using System.Transactions;
@@ -8,7 +9,9 @@
 using grate.Configuration;
 using grate.Migration;
 using grate.unittests.TestInfrastructure;
+using Microsoft.Data.SqlClient;
 using NUnit.Framework;
+using static System.StringSplitOptions;
 
 namespace grate.unittests.Generic;
 
@@ -28,6 +31,39 @@ public async Task Is_created_if_confed_and_it_does_not_exist()
         IEnumerable<string> databases = await GetDatabases();
         databases.Should().Contain(db);
     }
+
+    [Test]
+    public virtual async Task Is_created_with_custom_script_if_custom_create_database_folder_exists()
+    {
+        var scriptedDatabase = "CUSTOMSCRIPTEDDATABASE";
+        var confedDatabase = "DEFAULTDATABASE";
+
+        var config = GetConfiguration(confedDatabase, true);
+        var password = Context.AdminConnectionString
+            .Split(";", TrimEntries | RemoveEmptyEntries)
+            .SingleOrDefault(entry => entry.StartsWith("Password") || entry.StartsWith("Pwd"))?
+            .Split("=", TrimEntries | RemoveEmptyEntries)
+            .Last();
+
+        var customScript = Context.Syntax.CreateDatabase(scriptedDatabase, password);
+        TestConfig.WriteContent(Wrap(config.SqlFilesDirectory, config.Folders?.CreateDatabase?.Path), "createDatabase.sql", customScript);
+        try
+        {
+            await using var migrator = GetMigrator(config);
+            await migrator.Migrate();
+        }
+        catch (DbException)
+        {
+            //Do nothing because database name is wrong due to custom script
+        }
+
+        File.Delete(Path.Join(Wrap(config.SqlFilesDirectory, config.Folders?.CreateDatabase?.Path).ToString(), "createDatabase.sql"));
+
+        // The database should have been created by the custom script
+        IEnumerable<string> databases = (await GetDatabases()).ToList();
+        databases.Should().Contain(scriptedDatabase);
+        databases.Should().NotContain(confedDatabase);
+    }
 
     [Test]
     public async Task Is_not_created_if_not_confed()
@@ -180,4 +216,8 @@ private GrateConfiguration GetConfiguration(bool createDatabase, string? connect
         };
     }
 
+
+    protected static DirectoryInfo Wrap(DirectoryInfo root, string? subFolder) =>
+        new DirectoryInfo(Path.Combine(root.ToString(), subFolder ?? ""));
+
 }

grate.unittests/SqLite/Database.cs

@@ -42,5 +42,9 @@ protected override async Task<IEnumerable<string>> GetDatabases()
         return await ValueTask.FromResult(dbNames);
     }
 
+    [Ignore("SQLite does not support custom database creation script")]
+    public override Task Is_created_with_custom_script_if_custom_create_database_folder_exists() =>
+        Task.CompletedTask;
+
     protected override bool ThrowOnMissingDatabase => false;
 }

grate/Configuration/FoldersConfiguration.cs

@@ -23,15 +23,17 @@ public FoldersConfiguration(IDictionary<string, MigrationsFolder> source)
 
     public FoldersConfiguration() 
     { }
-
+
+    public MigrationsFolder? CreateDatabase { get; set; }
+    public MigrationsFolder? DropDatabase { get; set; }
 
     public static FoldersConfiguration Empty => new();
 
     public static IFoldersConfiguration Default(IKnownFolderNames? folderNames = null)
     {
         folderNames ??= KnownFolderNames.Default;
-        
-        return new FoldersConfiguration()
+
+        var foldersConfiguration = new FoldersConfiguration()
         {
             { KnownFolderKeys.BeforeMigration, new MigrationsFolder("BeforeMigration", folderNames.BeforeMigration, EveryTime, TransactionHandling: TransactionHandling.Autonomous) },
             { KnownFolderKeys.AlterDatabase , new MigrationsFolder("AlterDatabase", folderNames.AlterDatabase, AnyTime, ConnectionType.Admin, TransactionHandling.Autonomous) },
@@ -46,8 +48,12 @@ public static IFoldersConfiguration Default(IKnownFolderNames? folderNames = nul
             { KnownFolderKeys.Indexes, new MigrationsFolder("Indexes", folderNames.Indexes, AnyTime) },
             { KnownFolderKeys.RunAfterOtherAnyTimeScripts, new MigrationsFolder("Run after Other Anytime Scripts", folderNames.RunAfterOtherAnyTimeScripts, AnyTime) },
             { KnownFolderKeys.Permissions, new MigrationsFolder("Permissions", folderNames.Permissions, EveryTime, TransactionHandling: TransactionHandling.Autonomous) },
-            { KnownFolderKeys.AfterMigration, new MigrationsFolder("AfterMigration", folderNames.AfterMigration, EveryTime, TransactionHandling: TransactionHandling.Autonomous) }
+            { KnownFolderKeys.AfterMigration, new MigrationsFolder("AfterMigration", folderNames.AfterMigration, EveryTime, TransactionHandling: TransactionHandling.Autonomous) },
         };
+        foldersConfiguration.CreateDatabase = new MigrationsFolder("CreateDatabase", folderNames.CreateDatabase, AnyTime, ConnectionType.Admin, TransactionHandling.Autonomous);
+        foldersConfiguration.DropDatabase = new MigrationsFolder("DropDatabase", folderNames.DropDatabase, AnyTime, ConnectionType.Admin, TransactionHandling.Autonomous);
+
+        return foldersConfiguration;
     }
 
 }

grate/Configuration/IFoldersConfiguration.cs

@@ -4,4 +4,6 @@ namespace grate.Configuration;
 
 public interface IFoldersConfiguration: IDictionary<string, MigrationsFolder?>
 {
+    MigrationsFolder? CreateDatabase { get; set; }
+    MigrationsFolder? DropDatabase { get; set; }
 }

grate/Configuration/IKnownFolderNames.cs

@@ -3,6 +3,8 @@
 public interface IKnownFolderNames
 {
     string BeforeMigration { get; }
+    string CreateDatabase { get; }
+    string DropDatabase { get; }
     string AlterDatabase { get; }
     string RunAfterCreateDatabase { get; }
     string RunBeforeUp { get; }

grate/Configuration/KnownFolderKeys.cs

@@ -5,6 +5,7 @@ namespace grate.Configuration;
 public static class KnownFolderKeys
 {
     public const string BeforeMigration = nameof(BeforeMigration);
+    public const string CreateDatabase = nameof(CreateDatabase);
     public const string AlterDatabase = nameof(AlterDatabase);
     public const string RunAfterCreateDatabase = nameof(RunAfterCreateDatabase);
     public const string RunBeforeUp = nameof(RunBeforeUp);

grate/Configuration/KnownFolderNames.cs

@@ -3,6 +3,8 @@
 public record KnownFolderNames: IKnownFolderNames
 {
     public string BeforeMigration { get; init; } = "beforeMigration";
+    public string CreateDatabase { get; init; } = "createDatabase";
+    public string DropDatabase { get; init; } = "dropDatabase";
     public string AlterDatabase { get; init; } = "alterDatabase";
     public string RunAfterCreateDatabase { get; init; } = "runAfterCreateDatabase";
     public string RunBeforeUp { get; init; } = "runBeforeUp";

grate/Migration/DbMigrator.cs

@@ -154,6 +154,42 @@ async Task<bool> LogAndRunSql()
 
         return theSqlWasRun;
     }
+
+    public async Task<bool> RunSqlWithoutLogging(
+        string sql, 
+        string scriptName,
+        GrateEnvironment? environment,
+        ConnectionType connectionType, 
+        TransactionHandling transactionHandling)
+    {
+        async Task<bool> PrintLogAndRunSql()
+        {
+            _logger.LogInformation("  Running '{ScriptName}'.", scriptName);
+
+            if (Configuration.DryRun)
+            {
+                return false;
+            }
+            else
+            {
+                await RunTheActualSqlWithoutLogging(sql, scriptName, connectionType, transactionHandling);
+                return true;
+            }
+        }
+
+        if (!InCorrectEnvironment(scriptName, environment))
+        {
+            return false;
+        }
+
+        if (TokenReplacementEnabled)
+        {
+            sql = ReplaceTokensIn(sql);
+        }
+
+        return await PrintLogAndRunSql();;
+    }
+
 
     public async Task RestoreDatabase(string backupPath)
     {
@@ -273,6 +309,34 @@ private async Task RunTheActualSql(string sql,
 
         await RecordScriptInScriptsRunTable(scriptName, sql, migrationType, versionId, transactionHandling);
     }
+
+    private async Task RunTheActualSqlWithoutLogging(
+        string sql,
+        string scriptName,
+        ConnectionType connectionType, 
+        TransactionHandling transactionHandling)
+    {
+        foreach (var statement in GetStatements(sql))
+        {
+            try
+            {
+                await Database.RunSql(statement, connectionType, transactionHandling);
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError("Error running script \"{ScriptName}\": {ErrorMessage}", scriptName, ex.Message);
+
+                if (Transaction.Current is not null) {
+                    Database.Rollback();
+                }
+
+                await Database.CloseConnection();
+                Transaction.Current?.Dispose();
+                throw;
+            }
+        }
+    }
+
 
     private IEnumerable<string> GetStatements(string sql) => StatementSplitter.Split(sql);
 

grate/Migration/GrateMigrator.cs

@@ -229,7 +229,7 @@ private async Task CreateGrateStructure(IDbMigrator dbMigrator)
         return (versionId, newVersion);
     }
 
-    private static async Task<bool> CreateDatabaseIfItDoesNotExist(IDbMigrator dbMigrator)
+    private async Task<bool> CreateDatabaseIfItDoesNotExist(IDbMigrator dbMigrator)
     {
         bool databaseCreated;
         if (await dbMigrator.DatabaseExists())
@@ -238,7 +238,28 @@ private static async Task<bool> CreateDatabaseIfItDoesNotExist(IDbMigrator dbMig
         }
         else
         {
-            databaseCreated = await dbMigrator.CreateDatabase();
+            var config = dbMigrator.Configuration;
+            var createDatabaseFolder = config.Folders?.CreateDatabase;
+            var database = _migrator.Database;
+
+            var path = Wrap(config.SqlFilesDirectory, createDatabaseFolder?.Path  ?? "zz-xx-øø-definitely-does-not-exist");
+
+            if (createDatabaseFolder is not null && path.Exists)
+            {
+                //await LogAndProcess(config.SqlFilesDirectory, folder!, changeDropFolder, versionId, folder!.ConnectionType, folder.TransactionHandling);
+                var changeDropFolder = ChangeDropFolder(config, database.ServerName, database.DatabaseName);
+                databaseCreated = await ProcessWithoutLogging(
+                    config.SqlFilesDirectory,
+                    createDatabaseFolder,
+                    changeDropFolder,
+                    createDatabaseFolder.ConnectionType,
+                    createDatabaseFolder.TransactionHandling
+                );
+            }
+            else
+            {
+                databaseCreated = await dbMigrator.CreateDatabase();
+            }
         }
         return databaseCreated;
     }
@@ -330,6 +351,53 @@ private async Task Process(DirectoryInfo root, MigrationsFolder folder, string c
         }
 
     }
+
+    private async Task<bool> ProcessWithoutLogging(DirectoryInfo root, MigrationsFolder folder, string changeDropFolder,
+        ConnectionType connectionType, TransactionHandling transactionHandling)
+    {
+        var path = Wrap(root, folder.Path);
+
+        await EnsureConnectionIsOpen(connectionType);
+
+        var pattern = "*.sql";
+        var files = FileSystem.GetFiles(path, pattern);
+
+        var anySqlRun = false;
+
+        foreach (var file in files)
+        {
+            var sql = await File.ReadAllTextAsync(file.FullName);
+
+            // Normalize file names to log, so that results won't vary if you run on *nix VS Windows
+            var fileNameToLog = string.Join('/',
+                Path.GetRelativePath(path.ToString(), file.FullName).Split(Path.DirectorySeparatorChar));
+
+            bool theSqlRan = await _migrator.RunSqlWithoutLogging(sql, fileNameToLog, _migrator.Configuration.Environment,
+                connectionType, transactionHandling);
+
+            if (theSqlRan)
+            {
+                anySqlRun = true;
+                try
+                {
+                    CopyToChangeDropFolder(path.Parent!, file, changeDropFolder);
+                }
+                catch (Exception ex)
+                {
+                    _logger.LogWarning(ex, "Unable to copy {File} to {ChangeDropFolder}. \n{Exception}", file, changeDropFolder, ex.Message);
+                }
+            }
+        }
+
+        if (!anySqlRun)
+        {
+            _logger.LogInformation(" No sql run, either an empty folder, or all files run against destination previously.");
+        }
+
+        return anySqlRun;
+
+    }
+
 
     private void CopyToChangeDropFolder(DirectoryInfo migrationRoot, FileSystemInfo file, string changeDropFolder)
     {

grate/Migration/IDbMigrator.cs

@@ -26,10 +26,15 @@ public interface IDbMigrator: IAsyncDisposable
     Task<long> VersionTheDatabase(string newVersion);
     Task OpenAdminConnection();
     Task CloseAdminConnection();
+
     Task<bool> RunSql(string sql, string scriptName, MigrationType migrationType, long versionId,
         GrateEnvironment? environment,
         ConnectionType connectionType, TransactionHandling transactionHandling);
 
+    Task<bool> RunSqlWithoutLogging(string sql, string scriptName,
+        GrateEnvironment? environment,
+        ConnectionType connectionType, TransactionHandling transactionHandling);
+
     Task RestoreDatabase(string backupPath);
     void SetDefaultConnectionActive();
     Task<IDisposable> OpenNewActiveConnection();