Feature/rdmp 34 chi ingress

Application/ResearchDataManagementPlatform/ResearchDataManagementPlatform.csproj

@@ -111,4 +111,4 @@
   <ItemGroup>
     <ProjectReference Include="..\..\Rdmp.UI\Rdmp.UI.csproj" />
   </ItemGroup>
-</Project>
+</Project>

CHANGELOG.md

@@ -1,16 +1,23 @@
 
+
 # Changelog
 All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [8.1.6] = Unreleased
+## [8.2.0] - Unreleased
+
+## Changed
+
+- Add ability to redact CHI values on daata load and to redact existing catalogues
+
+## [8.1.6] - Unreleased
 
 ## Changed
 
 - Add Microsoft.Bcl.AsyncInterfaces 6.0.0 for plugin dependancy tree
-- Add prompt to reanem container when adding a cohort filter
+- Add prompt to rename container when adding a cohort filter
 
 ## [8.1.5] - 2024-04-03
 

Rdmp.Core.Tests/CommandExecution/ExecuteCommandConfirmRedactedCHITests.cs

@@ -0,0 +1,32 @@
+using NUnit.Framework;
+using Rdmp.Core.CommandExecution;
+using Rdmp.Core.CommandExecution.AtomicCommands;
+using Rdmp.Core.Curation.Data;
+using Tests.Common;
+
+namespace Rdmp.Core.Tests.CommandExecution;
+
+internal class ExecuteCommandConfirmRedactedCHITests: DatabaseTests
+{
+    [Test]
+    public void ConfirmRedactedCHI_Valid() {
+        var redactedChisCount = CatalogueRepository.GetAllObjects<RedactedCHI>().Length;
+        var rCHI = new RedactedCHI(CatalogueRepository, "1111111111", 0, "[fake].[table]","pkValue","[pkColumn]","[columnName]");
+        rCHI.SaveToDatabase();
+
+        var cmd = new ExecuteCommandConfirmRedactedCHI(new ThrowImmediatelyActivator(RepositoryLocator), rCHI);
+        Assert.DoesNotThrow(()=> cmd.Execute());
+        Assert.That(redactedChisCount, Is.EqualTo(CatalogueRepository.GetAllObjects<RedactedCHI>().Length));
+    }
+
+    [Test]
+    public void ConfirmRedactedCHI_InValid() {
+        var redactedChisCount = CatalogueRepository.GetAllObjects<RedactedCHI>().Length;
+        var rCHI = new RedactedCHI(CatalogueRepository, "1111111111", 0, "[fake].[table]", "pkValue", "[pkColumn]", "[columnName]");
+        rCHI.DeleteInDatabase();
+
+        var cmd = new ExecuteCommandConfirmRedactedCHI(new ThrowImmediatelyActivator(RepositoryLocator), rCHI);
+        Assert.DoesNotThrow(() => cmd.Execute());
+        Assert.That(redactedChisCount, Is.EqualTo(CatalogueRepository.GetAllObjects<RedactedCHI>().Length));
+    }
+}

Rdmp.Core.Tests/CommandExecution/ExecuteCommandIdentifyCHIInCatalogueTests.cs

@@ -0,0 +1,107 @@
+using NUnit.Framework;
+using Rdmp.Core.CommandExecution;
+using Rdmp.Core.CommandExecution.AtomicCommands;
+using Rdmp.Core.CommandLine.DatabaseCreation;
+using Rdmp.Core.Curation.Data;
+using Rdmp.Core.ReusableLibraryCode.Checks;
+using System.IO;
+using System.Linq;
+using Tests.Common;
+
+namespace Rdmp.Core.Tests.CommandExecution;
+
+internal class ExecuteCommandIdentifyCHIInCatalogueTests : DatabaseTests
+{
+
+    [Test]
+    public void IdentifyCHIInCatalogue_Correct()
+    {
+        var db = GetCleanedServer(FAnsi.DatabaseType.MicrosoftSQLServer);
+        var pipes = new CataloguePipelinesAndReferencesCreation(RepositoryLocator, null, null);
+        pipes.CreatePipelines(new PlatformDatabaseCreationOptions());
+        var creator = new ExampleDatasetsCreation(new ThrowImmediatelyActivator(RepositoryLocator), RepositoryLocator);
+        creator.Create(db, ThrowImmediatelyCheckNotifier.Quiet,
+            new PlatformDatabaseCreationOptions { Seed = 500, DropDatabases = true });
+        var biochemistry = CatalogueRepository.GetAllObjects<Catalogue>().Where(c => c.Name == "Biochemistry").First();
+        var cmd = new ExecuteCommandIdentifyCHIInCatalogue(new ThrowImmediatelyActivator(RepositoryLocator), biochemistry, false, null);
+        Assert.DoesNotThrow(() => cmd.Execute());
+        Assert.That(cmd.foundChis.Rows.Count, Is.EqualTo(0));
+    }
+
+    [Test]
+    public void IdentifyCHIInCatalogue_FoundCHI()
+    {
+        var db = GetCleanedServer(FAnsi.DatabaseType.MicrosoftSQLServer);
+        var creator = new ExampleDatasetsCreation(new ThrowImmediatelyActivator(RepositoryLocator), RepositoryLocator);
+        creator.Create(db, ThrowImmediatelyCheckNotifier.Quiet,
+            new PlatformDatabaseCreationOptions { Seed = 500, DropDatabases = true });
+
+
+        var tbl = db.DiscoverTables(true).Where(dt => dt.GetRuntimeName().Contains("Biochemistry")).First();
+        var updateSQL = $"UPDATE top (2) {tbl.GetRuntimeName()} set SampleType = '1111111111'";
+        using (var con = tbl.Database.Server.GetConnection())
+        {
+            con.Open();
+            var updateCmd = tbl.Database.Server.GetCommand(updateSQL, con);
+            updateCmd.ExecuteNonQuery();
+        }
+
+        var biochemistry = CatalogueRepository.GetAllObjects<Catalogue>().Where(c => c.Name == "Biochemistry").First();
+        var cmd = new ExecuteCommandIdentifyCHIInCatalogue(new ThrowImmediatelyActivator(RepositoryLocator), biochemistry, false, null);
+        Assert.DoesNotThrow(() => cmd.Execute());
+        Assert.That(cmd.foundChis.Rows.Count, Is.EqualTo(2));
+    }
+
+    [Test]
+    public void IdentifyCHIInCatalogue_FoundCHI_BailOuit()
+    {
+        var db = GetCleanedServer(FAnsi.DatabaseType.MicrosoftSQLServer);
+        var creator = new ExampleDatasetsCreation(new ThrowImmediatelyActivator(RepositoryLocator), RepositoryLocator);
+        creator.Create(db, ThrowImmediatelyCheckNotifier.Quiet,
+            new PlatformDatabaseCreationOptions { Seed = 500, DropDatabases = true });
+
+
+        var tbl = db.DiscoverTables(true).Where(dt => dt.GetRuntimeName().Contains("Biochemistry")).First();
+        var updateSQL = $"UPDATE top (2) {tbl.GetRuntimeName()} set SampleType = '1111111111'";
+        using (var con = tbl.Database.Server.GetConnection())
+        {
+            con.Open();
+            var updateCmd = tbl.Database.Server.GetCommand(updateSQL, con);
+            updateCmd.ExecuteNonQuery();
+        }
+
+        var biochemistry = CatalogueRepository.GetAllObjects<Catalogue>().Where(c => c.Name == "Biochemistry").First();
+        var cmd = new ExecuteCommandIdentifyCHIInCatalogue(new ThrowImmediatelyActivator(RepositoryLocator), biochemistry, true, null);
+        Assert.DoesNotThrow(() => cmd.Execute());
+        Assert.That(cmd.foundChis.Rows.Count, Is.EqualTo(1));
+    }
+
+    [Test]
+    public void IdentifyCHIInCatalogue_FoundCHI_AllowList()
+    {
+        var db = GetCleanedServer(FAnsi.DatabaseType.MicrosoftSQLServer);
+        var creator = new ExampleDatasetsCreation(new ThrowImmediatelyActivator(RepositoryLocator), RepositoryLocator);
+        creator.Create(db, ThrowImmediatelyCheckNotifier.Quiet,
+            new PlatformDatabaseCreationOptions { Seed = 500, DropDatabases = true });
+
+
+        var tbl = db.DiscoverTables(true).Where(dt => dt.GetRuntimeName().Contains("Biochemistry")).First();
+        var updateSQL = $"UPDATE top (2) {tbl.GetRuntimeName()} set SampleType = '1111111111'";
+        using (var con = tbl.Database.Server.GetConnection())
+        {
+            con.Open();
+            var updateCmd = tbl.Database.Server.GetCommand(updateSQL, con);
+            updateCmd.ExecuteNonQuery();
+        }
+
+        var biochemistry = CatalogueRepository.GetAllObjects<Catalogue>().Where(c => c.Name == "Biochemistry").First();
+        var tempFileToCreate = Path.Combine(TestContext.CurrentContext.TestDirectory, "allowList.yaml");
+        var allowList = File.Create(tempFileToCreate);
+        allowList.Close();
+        File.WriteAllLines(tempFileToCreate, new string[] { "Biochemistry:", "- SampleType" });
+        var cmd = new ExecuteCommandIdentifyCHIInCatalogue(new ThrowImmediatelyActivator(RepositoryLocator), biochemistry, false, tempFileToCreate);
+        Assert.DoesNotThrow(() => cmd.Execute());
+        Assert.That(cmd.foundChis.Rows.Count, Is.EqualTo(0));
+    }
+
+}

Rdmp.Core.Tests/CommandExecution/ExecuteCommandRedactCHIsFromCatalogueTests.cs

@@ -0,0 +1,68 @@
+using NUnit.Framework;
+using Rdmp.Core.CommandExecution;
+using Rdmp.Core.CommandExecution.AtomicCommands;
+using Rdmp.Core.CommandLine.DatabaseCreation;
+using Rdmp.Core.Curation.Data;
+using Rdmp.Core.ReusableLibraryCode.Checks;
+using System.Data;
+using System.IO;
+using System.Linq;
+using Tests.Common;
+
+namespace Rdmp.Core.Tests.CommandExecution;
+
+internal class ExecuteCommandRedactCHIsFromCatalogueTests: DatabaseTests
+{
+
+    [Test]
+    public void RedactCHIsFromCatalogue_ValidSingle()
+    {
+        var db = GetCleanedServer(FAnsi.DatabaseType.MicrosoftSQLServer);
+        var creator = new ExampleDatasetsCreation(new ThrowImmediatelyActivator(RepositoryLocator), RepositoryLocator);
+        creator.Create(db, ThrowImmediatelyCheckNotifier.Quiet,
+            new PlatformDatabaseCreationOptions { Seed = 500, DropDatabases = true });
+        var biochemistry = CatalogueRepository.GetAllObjects<Catalogue>().Where(c => c.Name == "Biochemistry").First();
+        var tbl = db.DiscoverTables(true).Where(dt => dt.GetRuntimeName().Contains("Biochemistry")).First();
+        var updateSQL = $"UPDATE top (2) {tbl.GetRuntimeName()} set SampleType = 'F1111111111'";
+        using var con = tbl.Database.Server.GetConnection();
+        con.Open();
+        var updateCmd = tbl.Database.Server.GetCommand(updateSQL, con);
+        updateCmd.ExecuteNonQuery();
+        var cmd = new ExecuteCommandRedactCHIsFromCatalogue(new ThrowImmediatelyActivator(RepositoryLocator), biochemistry, null);
+        Assert.DoesNotThrow(() => cmd.Execute());
+        using var dt = new DataTable();
+        using var findCmd = tbl.Database.Server.GetCommand($"select * from {tbl.GetRuntimeName()} where SampleType = 'F#########1'", con);
+        using var da = tbl.Database.Server.GetDataAdapter(findCmd);
+        da.Fill(dt);
+        Assert.That(dt.Rows.Count, Is.EqualTo(2));
+    }
+
+    [Test]
+    public void RedactCHIsFromCatalogue_Allowlist()
+    {
+        var db = GetCleanedServer(FAnsi.DatabaseType.MicrosoftSQLServer);
+        var pipes = new CataloguePipelinesAndReferencesCreation(RepositoryLocator, null, null);
+        pipes.CreatePipelines(new PlatformDatabaseCreationOptions());
+        var creator = new ExampleDatasetsCreation(new ThrowImmediatelyActivator(RepositoryLocator), RepositoryLocator);
+        creator.Create(db, ThrowImmediatelyCheckNotifier.Quiet,
+            new PlatformDatabaseCreationOptions { Seed = 500, DropDatabases = true });
+        var biochemistry = CatalogueRepository.GetAllObjects<Catalogue>().Where(c => c.Name == "Biochemistry").First();
+        var tbl = db.DiscoverTables(true).Where(dt => dt.GetRuntimeName().Contains("Biochemistry")).First();
+        var updateSQL = $"UPDATE top (2) {tbl.GetRuntimeName()} set SampleType = 'F1111111111'";
+        using var con = tbl.Database.Server.GetConnection();
+        con.Open();
+        var updateCmd = tbl.Database.Server.GetCommand(updateSQL, con);
+        updateCmd.ExecuteNonQuery();
+        var tempFileToCreate = Path.Combine(TestContext.CurrentContext.TestDirectory, "allowList.yaml");
+        var allowList = File.Create(tempFileToCreate);
+        allowList.Close();
+        File.WriteAllLines(tempFileToCreate, new string[] { "Biochemistry:", "- SampleType" });
+        var cmd = new ExecuteCommandRedactCHIsFromCatalogue(new ThrowImmediatelyActivator(RepositoryLocator), biochemistry, tempFileToCreate);
+        Assert.DoesNotThrow(() => cmd.Execute());
+        using var dt = new DataTable();
+        using var findCmd = tbl.Database.Server.GetCommand($"select * from {tbl.GetRuntimeName()} where SampleType = 'F#########1'", con);
+        using var da = tbl.Database.Server.GetDataAdapter(findCmd);
+        da.Fill(dt);
+        Assert.That(dt.Rows.Count, Is.EqualTo(0));
+    }
+}

Rdmp.Core.Tests/CommandExecution/ExecuteCommandRevertRedactedCHITests.cs

@@ -0,0 +1,112 @@
+using NUnit.Framework;
+using Rdmp.Core.CommandExecution.AtomicCommands;
+using Rdmp.Core.CommandExecution;
+using Rdmp.Core.Curation.Data;
+using System.Linq;
+using Tests.Common;
+using Rdmp.Core.CommandLine.DatabaseCreation;
+using Rdmp.Core.ReusableLibraryCode.Checks;
+using System.Data;
+using System;
+
+namespace Rdmp.Core.Tests.CommandExecution;
+
+internal class ExecuteCommandRevertRedactedCHITests : DatabaseTests
+{
+    [Test]
+    public void RevertRedactedCHI_Valid()
+    {
+        var db = GetCleanedServer(FAnsi.DatabaseType.MicrosoftSQLServer);
+        var creator = new ExampleDatasetsCreation(new ThrowImmediatelyActivator(RepositoryLocator), RepositoryLocator);
+        creator.Create(db, ThrowImmediatelyCheckNotifier.Quiet,
+            new PlatformDatabaseCreationOptions { Seed = 500, DropDatabases = true });
+        var biochemistry = CatalogueRepository.GetAllObjects<Catalogue>().Where(c => c.Name == "Biochemistry").First();
+        var tbl = db.DiscoverTables(true).Where(dt => dt.GetRuntimeName().Contains("Biochemistry")).First();
+        var updateSQL = $"UPDATE top (1) {tbl.GetRuntimeName()} set SampleType = 'R1111111111'";
+        using (var con = tbl.Database.Server.GetConnection())
+        {
+            con.Open();
+            var updateCmd = tbl.Database.Server.GetCommand(updateSQL, con);
+            updateCmd.ExecuteNonQuery();
+        }
+        var cmd = new ExecuteCommandRedactCHIsFromCatalogue(new ThrowImmediatelyActivator(RepositoryLocator), biochemistry, null);
+        Assert.DoesNotThrow(() => cmd.Execute());
+        var dt = new DataTable();
+        using (var con = tbl.Database.Server.GetConnection())
+        {
+            using (var findCmd = tbl.Database.Server.GetCommand($"select * from {tbl.GetRuntimeName()} where SampleType = 'R#########1'", con))
+            {
+                using var da = tbl.Database.Server.GetDataAdapter(findCmd);
+                da.Fill(dt);
+            }
+        }
+        Assert.That(dt.Rows.Count, Is.EqualTo(1));
+        var redactedChisCount = CatalogueRepository.GetAllObjects<RedactedCHI>().Length;
+        var rCHI = CatalogueRepository.GetAllObjects<RedactedCHI>().First();
+        var revertCmd = new ExecuteCommandRevertRedactedCHI(new ThrowImmediatelyActivator(RepositoryLocator), rCHI);
+        Assert.DoesNotThrow(() => revertCmd.Execute());
+        Assert.That(redactedChisCount, Is.EqualTo(CatalogueRepository.GetAllObjects<RedactedCHI>().Length +1));
+        dt = new DataTable();
+        using (var con = tbl.Database.Server.GetConnection())
+        {
+            using (var findCmd = tbl.Database.Server.GetCommand($"select * from {tbl.GetRuntimeName()} where SampleType = 'R1111111111'", con))
+            {
+                using var da = tbl.Database.Server.GetDataAdapter(findCmd);
+                da.Fill(dt);
+            }
+        }
+        Assert.That(dt.Rows.Count, Is.EqualTo(1));
+        dt.Dispose();
+    }
+
+    [Test]
+    public void RevertRedactedCHI_InValid()
+    {
+        var db = GetCleanedServer(FAnsi.DatabaseType.MicrosoftSQLServer);
+        var pipes = new CataloguePipelinesAndReferencesCreation(RepositoryLocator, null, null);
+        pipes.CreatePipelines(new PlatformDatabaseCreationOptions());
+        var creator = new ExampleDatasetsCreation(new ThrowImmediatelyActivator(RepositoryLocator), RepositoryLocator);
+        creator.Create(db, ThrowImmediatelyCheckNotifier.Quiet,
+            new PlatformDatabaseCreationOptions { Seed = 500, DropDatabases = true });
+        var biochemistry = CatalogueRepository.GetAllObjects<Catalogue>().Where(c => c.Name == "Biochemistry").First();
+        var tbl = db.DiscoverTables(true).Where(dt => dt.GetRuntimeName().Contains("Biochemistry")).First();
+        var updateSQL = $"UPDATE top (1) {tbl.GetRuntimeName()} set SampleType = 'R1111111111'";
+        using (var con = tbl.Database.Server.GetConnection())
+        {
+            con.Open();
+            var updateCmd = tbl.Database.Server.GetCommand(updateSQL, con);
+            updateCmd.ExecuteNonQuery();
+        }
+        var cmd = new ExecuteCommandRedactCHIsFromCatalogue(new ThrowImmediatelyActivator(RepositoryLocator), biochemistry, null);
+        Assert.DoesNotThrow(() => cmd.Execute());
+        var dt = new DataTable();
+        using (var con = tbl.Database.Server.GetConnection())
+        {
+            using (var findCmd = tbl.Database.Server.GetCommand($"select * from {tbl.GetRuntimeName()} where SampleType = 'R#########1'", con))
+            {
+                using var da = tbl.Database.Server.GetDataAdapter(findCmd);
+                da.Fill(dt);
+            }
+        }
+        Assert.That(dt.Rows.Count, Is.EqualTo(1));
+        dt.Dispose();
+        var redactedChisCount = CatalogueRepository.GetAllObjects<RedactedCHI>().Length;
+        var rCHI = CatalogueRepository.GetAllObjects<RedactedCHI>().First();
+        var revertCmd = new ExecuteCommandRevertRedactedCHI(new ThrowImmediatelyActivator(RepositoryLocator), rCHI);
+        Assert.DoesNotThrow(() => revertCmd.Execute());
+        Assert.DoesNotThrow(() => revertCmd.Execute());
+        Assert.That(redactedChisCount, Is.EqualTo(CatalogueRepository.GetAllObjects<RedactedCHI>().Length + 1));
+        dt = new DataTable();
+        using (var con = tbl.Database.Server.GetConnection())
+        {
+            using (var findCmd = tbl.Database.Server.GetCommand($"select * from {tbl.GetRuntimeName()} where SampleType = 'R1111111111'", con))
+            {
+                using var da = tbl.Database.Server.GetDataAdapter(findCmd);
+                da.Fill(dt);
+            }
+        }
+        Assert.That(dt.Rows.Count, Is.EqualTo(1));
+        dt.Dispose();
+
+    }
+    }