Merge branch 'develop' into task/RDMP-70Excessive-code-reuse-for-show…

…-command-to-user

.github/configs/codeql.yml

@@ -0,0 +1,12 @@
+name: CodeQL Configuration
+
+paths-ignore:
+  - "**/*.md"
+  - "**/*.txt"
+  - "**/obj/**"
+  - "**/bin/**"
+  - "**/*.Designer.cs"
+
+query-filters:
+  - exclude:
+      id: cs/catch-of-all-exceptions

.github/workflows/build.yml

@@ -195,7 +195,6 @@ jobs:
           AzureSignTool sign -kvu "${{ secrets.AZURE_KEY_VAULT_URI }}" -kvi "${{ secrets.AZURE_CLIENT_ID }}" -kvt "${{ secrets.AZURE_TENANT_ID }}" -kvs "${{ secrets.AZURE_CLIENT_SECRET }}" -kvc ${{ secrets.AZURE_CERT_NAME }} -tr http://timestamp.digicert.com -v PublishWindows/rdmp.exe
           AzureSignTool sign -kvu "${{ secrets.AZURE_KEY_VAULT_URI }}" -kvi "${{ secrets.AZURE_CLIENT_ID }}" -kvt "${{ secrets.AZURE_TENANT_ID }}" -kvs "${{ secrets.AZURE_CLIENT_SECRET }}" -kvc ${{ secrets.AZURE_CERT_NAME }} -tr http://timestamp.digicert.com -v PublishWinForms/ResearchDataManagementPlatform.exe
           mkdir -p dist
-          cmd /c wix\\build.cmd ${{ steps.version.outputs.rdmpversion }}
           (cd PublishWindows ; echo 7z a -mx=9 ../dist/rdmp-${{ steps.version.outputs.rdmpversion }}-cli-win-x64.zip . | cmd)
           (cd PublishLinux ; echo 7z a -mx=0 ../dist/rdmp-${{ steps.version.outputs.rdmpversion }}-cli-linux-x64.zip . | cmd)
           mv PublishLinux rdmp-${{ steps.version.outputs.rdmpversion }}-cli-linux
@@ -209,6 +208,7 @@ jobs:
           install-modules-with: cpanm
           install-modules: Archive::Zip Archive::Tar
       - name: Fix execute permissions
+        if: contains(github.ref, 'refs/tags/v') || contains('refs/heads/main refs/heads/develop',github.ref)
         shell: perl {0}
         run: |
           use strict;
@@ -240,6 +240,7 @@ jobs:
           $zip->memberNamed('rdmp')->unixFileAttributes( 0755 );
           $zip->overwriteAs($zipname);
       - name: Compress tar
+        if: contains(github.ref, 'refs/tags/v') || contains('refs/heads/main refs/heads/develop',github.ref)
         run: |
           7z a -txz dist/rdmp-${{ steps.version.outputs.rdmpversion }}-cli-linux-x64.tar.xz dist/rdmp-${{ steps.version.outputs.rdmpversion }}-cli-linux-x64.tar
           rm dist/rdmp-${{ steps.version.outputs.rdmpversion }}-cli-linux-x64.tar
@@ -252,6 +253,7 @@ jobs:
           dotnet pack $i -c Release --include-symbols --nologo -o . -v:m -p:Version=${{ steps.version.outputs.rdmpversion }}
           done
       - name: Calculate SHA256SUMS
+        if: contains(github.ref, 'refs/tags/v')
         run: '&{foreach ($i in Get-ChildItem dist -Exclude *SUMS|Get-FileHash) { echo "$($i.Hash) $(echo $i | Split-Path -Leaf)" }} > dist/SHA256SUMS'
       - name: Archive production artifacts
         uses: actions/upload-artifact@v4
@@ -290,4 +292,4 @@ jobs:
           file: dist/*
           tag: ${{ github.ref }}
           overwrite: true
-          file_glob: true 
+          file_glob: true 

.github/workflows/codeql.yml

@@ -31,6 +31,7 @@ jobs:
         with:
           languages: ${{ matrix.language }}
           queries: +security-and-quality
+          config-file: ./.github/configs/codeql.yml
 
       - name: Build
         run: dotnet build

AnalysisReport.sarif

@@ -1,4 +1,4 @@
-{
+{
   "$schema": "https://schemastore.azurewebsites.net/schemas/json/sarif-2.1.0-rtm.5.json",
   "version": "2.1.0",
   "runs": [
@@ -497,4 +497,4 @@
       "columnKind": "utf16CodeUnits"
     }
   ]
-}
+}

CHANGELOG.md

@@ -1,15 +1,49 @@
+
 # Changelog
 All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [8.1.5] - Unreleased
+## [8.1.6] = Unreleased
 
 ## Changed
+
+- Add Microsoft.Bcl.AsyncInterfaces 6.0.0 for plugin dependancy tree
+- Add prompt to reanem container when adding a cohort filter
+
+## [8.1.5] - 2024-04-03
+
+## Changed
+
+- Migrate from .net7 to .net8
 - Add timeout override to remote table selectors
 - Extractions now remember the last used pipeline
 - Allow for custom .bak file physical locations during data loads
+- Add ability to have multiple data loads for a single catalogue
+- Allow for Project Specific Catalogues to have multiple extraction identifiers
+- Fix issue with notification popups being inaccessable when RDMP is minimised
+- Allow for Catalogues with Non-Core extraction categories to be made Project specific
+- Bump coverlet.collector from 6.0.0 to 6.0.1
+- Bump svenstaro/upload-release-action from 2.7.0 to 2.9.0 
+- Bump Autoupdater.NET.Official from 1.8.4 to 1.8.5 
+- Bump CsvHelper from 30.0.1 to 31.0.0
+- Bump SSH.NET from 2023.0.1 to 2024.0.0 
+- Bump SixLabors.ImageSharp.Drawing from 2.1.0 to 2.1.1 
+- Bump MongoDB.Driver from 2.23.1 to 2.24.0
+- Bump NUnit from 4.0.1 to 4.1.0 
+- Bump FluentFTP from 49.0.1 to 49.0.2 
+- Bump YamlDotNet from 15.1.1 to 15.1.2
+- Bump SixLabors.ImageSharp from 3.1.2 to 3.1.3
+- Bump SixLabors.ImageSharp.Drawing from 2.1.1 to 2.1.2
+- Bump HIC.BadMedicine from 1.1.2 to 1.2.0 
+- Bump coverlet.collector from 6.0.1 to 6.0.2 
+- Bump HIC.FansiSql from 3.2.1 to 3.2.2
+- Bump NUnit.Analyzers from 4.0.1 to 4.1.0 
+- Bump Terminal.Gui from 1.15.1 to 1.16.0 
+- Bump HIC.BadMedicine from 1.2.0 to 1.2.1 
+- Bump NPOI from 2.6.2 to 2.7.0 
+
 
 ## [8.1.4] - 2024-02-19
 

Directory.Packages.props

@@ -3,22 +3,21 @@
 		<PackageVersion Include="CommandLineParser" Version="2.9.1"/>
 		<PackageVersion Include="Equ" Version="2.3.0"/>
 		<PackageVersion Include="ExcelNumberFormat" Version="1.1.0"/>
-		<PackageVersion Include="FluentFTP" Version="49.0.2"/>
+		<PackageVersion Include="FluentFTP" Version="50.0.1"/>
 		<PackageVersion Include="HIC.BadMedicine" Version="1.2.1"/>
 		<PackageVersion Include="HIC.FAnsiSql" Version="3.2.2"/>
-		<PackageVersion Include="LibArchive.Net" Version="0.1.4"/>
+		<PackageVersion Include="LibArchive.Net" Version="0.1.5"/>
 		<PackageVersion Include="Microsoft.SourceLink.GitHub" Version="8.0.0">
 			<PrivateAssets>all</PrivateAssets>
 			<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
 		</PackageVersion>
-		<PackageVersion Include="Microsoft.XmlSerializer.Generator" Version="8.0.0"/>
-		<PackageVersion Include="MongoDB.Driver" Version="2.24.0"/>
+		<PackageVersion Include="MongoDB.Driver" Version="2.25.0"/>
 		<PackageVersion Include="Newtonsoft.Json" Version="13.0.3"/>
 		<PackageVersion Include="NPOI" Version="2.7.0"/>
-		<PackageVersion Include="NLog" Version="5.2.8"/>
-		<PackageVersion Include="SixLabors.ImageSharp" Version="3.1.3"/>
-		<PackageVersion Include="SixLabors.ImageSharp.Drawing" Version="2.1.2"/>
-		<PackageVersion Include="Spectre.Console" Version="0.48.0"/>
+		<PackageVersion Include="NLog" Version="5.3.2"/>
+		<PackageVersion Include="SixLabors.ImageSharp" Version="3.1.4"/>
+		<PackageVersion Include="SixLabors.ImageSharp.Drawing" Version="2.1.3"/>
+		<PackageVersion Include="Spectre.Console" Version="0.49.1"/>
 		<PackageVersion Include="SSH.NET" Version="2024.0.0"/>
 		<PackageVersion Include="Terminal.Gui" Version="1.16.0"/>
 		<PackageVersion Include="YamlDotNet" Version="15.1.2"/>
@@ -35,9 +34,10 @@
 		<PackageVersion Include="HIC.System.Windows.Forms.DataVisualization" Version="1.0.1"/>
 		<PackageVersion Include="NunitXml.TestLogger" Version="3.1.20"/>
 		<PackageVersion Include="NUnit3TestAdapter" Version="4.5.0"/>
-		<PackageVersion Include="NUnit.Analyzers" Version="4.1.0"/>
+		<PackageVersion Include="NUnit.Analyzers" Version="4.2.0"/>
 		<PackageVersion Include="NSubstitute" Version="5.1.0"/>
 		<PackageVersion Include="Microsoft.NET.Test.Sdk" Version="17.9.0"/>
 		<PackageVersion Include="coverlet.collector" Version="6.0.2"/>
+		<PackageVersion Include="Microsoft.Bcl.AsyncInterfaces" Version="6.0.0" />
 	</ItemGroup>
 </Project>

Documentation/CodeTutorials/Packages.md

@@ -13,7 +13,6 @@
 | FluentFTP | [Github](https://github.com/robinrodricks/FluentFTP/) | [MIT](https://opensource.org/licenses/MIT) | FTP(S) client | |
 | MongoDB.Driver | [GitHub](https://github.com/mongodb/mongo-csharp-driver) | [Apache 2.0](https://opensource.org/licenses/Apache-2.0) | Database driver for MongoDB | |
 | Microsoft.SourceLink.GitHub | [GitHub](https://github.com/dotnet/sourcelink) | [MIT](https://opensource.org/licenses/MIT) | Enable source linkage from nupkg | Official MS project |
-| Microsoft.XmlSerializer.Generator | [Microsoft](https://learn.microsoft.com/en-us/dotnet/core/additional-tools/xml-serializer-generator) | [MIT](https://opensource.org/licenses/MIT) | XML handling improvements |
 | ObjectListView.Repack.NET6Plus | [GitHub](https://github.com/nasisakk/ObjectListViewRepack) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | |
 | Scintilla.NET | [GitHub](https://github.com/VPKSoft/Scintilla.NET) | [MIT](https://opensource.org/licenses/MIT) | |
 | VPKSoft.ScintillaLexers.NET | [GitHub](https://github.com/VPKSoft/ScintillaLexers) | [MIT](https://opensource.org/licenses/MIT) | |
@@ -41,4 +40,5 @@
 | Autoupdater.NET.Official | [GitHub](https://github.com/ravibpatel/AutoUpdater.NET) | MIT | Manages updating of the RDMP windows client directly from the RDMP GitHub Releases|
 | ConsoleControl | [GitHub](https://github.com/dwmkerr/consolecontrol)  | MIT | Runs RDMP cli subprocesses|
 | Terminal.Gui                            | [GitHub](https://github.com/gui-cs/Terminal.Gui)                           | [MIT](https://opensource.org/licenses/MIT)                                                     | Console user-interface|
+| Microsoft.Bcl.AsyncInterfaces | [GitHub](https://github.com/dotnet/runtime) |[MIT](https://opensource.org/licenses/MIT)  | |
 [DBMS]: ./Glossary.md#DBMS

HIC.DataManagementPlatform.sln

@@ -315,4 +315,4 @@ Global
 	GlobalSection(Performance) = preSolution
 		HasPerformanceSessions = true
 	EndGlobalSection
-EndGlobal
+EndGlobal

Rdmp.Core.Tests/CommandExecution/ExecuteCommandConfirmLogsTests.cs

@@ -1,4 +1,4 @@
-// Copyright (c) The University of Dundee 2018-2019
+// Copyright (c) The University of Dundee 2018-2024
 // This file is part of the Research Data Management Platform (RDMP).
 // RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
 // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
@@ -24,11 +24,10 @@ public void ConfirmLogs_NoEntries_Throws()
         var lmd = new LoadMetadata(CatalogueRepository, "MyLmd");
         var cata = new Catalogue(CatalogueRepository, "myCata")
         {
-            LoadMetadata_ID = lmd.ID,
             LoggingDataTask = "GGG"
         };
         cata.SaveToDatabase();
-
+        lmd.LinkToCatalogue(cata);
         var lm = new LogManager(lmd.GetDistinctLoggingDatabase());
         lm.CreateNewLoggingTaskIfNotExists("GGG");
 
@@ -45,11 +44,10 @@ public void ConfirmLogs_HappyEntries_Passes(bool withinTime)
         var lmd = new LoadMetadata(CatalogueRepository, "MyLmd");
         var cata = new Catalogue(CatalogueRepository, "myCata")
         {
-            LoadMetadata_ID = lmd.ID,
             LoggingDataTask = "FFF"
         };
         cata.SaveToDatabase();
-
+        lmd.LinkToCatalogue(cata);
         var lm = new LogManager(lmd.GetDistinctLoggingDatabase());
         lm.CreateNewLoggingTaskIfNotExists("FFF");
         var logEntry = lm.CreateDataLoadInfo("FFF", "pack o' cards", "going down gambling", null, true);
@@ -70,11 +68,10 @@ public void ConfirmLogs_SadEntry_BecauseNeverEnded_Throws()
         var lmd = new LoadMetadata(CatalogueRepository, "MyLmd");
         var cata = new Catalogue(CatalogueRepository, "myCata")
         {
-            LoadMetadata_ID = lmd.ID,
             LoggingDataTask = "FFF"
         };
         cata.SaveToDatabase();
-
+        lmd.LinkToCatalogue(cata);
         var lm = new LogManager(lmd.GetDistinctLoggingDatabase());
         lm.CreateNewLoggingTaskIfNotExists("FFF");
 
@@ -93,11 +90,10 @@ public void ConfirmLogs_SadEntryWithEx_Throws()
         var lmd = new LoadMetadata(CatalogueRepository, "MyLmd");
         var cata = new Catalogue(CatalogueRepository, "myCata")
         {
-            LoadMetadata_ID = lmd.ID,
             LoggingDataTask = "FFF"
         };
         cata.SaveToDatabase();
-
+        lmd.LinkToCatalogue(cata);
         var lm = new LogManager(lmd.GetDistinctLoggingDatabase());
         lm.CreateNewLoggingTaskIfNotExists("FFF");
         var logEntry = lm.CreateDataLoadInfo("FFF", "pack o' cards", "going down gambling", null, true);
@@ -116,11 +112,10 @@ public void ConfirmLogs_NotWithinTime_Throws()
         var lmd = new LoadMetadata(CatalogueRepository, "MyLmd");
         var cata = new Catalogue(CatalogueRepository, "myCata")
         {
-            LoadMetadata_ID = lmd.ID,
             LoggingDataTask = "FFF"
         };
         cata.SaveToDatabase();
-
+        lmd.LinkToCatalogue(cata);
         var lm = new LogManager(lmd.GetDistinctLoggingDatabase());
         lm.CreateNewLoggingTaskIfNotExists("FFF");
         var logEntry = lm.CreateDataLoadInfo("FFF", "pack o' cards", "going down gambling", null, true);
@@ -144,19 +139,18 @@ public void ConfirmLogs_With2CacheProgress_Throws()
         var lmd1 = new LoadMetadata(CatalogueRepository, "MyLmd");
         var cata = new Catalogue(CatalogueRepository, "myCata")
         {
-            LoadMetadata_ID = lmd1.ID,
             LoggingDataTask = "B"
         };
         cata.SaveToDatabase();
-
+        lmd1.LinkToCatalogue(cata);
         var lmd2 = new LoadMetadata(CatalogueRepository, "MyLmd");
         var cata2 = new Catalogue(CatalogueRepository, "myCata")
         {
-            LoadMetadata_ID = lmd2.ID,
             LoggingDataTask = "A"
         };
         cata2.SaveToDatabase();
-
+        var linkage2 = new LoadMetadataCatalogueLinkage(CatalogueRepository, lmd2, cata2);
+        linkage2.SaveToDatabase();
         var lp1 = new LoadProgress(CatalogueRepository, lmd1);
         var lp2 = new LoadProgress(CatalogueRepository, lmd2);
 

Rdmp.Core.Tests/CommandExecution/ExecuteCommandLinkCoulmnInfoWithDatasetTests.cs

@@ -34,7 +34,6 @@ public void TestLinkColumnInfoToDataset()
         var cmd = new ExecuteCommandCreateDataset(GetMockActivator(), "dataset");
         Assert.DoesNotThrow(() => cmd.Execute());
         var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects<Core.Curation.Data.Dataset>().First();
-        var foundCatalogue = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects<Catalogue>().Where(c => c.Name == "Dataset1").First();
         var linkCmd = new ExecuteCommandLinkColumnInfoToDataset(GetMockActivator(), _c1, founddataset);
         Assert.DoesNotThrow(() => linkCmd.Execute());
         var columInfo = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects<ColumnInfo>();
@@ -69,7 +68,6 @@ public void TestLinkColumnInfoToDatasetNotAll()
         var cmd = new ExecuteCommandCreateDataset(GetMockActivator(), "dataset");
         Assert.DoesNotThrow(() => cmd.Execute());
         var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects<Core.Curation.Data.Dataset>().First();
-        var foundCatalogue = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects<Catalogue>().Where(c => c.Name == "Dataset1").First();
         var linkCmd = new ExecuteCommandLinkColumnInfoToDataset(GetMockActivator(), _c1, founddataset, false);
         Assert.DoesNotThrow(() => linkCmd.Execute());
         var columInfo = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects<CatalogueItem>().Where(ci => _cata1.CatalogueItems.Contains(ci));
@@ -99,7 +97,6 @@ public void TestLinkColumInfoToDatasetBadDataset()
     {
         var cmd = new ExecuteCommandCreateDataset(GetMockActivator(), "dataset");
         Assert.DoesNotThrow(() => cmd.Execute());
-        var founddataset = GetMockActivator().RepositoryLocator.CatalogueRepository.GetAllObjects<Core.Curation.Data.Dataset>().First();
         var linkCmd = new ExecuteCommandLinkColumnInfoToDataset(GetMockActivator(), new ColumnInfo(), null, false);
         Assert.Throws<NullReferenceException>(() => linkCmd.Execute());
     }

Rdmp.Core.Tests/CommandExecution/TestExecuteCommandAssociateCatalogueWithLoadMetadata.cs

@@ -22,8 +22,8 @@ public void TestExecuteCommandAssociateCatalogueWithLoadMetadata_Simple()
 
         Assert.Multiple(() =>
         {
-            Assert.That(cata1.LoadMetadata, Is.Null);
-            Assert.That(cata2.LoadMetadata, Is.Null);
+            Assert.That(cata1.LoadMetadatas(), Is.Empty);
+            Assert.That(cata2.LoadMetadatas(), Is.Empty);
         });
 
         var lmd = new LoadMetadata(RepositoryLocator.CatalogueRepository, "mylmd");
@@ -36,8 +36,8 @@ public void TestExecuteCommandAssociateCatalogueWithLoadMetadata_Simple()
 
         Assert.Multiple(() =>
         {
-            Assert.That(cata1.LoadMetadata_ID, Is.EqualTo(lmd.ID));
-            Assert.That(cata2.LoadMetadata, Is.Null);
+            Assert.That(cata1.LoadMetadatas()[0].ID, Is.EqualTo(lmd.ID));
+            Assert.That(cata2.LoadMetadatas(), Is.Empty);
         });
     }
 }

Rdmp.Core.Tests/CommandLine/AutomationLoopTests/EndToEndCacheTest.cs

@@ -1,4 +1,4 @@
-// Copyright (c) The University of Dundee 2018-2019
+// Copyright (c) The University of Dundee 2018-2024
 // This file is part of the Research Data Management Platform (RDMP).
 // RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
 // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
@@ -56,10 +56,12 @@ protected override void SetUp()
 
         _cata = new Catalogue(CatalogueRepository, "EndToEndCacheTest")
         {
-            LoadMetadata_ID = _lmd.ID
         };
         _cata.SaveToDatabase();
 
+        var linkage = new LoadMetadataCatalogueLinkage(CatalogueRepository, _lmd, _cata);
+        linkage.SaveToDatabase();
+
         _lp = new LoadProgress(CatalogueRepository, _lmd);
         _cp = new CacheProgress(CatalogueRepository, _lp);
 

Rdmp.Core.Tests/CommandLine/AutomationLoopTests/EndToEndDLETest.cs

@@ -1,4 +1,4 @@
-// Copyright (c) The University of Dundee 2018-2019
+// Copyright (c) The University of Dundee 2018-2024
 // This file is part of the Research Data Management Platform (RDMP).
 // RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
 // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
@@ -52,9 +52,8 @@ public void TestDle_DodgyColumnNames(DatabaseType dbType)
 
         CreateFlatFileAttacher(lmd, "Troll.csv", cata.GetTableInfoList(false).Single());
 
-        cata.LoadMetadata_ID = lmd.ID;
         cata.SaveToDatabase();
-
+        lmd.LinkToCatalogue(cata);
         Assert.That(tbl.GetRowCount(), Is.EqualTo(0));
 
         RunDLE(lmd, 30000, true);