feat(website): Column remapping when submitting metadata files

docs/src/content/docs/for-users/submit-sequences.md

@@ -10,6 +10,7 @@ Loculus expects:
 
 -   Sequence data in `fasta` format with a unique submissionID per sequence.
 -   Metadata in `tsv` format for each sequence. If you upload through the Website, you can also use Excel files (`xls` or `xlsx` format). If you need help formatting metadata, there is a metadata template for each organism on the submission page.
+You can also map columns in your file to the expected upload column names by clicking the 'Add column mapping' button.
 
 ![Metadata template.](../../../assets/MetadataTemplate.png)
 

kubernetes/loculus/values.yaml

@@ -486,6 +486,7 @@ defaultOrganismConfig: &defaultOrganismConfig
               - Zaire
       - name: geoLocAdmin1
         displayName: Collection subdivision level 1
+        desired: true
         generateIndex: true
         autocomplete: true
         initiallyVisible: true
@@ -494,11 +495,13 @@ defaultOrganismConfig: &defaultOrganismConfig
         ingest: division
       - name: geoLocAdmin2
         displayName: Collection subdivision level 2
+        desired: true
         generateIndex: true
         autocomplete: true
         header: Sample details
       - name: geoLocCity
         displayName: Collection city
+        desired: true
         generateIndex: true
         autocomplete: true
         header: Sample details
@@ -511,13 +514,15 @@ defaultOrganismConfig: &defaultOrganismConfig
         header: Sample details
       - name: specimenCollectorSampleId
         displayName: Isolate name
+        desired: true
         header: Sample details
         ingest: ncbiIsolateName
         enableSubstringSearch: true
       - name: authors
         displayName: Authors
         type: authors
         header: Authors
+        desired: true
         enableSubstringSearch: true
         order: 40
         truncateColumnDisplayTo: 25
@@ -529,6 +534,7 @@ defaultOrganismConfig: &defaultOrganismConfig
         columnWidth: 140
       - name: authorAffiliations
         displayName: Author affiliations
+        desired: true
         enableSubstringSearch: true
         truncateColumnDisplayTo: 15
         header: Authors
@@ -587,13 +593,15 @@ defaultOrganismConfig: &defaultOrganismConfig
         oneHeader: true
       - name: cultureId
         displayName: Culture ID
+        desired: true
         header: Sample details
       - name: sampleReceivedDate
         ontology_id: GENEPIO:0001177
         definition: The date on which the sample was received by the laboratory.
         guidance: Alternative if "sampleCollectionDate" is not available. Record the date the sample was received by the laboratory. Required granularity includes year, month and day. Before sharing this data, ensure this date is not considered identifiable information. If this date is considered identifiable, it is acceptable to add "jitter" to the received date by adding or subtracting calendar days. Do not change the received date in your original records. Alternatively, collection_date may be used as a substitute in the data you share. The date should be provided in ISO 8601 standard format "YYYY-MM-DD".
         example: '2020-03-20'
         displayName: Sample received date
+        desired: true
         type: date
         preprocessing:
           function: parse_and_assert_past_date
@@ -659,13 +667,15 @@ defaultOrganismConfig: &defaultOrganismConfig
         example: Swab [GENEPIO:0100027]
         displayName: Collection device
         header: Sampling
+        desired: true
       - name: collectionMethod
         ontology_id: GENEPIO:0001241
         definition: The process used to collect the sample e.g. phlebotomy, necropsy.
         guidance: 'Provide a descriptor if a collection method was used for sampling. Use the pick list provided in the template. If a desired term is missing from the pick list, use this look-up service to identify a standardized term: https://www.ebi.ac.uk/ols/ontologies/obi. If not applicable, leave blank.'
         example: Bronchoalveolar lavage (BAL) [GENEPIO:0100032]
         displayName: Collection method
         header: Sampling
+        desired: true
       - name: foodProduct
         ontology_id: GENEPIO:0100444
         definition: A material consumed and digested for nutritional value or enjoyment.
@@ -833,6 +843,7 @@ defaultOrganismConfig: &defaultOrganismConfig
           inputs:
             date: sequencingDate
         header: Sequencing
+        desired: true
       - name: ampliconPcrPrimerScheme
         ontology_id: GENEPIO:0001456
         definition: The specifications of the primers (primer sequences, binding positions, fragment size generated etc) used to generate the amplicons to be sequenced.
@@ -854,13 +865,15 @@ defaultOrganismConfig: &defaultOrganismConfig
         example: Oxford Nanopore MinION [GENEPIO:0100142]
         displayName: Sequencing instrument
         header: Sequencing
+        desired: true
       - name: sequencingProtocol
         ontology_id: GENEPIO:0001454
         definition: The protocol used to generate the sequence.
         guidance: 'Provide a free text description of the methods and materials used to generate the sequence. Suggested text, fill in information where indicated.: "Viral sequencing was performed following a tiling amplicon strategy using the <fill in> primer scheme. Sequencing was performed using a <fill in> sequencing instrument. Libraries were prepared using <fill in> library kit. "'
         example: Genomes were generated through amplicon sequencing of 1200 bp amplicons with Freed schema primers. Libraries were created using Illumina DNA Prep kits, and sequence data was produced using Miseq Micro v2 (500 cycles) sequencing kits.
         displayName: Sequencing protocol
         header: Sequencing
+        desired: true
       - name: sequencingAssayType
         ontology_id: GENEPIO:0100997
         definition: The overarching sequencing methodology that was used to determine the sequence of a biomaterial.
@@ -932,6 +945,7 @@ defaultOrganismConfig: &defaultOrganismConfig
         displayName: Depth of coverage
         type: int
         header: Sequencing
+        desired: true
       - name: breadthOfCoverage
         ontology_id: GENEPIO:0001475
         definition: The threshold used as a cut-off for the depth of coverage.
@@ -1003,6 +1017,7 @@ defaultOrganismConfig: &defaultOrganismConfig
         autocomplete: true
         header: "Host"
         ingest: ncbiHostName
+        desired: true
       - name: hostNameCommon
         generateIndex: true
         autocomplete: true
@@ -1016,6 +1031,7 @@ defaultOrganismConfig: &defaultOrganismConfig
           url: "https://www.ncbi.nlm.nih.gov/Taxonomy/Browser/wwwtax.cgi?mode=info&id=__value__"
         header: "Host"
         ingest: ncbiHostTaxId
+        desired: true
       - name: isLabHost
         type: boolean
         autocomplete: true

website/package.json

@@ -45,6 +45,7 @@
         "luxon": "^3.5.0",
         "neverthrow": "^8.1.1",
         "openid-client": "^5.7.1",
+        "papaparse": "^5.5.1",
         "react": "^18.3.1",
         "react-chartjs-2": "^5.3.0",
         "react-confirm-alert": "^3.0.6",
@@ -75,6 +76,7 @@
         "@types/lodash": "^4.17.14",
         "@types/luxon": "^3.4.2",
         "@types/node": "^22.10.7",
+        "@types/papaparse": "^5.3.15",
         "@types/react": "^18.3.12",
         "@types/react-dom": "^18.3.1",
         "@types/uuid": "^10.0.0",

website/src/components/MetadataTable.astro

@@ -1,5 +1,5 @@
 ---
-import { getConfiguredOrganisms, getSchema } from '../config';
+import { getConfiguredOrganisms, getGroupedInputFields, getSchema } from '../config';
 import OrganismTableSelector from './OrganismMetadataTableSelector';
 import type { OrganismMetadata } from './OrganismMetadataTableSelector';
 
@@ -9,7 +9,7 @@ const organisms: OrganismMetadata[] = configuredOrganisms.map((organism) => {
         key: organism.key,
         displayName: organism.displayName,
         metadata: getSchema(organism.key).metadata,
-        inputFields: getSchema(organism.key).inputFields,
+        groupedInputFields: getGroupedInputFields(organism.key, 'submit'),
     };
 });
 ---

website/src/components/OrganismMetadataTableSelector.tsx

@@ -1,16 +1,15 @@
-import React, { useState, useEffect } from 'react';
+import { useState, useEffect } from 'react';
 import type { FC } from 'react';
 
 import { routes } from '../routes/routes.ts';
 import type { Metadata, InputField } from '../types/config.ts';
-import { groupFieldsByHeader } from '../utils/groupFieldsByHeader.ts';
 import IwwaArrowDown from '~icons/iwwa/arrow-down';
 
 export type OrganismMetadata = {
     key: string;
     displayName: string;
     metadata: Metadata[];
-    inputFields: InputField[];
+    groupedInputFields: Map<string, InputField[]>;
 };
 
 type Props = {
@@ -40,7 +39,7 @@ const OrganismMetadataTableSelector: FC<Props> = ({ organisms }) => {
 
     useEffect(() => {
         if (selectedOrganism) {
-            setGroupedFields(groupFieldsByHeader(selectedOrganism.inputFields, selectedOrganism.metadata));
+            setGroupedFields(selectedOrganism.groupedInputFields);
         }
     }, [selectedOrganism]);
 

website/src/components/Submission/DataUploadForm.tsx

@@ -5,6 +5,8 @@ import { type FormEvent, useState } from 'react';
 
 import { dataUploadDocsUrl } from './dataUploadDocsUrl.ts';
 import { getClientLogger } from '../../clientLogger.ts';
+import type { ColumnMapping } from './FileUpload/ColumnMapping.ts';
+import { ColumnMappingModal } from './FileUpload/ColumnMappingModal.tsx';
 import { UploadComponent } from './FileUpload/UploadComponent.tsx';
 import DataUseTermsSelector from '../../components/DataUseTerms/DataUseTermsSelector';
 import useClientFlag from '../../hooks/isClient.ts';
@@ -23,7 +25,8 @@ import { dateTimeInMonths } from '../../utils/DateTimeInMonths.tsx';
 import { createAuthorizationHeader } from '../../utils/createAuthorizationHeader.ts';
 import { stringifyMaybeAxiosError } from '../../utils/stringifyMaybeAxiosError.ts';
 import { withQueryProvider } from '../common/withQueryProvider.tsx';
-import { FASTA_FILE_KIND, METADATA_FILE_KIND } from './FileUpload/fileProcessing.ts';
+import { FASTA_FILE_KIND, METADATA_FILE_KIND, type ProcessedFile, RawFile } from './FileUpload/fileProcessing.ts';
+import type { InputField } from '../../types/config.ts';
 
 export type UploadAction = 'submit' | 'revise';
 
@@ -34,6 +37,7 @@ type DataUploadFormProps = {
     action: UploadAction;
     group: Group;
     referenceGenomeSequenceNames: ReferenceGenomesSequenceNames;
+    metadataTemplateFields: Map<string, InputField[]>;
     onSuccess: () => void;
     onError: (message: string) => void;
 };
@@ -122,9 +126,12 @@ const InnerDataUploadForm = ({
     onError,
     group,
     referenceGenomeSequenceNames,
+    metadataTemplateFields,
 }: DataUploadFormProps) => {
-    const [metadataFile, setMetadataFile] = useState<File | null>(null);
-    const [sequenceFile, setSequenceFile] = useState<File | null>(null);
+    const [metadataFile, setMetadataFile] = useState<ProcessedFile | null>(null);
+    // The columnMapping can be null; if null -> don't apply mapping.
+    const [columnMapping, setColumnMapping] = useState<ColumnMapping | null>(null);
+    const [sequenceFile, setSequenceFile] = useState<ProcessedFile | null>(null);
     const [exampleEntries, setExampleEntries] = useState<number | undefined>(10);
 
     const { submit, revise, isLoading } = useSubmitFiles(accessToken, organism, clientConfig, onSuccess, onError);
@@ -145,11 +152,11 @@ const InnerDataUploadForm = ({
         const metadataFile = createTempFile(exampleMetadataContent, 'text/tab-separated-values', 'metadata.tsv');
         const sequenceFile = createTempFile(sequenceFileContent, 'application/octet-stream', 'sequences.fasta');
 
-        setMetadataFile(metadataFile);
-        setSequenceFile(sequenceFile);
+        setMetadataFile(new RawFile(metadataFile));
+        setSequenceFile(new RawFile(sequenceFile));
     };
 
-    const handleSubmit = (event: FormEvent) => {
+    const handleSubmit = async (event: FormEvent) => {
         event.preventDefault();
 
         if (!agreedToINSDCUploadTerms) {
@@ -173,12 +180,18 @@ const InnerDataUploadForm = ({
             return;
         }
 
+        let finalMetadataFile = metadataFile.inner();
+
+        if (columnMapping !== null) {
+            finalMetadataFile = await columnMapping.applyTo(metadataFile);
+        }
+
         switch (action) {
             case 'submit': {
                 const groupId = group.groupId;
                 submit({
-                    metadataFile,
-                    sequenceFile,
+                    metadataFile: finalMetadataFile,
+                    sequenceFile: sequenceFile.inner(),
                     groupId,
                     dataUseTermsType,
                     restrictedUntil:
@@ -189,7 +202,7 @@ const InnerDataUploadForm = ({
                 break;
             }
             case 'revise':
-                revise({ metadataFile, sequenceFile });
+                revise({ metadataFile: finalMetadataFile, sequenceFile: sequenceFile.inner() });
                 break;
         }
     };
@@ -255,8 +268,8 @@ const InnerDataUploadForm = ({
                                 <DevExampleData
                                     setExampleEntries={setExampleEntries}
                                     exampleEntries={exampleEntries}
-                                    metadataFile={metadataFile}
-                                    sequenceFile={sequenceFile}
+                                    metadataFile={metadataFile ? metadataFile.inner() : null}
+                                    sequenceFile={sequenceFile ? sequenceFile.inner() : null}
                                     handleLoadExampleData={handleLoadExampleData}
                                 />
                             )}
@@ -274,12 +287,22 @@ const InnerDataUploadForm = ({
                             </div>
                             <div className='w-60 space-y-2'>
                                 <label className='text-gray-900 font-medium text-sm block'>Metadata File</label>
-                                <UploadComponent
-                                    setFile={setMetadataFile}
-                                    name='metadata_file'
-                                    ariaLabel='Metadata File'
-                                    fileKind={METADATA_FILE_KIND}
-                                />
+                                <div className='flex flex-col items-center w-full'>
+                                    <UploadComponent
+                                        setFile={setMetadataFile}
+                                        name='metadata_file'
+                                        ariaLabel='Metadata File'
+                                        fileKind={METADATA_FILE_KIND}
+                                    />
+                                    {metadataFile !== null && (
+                                        <ColumnMappingModal
+                                            inputFile={metadataFile}
+                                            columnMapping={columnMapping}
+                                            setColumnMapping={setColumnMapping}
+                                            groupedInputFields={metadataTemplateFields}
+                                        />
+                                    )}
+                                </div>
                             </div>
                         </div>
                     </form>
@@ -368,7 +391,7 @@ const InnerDataUploadForm = ({
                         name='submit'
                         type='submit'
                         className='rounded-md py-2 text-sm font-semibold shadow-sm focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 bg-primary-600 text-white hover:bg-primary-500'
-                        onClick={handleSubmit}
+                        onClick={(e) => void handleSubmit(e)}
                         disabled={isLoading || !isClient}
                     >
                         <div className={`absolute ml-1.5 inline-flex ${isLoading ? 'visible' : 'invisible'}`}>