diff --git a/drc-portals/app/info/documentation/C2M2/[doc]/page.tsx b/drc-portals/app/info/documentation/C2M2/[doc]/page.tsx new file mode 100644 index 00000000..85eb162b --- /dev/null +++ b/drc-portals/app/info/documentation/C2M2/[doc]/page.tsx @@ -0,0 +1,50 @@ +import { Grid } from "@mui/material"; +import ReactMarkdown from 'react-markdown' +import remarkGfm from "remark-gfm" +import { fetchC2m2Markdown, LinkRenderer, HeadingRenderer } from '@/components/misc/ReactMarkdownRenderers' + +export default async function StandardsPage( + { params } : { params: { doc: string } } +) { + function fixProblematic ( page: string ) { + const problematic = [ + '-subject_phenotype.tsv', + '-phenotype_gene.tsv', + '-phenotype_disease.tsv', + '-phenotype.tsv', + '-collection_phenotype.tsv', + '-collection_disease.tsv', + '-analysis_type.tsv' + ] + for (let pg of problematic) { + if (page.endsWith(pg)) { + return page.replace('TableInfo', 'Tableinfo').concat('.md') + } + } + return page.concat('.md') + } + + const suffix = fixProblematic(params.doc) + const markdown = await fetchC2m2Markdown(suffix) + const title = '# ' + suffix.split('-')[1].split('.md')[0] + '\n\n' + return ( + + +
+ + {''.concat( + title, + markdown.replace('https://docs.nih-cfde.org/en/latest/c2m2/draft-C2M2_specification/', 'https://github.com/nih-cfde/c2m2/blob/master/draft-C2M2_specification/'), + '\n#### Return to [C2M2 Documentation](./)' + )} + + + + ) +} \ No newline at end of file diff --git a/drc-portals/app/info/documentation/C2M2/intro.md b/drc-portals/app/info/documentation/C2M2/intro.md new file mode 100644 index 00000000..dfb6c169 --- /dev/null +++ b/drc-portals/app/info/documentation/C2M2/intro.md @@ -0,0 +1,40 @@ +## Introduction +The Crosscut Metadata Model (C2M2) is a flexible metadata standard for describing experimental resources in biomedicine and related fields. A complete C2M2 submission, also called an "instance" or a "datapackage", is a zipped folder containing multiple tab-delimited files (TSVs) representing metadata records along with a JSON schema. To read more about datapackages, skip to the [Frictionless Data Packages](#frictionless-data-packages) section. + +Each TSV file is a **data table** containing various **data records** (rows) and their values for different **fields** (columns). **Entity tables** describe various types of data objects, while **association tables** describe the relationships between different entities. + +This page is adapted from the [original C2M2 documentation](https://github.com/nih-cfde/published-documentation) developed by the CFDE Coordination Center (CFDE-CC). + +## Resources +* The [c2m2-frictionless-dataclass](https://github.com/nih-cfde/c2m2-frictionless-dataclass/tree/main) + - This repository includes the `c2m2-frictionless` Python package, which contains specific helper functions for C2M2 datapackage building and validation. + - This package is not designed to generate a complete C2M2 datapackage from any given data, but should be used in collaboration with the provided schema and ontology preparation scripts, as well as with DCC-specific scripts. +* The most up-to-date [C2M2 JSON schema](https://osf.io/c63aw/) +* The most up-to-date [C2M2 ontology preparation script and files](https://osf.io/bq6k9/) +* The original [C2M2 documentation](https://github.com/nih-cfde/c2m2/blob/master/draft-C2M2_specification/README.md) from the CFDE Coordination Center contains more details on the concepts discussed here. + +## Frictionless Data Packages +C2M2 instances are also known as "datapackages" based on the [Data Package](http://frictionlessdata.io/docs/data-package/) meta-specification from [Frictionless Data](http://frictionlessdata.io/). + +From the original C2M2 documentation: +> The Data Package meta-specification is a platform-agnostic toolkit for defining format and content requirements for files so that automatic validation can be performed on those files, just as a database management system stores definitions for database tables and automatically validates incoming data based on those definitions. Using this toolkit, the C2M2 JSON Schema specification defines foreign-key relationships between metadata fields (TSV columns), rules governing missing data, required content types and formats for particular fields, and other similar database management constraints. These architectural rules help to guarantee the internal structural integrity of each C2M2 submission, while also serving as a baseline standard to create compatibility across multiple submissions received from different DCCs. + +## Identifiers +In order to standardize and integrate information across DCCs, there must be a system of assigning unambiguous identifiers to individual DCC concepts and resources. These are the "C2M2 IDs", consisting of a `id_namespace` prefix and `local_id` suffix. Additionally, the C2M2 also allows individual resources to be assigned a `persistent_id`. + +From the original C2M2 documentation: +> Optional `persistent_id` identifiers are meant to be stable enough to be scientifically cited, and to provide for further investigation by accessing related resolver services. To be used as a C2M2 `persistent_id`, an ID +> 1. will represent an explicit commitment by the managing DCC that the attachment of the ID to the resource it represents is permanent and final +> 2. must be a format-compliant URI or a compact identifier, where the protocol (the "scheme" or "prefix") specified in the ID is registered with at least one of the following (see the given lists for examples of URIs and compact identifiers) +> - the IANA (list of registered schemes) +> - scheme used must be assigned either "Permanent" or "Provisional" status +> - Identifiers.org (list of registered prefixes) +> - N2T (Name-To-Thing) (list of registered prefixes) +> - protocols not appearing in the above registries but explicitly approved by the CFDE-CC. Currently, this list is limited to one protocol, namely drs:// URIs identifying GA4GH Data Repository Service resources. +> 3. if representing a file, an ID used as a `persistent_id` cannot be a direct-download URL for that file: it must instead be an identifier permanently attached to the file and only indirectly resolvable (through the scheme or prefix specified within the ID) to the file itself + +## C2M2 Tables + +*Sourced from the [CFDE Coordination Center Documentation Wiki](https://github.com/nih-cfde/published-documentation/wiki/C2M2-Table-Summary)* + +All of the tables in a C2M2 datapackage are inter-linked via foreign key relationships, as shown in the following diagram of the complete C2M2 system. \ No newline at end of file diff --git a/drc-portals/app/info/documentation/C2M2/page.tsx b/drc-portals/app/info/documentation/C2M2/page.tsx new file mode 100644 index 00000000..a4c8c8c8 --- /dev/null +++ b/drc-portals/app/info/documentation/C2M2/page.tsx @@ -0,0 +1,62 @@ +import { Grid, Typography } from "@mui/material"; +import ReactMarkdown from 'react-markdown' +import remarkGfm from 'remark-gfm' +import { fetchC2m2Markdown, LinkRenderer, HeadingRenderer } from '@/components/misc/ReactMarkdownRenderers' +import { readFileSync } from 'fs' + +const toc = ` +## Table of Contents +* [Introduction](#introduction) +* [Resources](#resources) +* [Frictionless Data Packages](#frictionless-data-packages) +* [Identifiers](#identifiers) +* [C2M2 Tables](#c2m2-tables) +* [Submission Prep Script](#submission-prep-script) +* [General Steps](#general-steps) +* [Tutorial](#tutorial) +` + +export default async function C2M2Page() { + const intro = readFileSync( + 'app/info/documentation/C2M2/intro.md', + {encoding:'utf8', flag:'r'} + ) + const c2m2Tables = await fetchC2m2Markdown( + 'C2M2-Table-Summary.md' + ) + const submissionPrep = await fetchC2m2Markdown( + 'submission-prep-script.md' + ) + const tutorial = readFileSync( + 'app/info/documentation/C2M2/tutorial.md', + {encoding:'utf8', flag:'r'} + ) + return ( + + + C2M2 + + +
+ + {''.concat( + toc, // table of contents + intro, // intro + c2m2Tables.replaceAll('./', './C2M2/').replace('/submission-prep-script', '#submission-prep-script'), // C2M2 table descriptions from original docs + '\n## Submission Prep Script\n', + '*Sourced from the [CFDE Coordination Center Documentation Wiki](https://github.com/nih-cfde/published-documentation/wiki/submission-prep-script)*\n', + submissionPrep.replace('## Overview', '').replace('## Usage', '## General Steps').split('This script is under')[0], // + tutorial, // overview of steps + '\n#### Return to [Documentation](./)' + )} + + + + ) +} \ No newline at end of file diff --git a/drc-portals/app/info/documentation/C2M2/tutorial.md b/drc-portals/app/info/documentation/C2M2/tutorial.md new file mode 100644 index 00000000..244263fd --- /dev/null +++ b/drc-portals/app/info/documentation/C2M2/tutorial.md @@ -0,0 +1,104 @@ +## Tutorial +For the April 2022 CFDE Cross-Pollination meeting, the LINCS DCC demonstrated a Jupyter Notebook tutorial on building the `file`, `biosample`, and `subject` tables for LINCS L1000 signature data. The code and files can be found at the following link: + +[LINCS C2M2 Demo (04-05-2022 Cross-Pollination)](https://github.com/nih-cfde/LINCS-metadata/tree/main/scripts/April-CrossPollination-Demo) + +Code snippets from this tutorial corresponding to each step are reproduced below. Note that the C2M2 datapackage building process will vary across DCCs, depending on the types of generated data, ontologies, and access standards in place. In general, the process will be as follows: + +1. Become familiar with the current structure of the C2M2, including the required fields across the entity and association tables, and download the most recent version of the [JSON schema](https://osf.io/c63aw/). Gather any metadata mapping files you may need. + +2. Identify the relevant namespace for all files, and build the `id_namespace` and `dcc` tables first. + + - For LINCS, the namespace "https://lincsproject.org" is used. The following code will generate the LINCS `id_namespace` table: + ``` + pd.DataFrame( + [ + [ + 'https://www.lincsproject.org', # id + 'LINCS', # abbreviation + 'Library of Integrated Network-Based Cellular Signatures', # name + 'A network-based understanding of biology' # description + ] + ], + columns=['id', 'abbreviation', 'name', 'description'] + ).to_csv('id_namespace.tsv', sep='\t', index=False) + ``` + +3. Identify all relevant projects and their associated files that will be included in the C2M2 datapackage. Generate container entity tables (`project`, `collection`) that describe logic, theme, or funding-based groups of the core entities. Note that projects and collections may be nested, in the `project_in_project` and `collection_in_collection` tables. + + - In the LINCS tutorial, the data comes from the 2021 release of the LINCS L1000 Connectivity Map dataset. In creating a project representing the files from this dataset, there must also be an overarching root project for all LINCS data. + ``` + pd.DataFrame( + [ + [ + 'https://www.lincsproject.org', # id_namespace + 'LINCS', # local_id + 'https://www.lincsproject.org', # persistent_id + date(2013, 1, 1), # creation_time + 'LINCS', # abbreviation + 'Library of Integrated Network-Based Cellular Signatures', # name + 'A network-based understanding of biology' # description + ], + [ + 'https://www.lincsproject.org', # id_namespace + 'LINCS-2021', # local_id + 'https://clue.io/data/CMap2020#LINCS2020', # persistent_id + date(2020, 11, 20), # creation_time + 'LINCS-2021', # abbreviation + 'LINCS 2021 Data Release', # name + 'The 2021 beta release of the CMap LINCS resource' # description + ] + ], + columns=[ + 'id_namespace', 'local_id', 'persistent_id', 'creation_time', + 'abbreviation', 'name', 'description' + ] + ).to_csv('project.tsv', sep='\t', index=False) + + pd.DataFrame( + [ + [ + 'https://www.lincsproject.org', # parent_project_id_namespace + 'LINCS', # parent_project_local_id + 'https://www.lincsproject.org/', # child_project_id_namespace + 'LINCS-2021' # child_project_local_id + ] + ], + columns=[ + 'parent_project_id_namespace', 'parent_project_local_id', + 'child_project_id_namespace', 'child_project_local_id' + ] + ).to_csv('project_in_project.tsv', sep='\t', index=False) + ``` + +4. Determine the relationships between files and their associated samples or biological subjects, as well as all relevant assay types, analysis methods, data types, file formats, etc. Also identify all appropriate ontological mappings, if any, corresponding to each value from above. + + - The LINCS DCC includes internal drug, gene, and cell line identifiers, which were mapped to PubChem, Ensembl, and Disease Ontology/UBERON manually ahead of time, but other DCCs may already make use of the CFDE-supported ontologies. + - For instance, the LINCS signature `L1000_LINCS_DCIC_ABY001_A375_XH_A16_lapatinib_10uM.tsv.gz` comes from the biosample `ABY001_A375_XH_A16_lapatinib_10uM` (in this case an experimental condition) and the subject cell line `A375`; has a data type of expression matrix (`data:0928`); is stored as a TSV file format (`format:3475`) with GZIP compression format (`format:3989`); and has a MIME type of `text/tab-separated-values`. + - The `ABY001_A375_XH_A16_lapatinib_10uM` biosample was obtained via the L1000 sequencing assay type (`OBI:0002965`); comes from a cell line derived from the skin (`UBERON:0002097`); and was treated with the compound lapatinib (`CID:208908`). + +6. Either manually or programmatically, generate each data table, starting with the core entity tables (`file`, `biosample`, `subject`). This step will depend entirely on the format of a DCC's existing metadata and ontology mapping tables. + +7. Generate the inter-entity linkage association tables (`file_describes_biosample`, `file_describes_subject`, `biosample_from_subject`). + + - In the LINCS tutorial, since the filenames come directly from the biosamples, once the `file` and `biosample` tables have both been built, `file_describes_biosample` can be generated. + ``` + fdb = file_df[['id_namespace', 'local_id']].copy() + fdb = fdb.rename( + columns={ + 'id_namespace': 'file_id_namespace', + 'local_id': 'file_local_id' + } + ) + fdb['biosample_id_namespace'] = fdb['file_id_namespace'] + fdb['biosample_local_id'] = fdb['file_local_id'].apply(file_2_biosample_map_function) + ``` + +8. Assign files, biosamples, and subjects to any collections, if applicable, using the `file_in_collection`, `subject_in_collection`, and `biosample_in_collection` tables. + + - Collections are optional, and can represent files from the same publications or other logical groupings outside of funding. + +9. Use provided [C2M2 submission preparation script and ontology support files](https://osf.io/bq6k9/) to automatically build term entity tables from your created files. + +10. Validate the final datapackage, containing all files and the schema, and submit! + diff --git a/drc-portals/app/info/documentation/markdown/C2M2.md b/drc-portals/app/info/documentation/markdown/C2M2.md deleted file mode 100644 index 8bb277a5..00000000 --- a/drc-portals/app/info/documentation/markdown/C2M2.md +++ /dev/null @@ -1,275 +0,0 @@ -## Table of Contents -- [Table of Contents](#table-of-contents) -- [Introduction](#introduction) -- [Tutorials and Examples](#tutorials-and-examples) -- [Helper Packages](#helper-packages) -- [General Steps](#general-steps) -- [C2M2 Tables](#c2m2-tables) - - [Core Entity Tables](#core-entity-tables) - - [Container Entity Tables](#container-entity-tables) - - [Association Tables](#association-tables) - - [Container Entity Tables](#container-entity-tables-1) - - [Association Tables](#association-tables-1) - - [Controlled Vocabularies](#controlled-vocabularies) - -## Introduction -The Crosscut Metadata Model (C2M2) is a flexible metadata standard for describing experimental resources in biomedicine and related fields. A complete C2M2 submission, also called an "instance" or a "datapackage", is a zipped folder containing multiple tab-delimited files (TSVs) representing metadata records along with a JSON schema. - -Each TSV file is a **data table** containing various **data records** (rows) and their values for different **fields** (columns). **Entity tables** describe various types of data objects, while **association tables** describe the relationships between different entities. - -## Tutorials and Examples -For the April 2022 CFDE Cross-Pollination meeting, the LINCS DCC demonstrated a Jupyter Notebook tutorial on building the `file`, `biosample`, and `subject` tables for LINCS L1000 signature data. The code and files can be found at the following link: - -[LINCS C2M2 Demo (04-05-2022 Cross-Pollination)](https://github.com/nih-cfde/LINCS-metadata/tree/main/scripts/April-CrossPollination-Demo) - -Code snippets from this tutorial corresponding to each step are reproduced below. - -## Helper Packages -The [c2m2-frictionless-dataclass](https://github.com/nih-cfde/c2m2-frictionless-dataclass/tree/main) repository includes the `c2m2-frictionless` Python package, which contains specific helper functions for C2M2 datapackage building and validation. This package is not designed to generate a complete C2M2 datapackage from any given data, but should be used in collaboration with the provided schema and ontology preparation scripts, as well as with DCC-specific scripts. - -## General Steps -The C2M2 datapackage building process will vary across DCCs, depending on the types of generated data, ontologies, and access standards in place. In general, the process will be as follows: - -1. Become familiar with the current structure of the C2M2, including the required fields across the entity and association tables, and download the most recent version of the [JSON schema](https://osf.io/c63aw/). Gather any metadata mapping files you may need. - -2. Identify the relevant namespace for all files, and build the `id_namespace` and `dcc` tables first. - - - For LINCS, the namespace "https://lincsproject.org" is used. The following code will generate the LINCS `id_namespace` table: - ``` - pd.DataFrame( - [ - [ - 'https://www.lincsproject.org', # id - 'LINCS', # abbreviation - 'Library of Integrated Network-Based Cellular Signatures', # name - 'A network-based understanding of biology' # description - ] - ], - columns=['id', 'abbreviation', 'name', 'description'] - ).to_csv('id_namespace.tsv', sep='\t', index=False) - ``` - -3. Identify all relevant projects and their associated files that will be included in the C2M2 datapackage. Generate container entity tables (`project`, `collection`) that describe logic, theme, or funding-based groups of the core entities. Note that projects and collections may be nested, in the `project_in_project` and `collection_in_collection` tables. - - - In the LINCS tutorial, the data comes from the 2021 release of the LINCS L1000 Connectivity Map dataset. In creating a project representing the files from this dataset, there must also be an overarching root project for all LINCS data. - ``` - pd.DataFrame( - [ - [ - 'https://www.lincsproject.org', # id_namespace - 'LINCS', # local_id - 'https://www.lincsproject.org', # persistent_id - date(2013, 1, 1), # creation_time - 'LINCS', # abbreviation - 'Library of Integrated Network-Based Cellular Signatures', # name - 'A network-based understanding of biology' # description - ], - [ - 'https://www.lincsproject.org', # id_namespace - 'LINCS-2021', # local_id - 'https://clue.io/data/CMap2020#LINCS2020', # persistent_id - date(2020, 11, 20), # creation_time - 'LINCS-2021', # abbreviation - 'LINCS 2021 Data Release', # name - 'The 2021 beta release of the CMap LINCS resource' # description - ] - ], - columns=[ - 'id_namespace', 'local_id', 'persistent_id', 'creation_time', - 'abbreviation', 'name', 'description' - ] - ).to_csv('project.tsv', sep='\t', index=False) - - pd.DataFrame( - [ - [ - 'https://www.lincsproject.org', # parent_project_id_namespace - 'LINCS', # parent_project_local_id - 'https://www.lincsproject.org/', # child_project_id_namespace - 'LINCS-2021' # child_project_local_id - ] - ], - columns=[ - 'parent_project_id_namespace', 'parent_project_local_id', - 'child_project_id_namespace', 'child_project_local_id' - ] - ).to_csv('project_in_project.tsv', sep='\t', index=False) - ``` - -4. Determine the relationships between files and their associated samples or biological subjects, as well as all relevant assay types, analysis methods, data types, file formats, etc. Also identify all appropriate ontological mappings, if any, corresponding to each value from above. - - - The LINCS DCC includes internal drug, gene, and cell line identifiers, which were mapped to PubChem, Ensembl, and Disease Ontology/UBERON manually ahead of time, but other DCCs may already make use of the CFDE-supported ontologies. - - For instance, the LINCS signature `L1000_LINCS_DCIC_ABY001_A375_XH_A16_lapatinib_10uM.tsv.gz` comes from the biosample `ABY001_A375_XH_A16_lapatinib_10uM` (in this case an experimental condition) and the subject cell line `A375`; has a data type of expression matrix (`data:0928`); is stored as a TSV file format (`format:3475`) with GZIP compression format (`format:3989`); and has a MIME type of `text/tab-separated-values`. - - The `ABY001_A375_XH_A16_lapatinib_10uM` biosample was obtained via the L1000 sequencing assay type (`OBI:0002965`); comes from a cell line derived from the skin (`UBERON:0002097`); and was treated with the compound lapatinib (`CID:208908`). - -6. Either manually or programmatically, generate each data table, starting with the core entity tables (`file`, `biosample`, `subject`). This step will depend entirely on the format of a DCC's existing metadata and ontology mapping tables. - -7. Generate the inter-entity linkage association tables (`file_describes_biosample`, `file_describes_subject`, `biosample_from_subject`). - - - In the LINCS tutorial, since the filenames come directly from the biosamples, once the `file` and `biosample` tables have both been built, `file_describes_biosample` can be generated. - ``` - fdb = file_df[['id_namespace', 'local_id']].copy() - fdb = fdb.rename( - columns={ - 'id_namespace': 'file_id_namespace', - 'local_id': 'file_local_id' - } - ) - fdb['biosample_id_namespace'] = fdb['file_id_namespace'] - fdb['biosample_local_id'] = fdb['file_local_id'].apply(file_2_biosample_map_function) - ``` - -8. Assign files, biosamples, and subjects to any collections, if applicable, using the `file_in_collection`, `subject_in_collection`, and `biosample_in_collection` tables. - - - Collections are optional, and can represent files from the same publications or other logical groupings outside of funding. - -9. Use provided [C2M2 submission preparation script and ontology support files](https://osf.io/bq6k9/) to automatically build term entity tables from your created files. - -10. Validate the final datapackage, containing all files and the schema, and submit! - -## C2M2 Tables - -### Core Entity Tables -The basic experimental resources within the C2M2 are `file`, `biosample`, and `subject`. Each of these tables share a set of common metadata fields: - -- `id_namespace`: URI-prefix identifier -- `local_id`: URi-suffix identifier -- `persistent_id`: file_Resolvable URI permanently attached to an entity (e.g. download link or landing page) -- `creation_time`: Timestampe of entity creation/observation in YYYY-MM-DDTHH:MM:SS±NN:NN format -- `project_id_namespace`: URI-prefix identifier of overarching project for which the entity was created -- `project_local_id`: URI-suffix identifier of overarching project for which the entity was created - -Additionally, each table has separate unique metadata fields: - -**File:** A stable digital asset -- `size_in_bytes`: Filesize in bytes -- `uncompressed_size_in_bytes`: Fileize in bytes when decompressed -- `sha256`: SHA-256 checksum of the file -- `md5`: MD5 checksum of the file -- `filename`: Name of file with no path prefixes -- `file_format`: EDAM ID for the digital format of the file -- `compression_format`: EDAM ID for the compression method of the file, if applicable -- `data_type`: EDAM ID for data stored in the file -- `assay_type`: OBI ID for type of experiment performed to generate the file -- `analysis_type`: OBI ID for any analytic operation performed on file -- `mime_type`: [MIME](https://www.iana.org/assignments/media-types/media-types.xhtml) type describing this file - -**Biosample:** A tissue sample or physical specimen -- `sample_prep_method`: OBI ID for sample preparation method -- `anatomy`: UBERON ID for tissue origin of sample, if applicable - -**Subject:** A biological entity from which a biosample can be generated -- `granularity`: One of multiple [classes](https://osf.io/gpf3d) of biosample sources -- `sex`: Biological sex of subject, if applicable -- `ethnicity`: Self-reported ethnicity of subject, if applicable -- `age_at_enrollment`: Age of subject at enrollment time in primary project, if applicable - -### Container Entity Tables -The two types of container entities, `project` and `collection`, represent groups of core entities (`file`, `biosample`, or `subject`) that may be broadly related for funding, scientific, experimental, or other reasons. The specific grouping of entities, how granular the groupings are, and how many groups to generate, are all decided by individual DCCs. However, to have a complete C2M2 package, at least one root `project` must be defined that encompasses all entities generated by the DCC. - -Container entities can also be structured hierarchically and intersectionally, with smaller `project` and `collection` sub-groups belonging to larger ones. While entities can only belong to one `project` (see [Core Entity Tables](#core-entity-tables)), they may belong to multiple `collections`; this is due to the original concept of the C2M2 `project` being designed as a grouping of entities within one administrative, grant, or contractual realm, whereas a `collection` is a more generic dataset. It is expected that entities will be only be assigned to the `project` from which they were originally produced or funded. - -**Project:** A grouping of entities under one administrative, contractual, or funding-based scope -**Collection:** A more generalized grouping consisting of entities with some relevance, to be determined by individual DCCs - -### Association Tables -Association tables define the relationships between different types of entities. Each row in an association table stores two sets of foreign key identifiers, one set for each of the two entities that are connected. For instance, in the `file_describes_biosample` table, each row looks like the following: - -`| file_id_namespace | file_local_id | biosample_id_namespace | biosample_local_id |` - -where the first two fields correspond to an existing record in the `file` table and the second two fields correspond to an existing record in the `biosample` table. In total, there are four association tables following this format: - -- `file_describes_biosample` -- `file_describes_subject` -- `biosample_from_subject` -- `collection_defined_by_project` -- `file_describes_collection` - -Note that all of these association tables may be optionally filled out. In other words, entities do not need to be connected with other entities in order to generate a complete C2M2 submission. - -### Container Entity Tables -The two types of container entities, `project` and `collection`, represent groups of core entities (`file`, `biosample`, or `subject`) that may be broadly related for funding, scientific, experimental, or other reasons. The specific grouping of entities, how granular the groupings are, and how many groups to generate, are all decided by individual DCCs. However, to have a complete C2M2 package, at least one root `project` must be defined that encompasses all entities generated by the DCC. - -Container entities can also be structured hierarchically and intersectionally, with smaller `project` and `collection` sub-groups belonging to larger ones. While entities can only belong to one `project` (see [Core Entity Tables](#core-entity-tables)), they may belong to multiple `collections`; this is due to the original concept of the C2M2 `project` being designed as a grouping of entities within one administrative, grant, or contractual realm, whereas a `collection` is a more generic dataset. It is expected that entities will only be assigned to the `project` from which they were originally produced or funded. - -**Project:** A grouping of entities under one administrative, contractual, or funding-based scope -**Collection:** A more generalized grouping consisting of entities with some relevance, to be determined by individual DCCs - -### Association Tables -Association tables define the relationships between different types of entities. Each row in an association table stores two sets of foreign key identifiers, one set for each of the two entities that are connected. For instance, in the `file_describes_biosample` table, each row looks like the following: - -`| file_id_namespace | file_local_id | biosample_id_namespace | biosample_local_id |` - -where the first two fields correspond to an existing record in the `file` table and the second two fields correspond to an existing record in the `biosample` table. In total, there are four association tables following this format: - -- `file_describes_biosample` -- `file_describes_subject` -- `biosample_from_subject` -- `collection_defined_by_project` -- `file_describes_collection` - -Note that all of these association tables may be optionally filled out. In other words, entities do not need to be connected with other entities in order to generate a complete C2M2 submission. - -### Controlled Vocabularies -C2M2 metadata is harmonized with various standard biomedical ontologies. While including ontological metadata annotations is largely optional, it is highly recommended in order to facilitate increased interoperability across datapackages, which is one of the overall goals of the C2M2. - -Currently, the C2M2 is integrated with the following ontologies, each of which can be used to annotate specific table fields: -- Ontology for Biomedical Investigations (OBI) -- EMBRACE Data and Methods (EDAM) -- Uber-Anatomy Ontology (UBERON) -- NCBI Taxonomy -- Disease Ontology (DOID) -- Human Phenotype Ontology (HPO) -- PubChem Substance (SID) and Compound (CID) -- Ensembl -- UniProtKB - -These controlled vocabulary terms are correlated to different tables and table cells. Some terms can be related to core entities through one of the direct association tables: - -- `biosample_substance`: biosample --> PubChem SID -- `biosample_disease`: biosample --> DOID -- `biosample_gene`: biosample --> Ensembl ID -- `subject_substance`: subject --> PubChem SID -- `subject_disease`: subject --> DOID -- `subject_phenotype`: subject --> HPO ID -- `collection_gene`: collection --> Ensembl ID -- `collection_compound`: collection --> PubChem CID -- `collection_substance`: collection --> PubChem SID -- `collection_taxonomy`: collection --> NCBI Taxonomy ID -- `collection_anatomy`: collection --> UBERON ID -- `collection_protein`: collection --> UniProtKB ID - -Each row in the above tables will contain the namespace and foreign key for the core entity or collection, and then the identifier for the controlled vocabulary term. For instance, a row in `biosample_substance.tsv` will look like the following: - -`| biosample_id_namespace | biosample_local_id | substance |` - -where `substance` is a valid PubChem Substance ID (SID). - -Other controlled vocabulary terms have designated fields within specific entity tables: -- `file.tsv` - - `file.assay_type`: OBI ID - - `file.analysis_type`: OBI ID - - `file.file_format`: EDAM ID - - `file.compression_format`: EDAM ID - - `file.data_type`: EDAM ID -- `biosample.tsv` - - `biosample.sample_prep_method`: OBI ID - - `biosample.anatomy`: UBERON ID - -Taxonomic information on `subjects` is stored in the `subject_role_taxonomy` table as follows: - -`| subject_id_namespace | subject_local_id | role_id | taxonomy_id |` - -where `taxonomy_id` is an NCBI Taxonomy Database ID and `role_id` is one of the following options: -- `cfde_subject_role:0` -- single organism -- `cfde_subject_role:1` -- host -- `cfde_subject_role:2` -- symbiont -- `cfde_subject_role:3` -- pathogen -- `cfde_subject_role:4` -- microbiome taxon -- `cfde_subject_role:5` -- cell line ancestor -- `cfde_subject_role:6` -- synthetic - -Once all of the associated controlled vocabulary term fields and tables have been filled out, you will need to use the provided [C2M2 submission support script and ontology files](https://osf.io/bq6k9/) to automatically generate the standalone controlled-vocabulary term tables associated with your metadata, and complete your C2M2 metadata submission. - - -Return to [Documentation page](/info/documentation) \ No newline at end of file diff --git a/drc-portals/app/info/documentation/markdown/FAIRshake.md b/drc-portals/app/info/documentation/markdown/FAIRshake.md index dcc1375e..d60d6153 100644 --- a/drc-portals/app/info/documentation/markdown/FAIRshake.md +++ b/drc-portals/app/info/documentation/markdown/FAIRshake.md @@ -63,4 +63,4 @@ The steps below are adapted from the [FAIRshake User Guide V2](https://fairshake - **Saving** the assessment means the responses, comments, and URLs will not be public, but will be visible to the creators of the tool, project, and assessment. - **Deleting** the assessment will remove all data and responses from the assessment. -Return to [Documentation page](/info/documentation) +#### Return to [Documentation](./) diff --git a/drc-portals/app/info/documentation/markdown/KGAssertions.md b/drc-portals/app/info/documentation/markdown/KGAssertions.md index 4573d928..1feec9d3 100644 --- a/drc-portals/app/info/documentation/markdown/KGAssertions.md +++ b/drc-portals/app/info/documentation/markdown/KGAssertions.md @@ -49,4 +49,4 @@ The CFDE Data Distillery Partnership is building a **Data Distillery Knowledge G - [CFDE Data Distillery Project Github Repository](https://github.com/TaylorResearchLab/CFDE_DataDistillery) - [CFDE Data Distillery User Guide](https://github.com/TaylorResearchLab/CFDE_DataDistillery/tree/main/user_guide) -Return to [Documentation page](/info/documentation) \ No newline at end of file +#### Return to [Documentation](./) diff --git a/drc-portals/app/info/documentation/markdown/OpenAPI.md b/drc-portals/app/info/documentation/markdown/OpenAPI.md index ed5e2da6..189f8f12 100644 --- a/drc-portals/app/info/documentation/markdown/OpenAPI.md +++ b/drc-portals/app/info/documentation/markdown/OpenAPI.md @@ -384,4 +384,4 @@ As an example, the following tutorial will adapt two endpoints from the [FAIRsha While not required, it is highly recommended to use a dedicated OpenAPI or SmartAPI editor, such as Swagger, to generate API specifications. Such tools designated for writing OpenAPIs/SmartAPIs can help with debugging and auto-formatting, as well as with testing and publishing API documentation. -Return to [Documentation page](/info/documentation) +#### Return to [Documentation](./) diff --git a/drc-portals/app/info/documentation/markdown/PWBMetanodes.md b/drc-portals/app/info/documentation/markdown/PWBMetanodes.md index 5aae5aa8..0498ac5e 100644 --- a/drc-portals/app/info/documentation/markdown/PWBMetanodes.md +++ b/drc-portals/app/info/documentation/markdown/PWBMetanodes.md @@ -91,4 +91,4 @@ The PPWB is designed to allow for the independent and parallel development of in 6. Once the meta node is functional, submit a pull request to the remote main branch, and add any documentation as needed. -Return to [Documentation page](/info/documentation) \ No newline at end of file +#### Return to [Documentation](./) diff --git a/drc-portals/components/misc/ReactMarkdownRenderers.tsx b/drc-portals/components/misc/ReactMarkdownRenderers.tsx index 64803bfd..a44bd3d7 100644 --- a/drc-portals/components/misc/ReactMarkdownRenderers.tsx +++ b/drc-portals/components/misc/ReactMarkdownRenderers.tsx @@ -1,7 +1,23 @@ import React from "react"; +export async function fetchC2m2Markdown( doc: string ) { + const prefix = 'https://raw.githubusercontent.com/wiki/nih-cfde/published-documentation/' + return ( + fetch(prefix.concat(doc)) + .then((res) => res.text()) + .then((text) => text) + .catch((err) => "\n\n`Error fetching external content`") + ) +} + export function LinkRenderer(props: any) { - if (props.href.startsWith("#") || props.href.startsWith("/")) { + if (props.href.startsWith("/img")) { + return ( + + {props.children} + + ) + } else if (props.href.startsWith("#") || props.href.startsWith("./") ) { return ( {props.children} diff --git a/drc-portals/public/img/standards/Full_C2M2.png b/drc-portals/public/img/standards/Full_C2M2.png new file mode 100644 index 00000000..d277f0b3 Binary files /dev/null and b/drc-portals/public/img/standards/Full_C2M2.png differ