Revert breaking metadata changes

mlcommons · Feb 26, 2024 · 50ac1e8 · 50ac1e8
1 parent 5468eea
commit 50ac1e8
Show file tree

Hide file tree

Showing 14 changed files with 38 additions and 122 deletions.
diff --git a/datasets/1.0/bigcode-the-stack/metadata.json b/datasets/1.0/bigcode-the-stack/metadata.json
@@ -47,18 +47,25 @@
   "name": "bigcode-the-stack",
   "conformsTo": "http://mlcommons.org/croissant/1.0",
   "description": "The Stack contains over 6TB of permissively-licensed source code files covering 358 programming languages. The dataset was created as part of the BigCode Project, an open scientific collaboration working on the responsible development of Large Language Models for Code (Code LLMs). The Stack serves as a pre-training dataset for Code LLMs, i.e., code-generating AI systems which enable the synthesis of programs from natural language descriptions as well as other from code snippets.",
+  "citeAs": "@article{Kocetkov2022TheStack, title={The Stack: 3 TB of permissively licensed source code}, author={Kocetkov, Denis and Li, Raymond and Ben Allal, Loubna and Li, Jia and Mou,Chenghao and Mu\u00f1oz Ferrandis, Carlos and Jernite, Yacine and Mitchell, Margaret and Hughes, Sean and Wolf, Thomas and Bahdanau, Dzmitry and von Werra, Leandro and de Vries, Harm}, journal={Preprint}, year={2022} }",
   "creator": [
     {
-      "@type": "Organization",
-      "name": "Harm de Vries"
+      "@type": "Person",
+      "name": "Harm de Vries",
+      "email": "[email protected]"
     },
     {
-      "@type": "Organization",
-      "name": "Leandro von Werra"
+      "@type": "Person",
+      "name": "Leandro von Werra",
+      "email": "[email protected]"
     }
   ],
-  "citeAs": "@article{Kocetkov2022TheStack, title={The Stack: 3 TB of permissively licensed source code}, author={Kocetkov, Denis and Li, Raymond and Ben Allal, Loubna and Li, Jia and Mou,Chenghao and Mu\u00f1oz Ferrandis, Carlos and Jernite, Yacine and Mitchell, Margaret and Hughes, Sean and Wolf, Thomas and Bahdanau, Dzmitry and von Werra, Leandro and de Vries, Harm}, journal={Preprint}, year={2022} }",
+  "keywords": [
+    "crowdsourced",
+    "expert-generated"
+  ],
   "license": "other",
+  "sameAs": "https://www.bigcode-project.org/docs/about/the-stack/",
   "sdLicense": "https://www.apache.org/licenses/LICENSE-2.0",
   "url": "https://huggingface.co/datasets/bigcode/the-stack",
   "distribution": [

diff --git a/...mlcroissant/mlcroissant/_src/tests/graphs/0.8/distribution_bad_contained_in/metadata.json b/...mlcroissant/mlcroissant/_src/tests/graphs/0.8/distribution_bad_contained_in/metadata.json
@@ -2,10 +2,8 @@
   "@context": {
     "@language": "en",
     "@vocab": "https://schema.org/",
-    "citeAs": "cr:citeAs",
     "column": "ml:column",
     "conformsTo": "dct:conformsTo",
-    "cr": "http://mlcommons.org/croissant/",
     "data": {
       "@id": "ml:data",
       "@type": "@json"
@@ -20,15 +18,11 @@
     "extract": "ml:extract",
     "field": "ml:field",
     "fileProperty": "ml:fileProperty",
-    "fileObject": "cr:fileObject",
-    "fileSet": "cr:fileSet",
     "format": "ml:format",
     "includes": "ml:includes",
     "isEnumeration": "ml:isEnumeration",
-    "isLiveDataset": "cr:isLiveDataset",
     "jsonPath": "ml:jsonPath",
-    "key": "cr:key",
-    "md5": "cr:md5",
+    "ml": "http://mlcommons.org/schema/",
     "parentField": "ml:parentField",
     "path": "ml:path",
     "personalSensitiveInformation": "ml:personalSensitiveInformation",
@@ -41,8 +35,7 @@
     "separator": "ml:separator",
     "source": "ml:source",
     "subField": "ml:subField",
-    "transform": "ml:transform",
-    "ml": "http://mlcommons.org/schema/"
+    "transform": "ml:transform"
   },
   "@type": "sc:Dataset",
   "name": "mydataset",

diff --git a/python/mlcroissant/mlcroissant/_src/tests/graphs/0.8/distribution_bad_type/metadata.json b/python/mlcroissant/mlcroissant/_src/tests/graphs/0.8/distribution_bad_type/metadata.json
@@ -2,10 +2,8 @@
   "@context": {
     "@language": "en",
     "@vocab": "https://schema.org/",
-    "citeAs": "cr:citeAs",
     "column": "ml:column",
     "conformsTo": "dct:conformsTo",
-    "cr": "http://mlcommons.org/croissant/",
     "data": {
       "@id": "ml:data",
       "@type": "@json"
@@ -20,15 +18,11 @@
     "extract": "ml:extract",
     "field": "ml:field",
     "fileProperty": "ml:fileProperty",
-    "fileObject": "cr:fileObject",
-    "fileSet": "cr:fileSet",
     "format": "ml:format",
     "includes": "ml:includes",
     "isEnumeration": "ml:isEnumeration",
-    "isLiveDataset": "cr:isLiveDataset",
     "jsonPath": "ml:jsonPath",
-    "key": "cr:key",
-    "md5": "cr:md5",
+    "ml": "http://mlcommons.org/schema/",
     "parentField": "ml:parentField",
     "path": "ml:path",
     "personalSensitiveInformation": "ml:personalSensitiveInformation",
@@ -41,8 +35,7 @@
     "separator": "ml:separator",
     "source": "ml:source",
     "subField": "ml:subField",
-    "transform": "ml:transform",
-    "ml": "http://mlcommons.org/schema/"
+    "transform": "ml:transform"
   },
   "@type": "sc:Dataset",
   "name": "mydataset",

diff --git a/python/mlcroissant/mlcroissant/_src/tests/graphs/0.8/distribution_missing_name/metadata.json b/python/mlcroissant/mlcroissant/_src/tests/graphs/0.8/distribution_missing_name/metadata.json
@@ -2,10 +2,8 @@
   "@context": {
     "@language": "en",
     "@vocab": "https://schema.org/",
-    "citeAs": "cr:citeAs",
     "column": "ml:column",
     "conformsTo": "dct:conformsTo",
-    "cr": "http://mlcommons.org/croissant/",
     "data": {
       "@id": "ml:data",
       "@type": "@json"
@@ -20,15 +18,11 @@
     "extract": "ml:extract",
     "field": "ml:field",
     "fileProperty": "ml:fileProperty",
-    "fileObject": "cr:fileObject",
-    "fileSet": "cr:fileSet",
     "format": "ml:format",
     "includes": "ml:includes",
     "isEnumeration": "ml:isEnumeration",
-    "isLiveDataset": "cr:isLiveDataset",
     "jsonPath": "ml:jsonPath",
-    "key": "cr:key",
-    "md5": "cr:md5",
+    "ml": "http://mlcommons.org/schema/",
     "parentField": "ml:parentField",
     "path": "ml:path",
     "personalSensitiveInformation": "ml:personalSensitiveInformation",
@@ -41,8 +35,7 @@
     "separator": "ml:separator",
     "source": "ml:source",
     "subField": "ml:subField",
-    "transform": "ml:transform",
-    "ml": "http://mlcommons.org/schema/"
+    "transform": "ml:transform"
   },
   "@type": "sc:Dataset",
   "name": "mydataset",

diff --git a/...mlcroissant/_src/tests/graphs/0.8/distribution_missing_property_content_url/metadata.json b/...mlcroissant/_src/tests/graphs/0.8/distribution_missing_property_content_url/metadata.json
@@ -2,10 +2,8 @@
   "@context": {
     "@language": "en",
     "@vocab": "https://schema.org/",
-    "citeAs": "cr:citeAs",
     "column": "ml:column",
     "conformsTo": "dct:conformsTo",
-    "cr": "http://mlcommons.org/croissant/",
     "data": {
       "@id": "ml:data",
       "@type": "@json"
@@ -20,15 +18,11 @@
     "extract": "ml:extract",
     "field": "ml:field",
     "fileProperty": "ml:fileProperty",
-    "fileObject": "cr:fileObject",
-    "fileSet": "cr:fileSet",
     "format": "ml:format",
     "includes": "ml:includes",
     "isEnumeration": "ml:isEnumeration",
-    "isLiveDataset": "cr:isLiveDataset",
     "jsonPath": "ml:jsonPath",
-    "key": "cr:key",
-    "md5": "cr:md5",
+    "ml": "http://mlcommons.org/schema/",
     "parentField": "ml:parentField",
     "path": "ml:path",
     "personalSensitiveInformation": "ml:personalSensitiveInformation",
@@ -41,8 +35,7 @@
     "separator": "ml:separator",
     "source": "ml:source",
     "subField": "ml:subField",
-    "transform": "ml:transform",
-    "ml": "http://mlcommons.org/schema/"
+    "transform": "ml:transform"
   },
   "@type": "sc:Dataset",
   "name": "mydataset",

diff --git a/python/mlcroissant/mlcroissant/_src/tests/graphs/0.8/metadata_bad_type/metadata.json b/python/mlcroissant/mlcroissant/_src/tests/graphs/0.8/metadata_bad_type/metadata.json
@@ -2,10 +2,8 @@
   "@context": {
     "@language": "en",
     "@vocab": "https://schema.org/",
-    "citeAs": "cr:citeAs",
     "column": "ml:column",
     "conformsTo": "dct:conformsTo",
-    "cr": "http://mlcommons.org/croissant/",
     "data": {
       "@id": "ml:data",
       "@type": "@json"
@@ -20,15 +18,11 @@
     "extract": "ml:extract",
     "field": "ml:field",
     "fileProperty": "ml:fileProperty",
-    "fileObject": "cr:fileObject",
-    "fileSet": "cr:fileSet",
     "format": "ml:format",
     "includes": "ml:includes",
     "isEnumeration": "ml:isEnumeration",
-    "isLiveDataset": "cr:isLiveDataset",
     "jsonPath": "ml:jsonPath",
-    "key": "cr:key",
-    "md5": "cr:md5",
+    "ml": "http://mlcommons.org/schema/",
     "parentField": "ml:parentField",
     "path": "ml:path",
     "personalSensitiveInformation": "ml:personalSensitiveInformation",
@@ -41,8 +35,7 @@
     "separator": "ml:separator",
     "source": "ml:source",
     "subField": "ml:subField",
-    "transform": "ml:transform",
-    "ml": "http://mlcommons.org/schema/"
+    "transform": "ml:transform"
   },
   "@type": "sc:WRONG_TYPE",
   "name": "mydataset",

diff --git a/...lcroissant/mlcroissant/_src/tests/graphs/0.8/metadata_missing_property_name/metadata.json b/...lcroissant/mlcroissant/_src/tests/graphs/0.8/metadata_missing_property_name/metadata.json
@@ -2,10 +2,8 @@
   "@context": {
     "@language": "en",
     "@vocab": "https://schema.org/",
-    "citeAs": "cr:citeAs",
     "column": "ml:column",
     "conformsTo": "dct:conformsTo",
-    "cr": "http://mlcommons.org/croissant/",
     "data": {
       "@id": "ml:data",
       "@type": "@json"
@@ -20,15 +18,11 @@
     "extract": "ml:extract",
     "field": "ml:field",
     "fileProperty": "ml:fileProperty",
-    "fileObject": "cr:fileObject",
-    "fileSet": "cr:fileSet",
     "format": "ml:format",
     "includes": "ml:includes",
     "isEnumeration": "ml:isEnumeration",
-    "isLiveDataset": "cr:isLiveDataset",
     "jsonPath": "ml:jsonPath",
-    "key": "cr:key",
-    "md5": "cr:md5",
+    "ml": "http://mlcommons.org/schema/",
     "parentField": "ml:parentField",
     "path": "ml:path",
     "personalSensitiveInformation": "ml:personalSensitiveInformation",
@@ -41,8 +35,7 @@
     "separator": "ml:separator",
     "source": "ml:source",
     "subField": "ml:subField",
-    "transform": "ml:transform",
-    "ml": "http://mlcommons.org/schema/"
+    "transform": "ml:transform"
   },
   "@type": "sc:Dataset",
   "description": "This is a description.",

diff --git a/python/mlcroissant/mlcroissant/_src/tests/graphs/0.8/mlfield_bad_source/metadata.json b/python/mlcroissant/mlcroissant/_src/tests/graphs/0.8/mlfield_bad_source/metadata.json
@@ -2,10 +2,8 @@
   "@context": {
     "@language": "en",
     "@vocab": "https://schema.org/",
-    "citeAs": "cr:citeAs",
     "column": "ml:column",
     "conformsTo": "dct:conformsTo",
-    "cr": "http://mlcommons.org/croissant/",
     "data": {
       "@id": "ml:data",
       "@type": "@json"
@@ -20,15 +18,11 @@
     "extract": "ml:extract",
     "field": "ml:field",
     "fileProperty": "ml:fileProperty",
-    "fileObject": "cr:fileObject",
-    "fileSet": "cr:fileSet",
     "format": "ml:format",
     "includes": "ml:includes",
     "isEnumeration": "ml:isEnumeration",
-    "isLiveDataset": "cr:isLiveDataset",
     "jsonPath": "ml:jsonPath",
-    "key": "cr:key",
-    "md5": "cr:md5",
+    "ml": "http://mlcommons.org/schema/",
     "parentField": "ml:parentField",
     "path": "ml:path",
     "personalSensitiveInformation": "ml:personalSensitiveInformation",
@@ -41,8 +35,7 @@
     "separator": "ml:separator",
     "source": "ml:source",
     "subField": "ml:subField",
-    "transform": "ml:transform",
-    "ml": "http://mlcommons.org/schema/"
+    "transform": "ml:transform"
   },
   "@type": "sc:Dataset",
   "name": "mydataset",

diff --git a/python/mlcroissant/mlcroissant/_src/tests/graphs/0.8/mlfield_bad_type/metadata.json b/python/mlcroissant/mlcroissant/_src/tests/graphs/0.8/mlfield_bad_type/metadata.json
@@ -2,10 +2,8 @@
   "@context": {
     "@language": "en",
     "@vocab": "https://schema.org/",
-    "citeAs": "cr:citeAs",
     "column": "ml:column",
     "conformsTo": "dct:conformsTo",
-    "cr": "http://mlcommons.org/croissant/",
     "data": {
       "@id": "ml:data",
       "@type": "@json"
@@ -20,15 +18,11 @@
     "extract": "ml:extract",
     "field": "ml:field",
     "fileProperty": "ml:fileProperty",
-    "fileObject": "cr:fileObject",
-    "fileSet": "cr:fileSet",
     "format": "ml:format",
     "includes": "ml:includes",
     "isEnumeration": "ml:isEnumeration",
-    "isLiveDataset": "cr:isLiveDataset",
     "jsonPath": "ml:jsonPath",
-    "key": "cr:key",
-    "md5": "cr:md5",
+    "ml": "http://mlcommons.org/schema/",
     "parentField": "ml:parentField",
     "path": "ml:path",
     "personalSensitiveInformation": "ml:personalSensitiveInformation",
@@ -41,8 +35,7 @@
     "separator": "ml:separator",
     "source": "ml:source",
     "subField": "ml:subField",
-    "transform": "ml:transform",
-    "ml": "http://mlcommons.org/schema/"
+    "transform": "ml:transform"
   },
   "@type": "sc:Dataset",
   "name": "mydataset",

diff --git a/...mlcroissant/mlcroissant/_src/tests/graphs/0.8/mlfield_missing_property_name/metadata.json b/...mlcroissant/mlcroissant/_src/tests/graphs/0.8/mlfield_missing_property_name/metadata.json
@@ -2,10 +2,8 @@
   "@context": {
     "@language": "en",
     "@vocab": "https://schema.org/",
-    "citeAs": "cr:citeAs",
     "column": "ml:column",
     "conformsTo": "dct:conformsTo",
-    "cr": "http://mlcommons.org/croissant/",
     "data": {
       "@id": "ml:data",
       "@type": "@json"
@@ -20,15 +18,11 @@
     "extract": "ml:extract",
     "field": "ml:field",
     "fileProperty": "ml:fileProperty",
-    "fileObject": "cr:fileObject",
-    "fileSet": "cr:fileSet",
     "format": "ml:format",
     "includes": "ml:includes",
     "isEnumeration": "ml:isEnumeration",
-    "isLiveDataset": "cr:isLiveDataset",
     "jsonPath": "ml:jsonPath",
-    "key": "cr:key",
-    "md5": "cr:md5",
+    "ml": "http://mlcommons.org/schema/",
     "parentField": "ml:parentField",
     "path": "ml:path",
     "personalSensitiveInformation": "ml:personalSensitiveInformation",
@@ -41,8 +35,7 @@
     "separator": "ml:separator",
     "source": "ml:source",
     "subField": "ml:subField",
-    "transform": "ml:transform",
-    "ml": "http://mlcommons.org/schema/"
+    "transform": "ml:transform"
   },
   "@type": "sc:Dataset",
   "name": "mydataset",