Modifications for bigcode-the-stack dataset (#522)

mlcommons · Feb 15, 2024 · 65b67d1 · 65b67d1
1 parent cd64e12
commit 65b67d1
Show file tree

Hide file tree

Showing 5 changed files with 17 additions and 8 deletions.
diff --git a/.devcontainer/postCreateCommand.sh b/.devcontainer/postCreateCommand.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 
-sudo apt-get update && sudo apt-get install -y libgraphviz-dev git-lfs
+sudo apt-get update && sudo apt-get install -y graphviz libgraphviz-dev git-lfs
 
 # Install dependencies except mlcroissant itself
-cd ../python/mlcroissant
-pip install -e ../python/mlcroissant/.[dev]
+cd /workspaces/croissant/python/mlcroissant
+pip install -e .[dev]
diff --git a/datasets/1.0/bigcode-the-stack/metadata.json b/datasets/1.0/bigcode-the-stack/metadata.json
@@ -92,10 +92,7 @@
           "source": {
             "fileSet": "parquet-files",
             "extract": {
-              "fileProperty": "fullpath"
-            },
-            "transform": {
-              "regex": "^data\\/(\\d\\w\\+\\-)+\\/train-\\d\\d\\d\\d\\d-of-\\d\\d\\d\\d\\d\\.parquet$"
+              "column": "lang"
             }
           }
         },