integration data pr

tum-ai · Dec 22, 2024 · 4de41af · 4de41af
1 parent 25873df
commit 4de41af
Show file tree

Hide file tree

Showing 9 changed files with 9 additions and 40,008 deletions.
diff --git a/config/dataset_args/expression.yaml b/config/dataset_args/expression.yaml
diff --git a/config/run_specific_config/debug_config.yaml b/config/run_specific_config/debug_config.yaml
@@ -1,5 +1,5 @@
 dataset_args:
-  dataset_name: expression_dataset
+  dataset_name: arithmetic
 
 training_args:
   trial: debug

diff --git a/data/expression-dataset/split_data.py → data/arithmetics_dataset/split_data.py b/data/expression-dataset/split_data.py → data/arithmetics_dataset/split_data.py
diff --git a/data/expression-dataset/test.jsonl b/data/expression-dataset/test.jsonl
diff --git a/data/expression-dataset/test_no_negative_data.jsonl b/data/expression-dataset/test_no_negative_data.jsonl
diff --git a/data/expression-dataset/train.jsonl b/data/expression-dataset/train.jsonl
diff --git a/data/expression-dataset/val.jsonl b/data/expression-dataset/val.jsonl
diff --git a/src/ntl/args.py b/src/ntl/args.py
@@ -154,6 +154,6 @@ class DatasetArguments:
     dataset_name: str = field(
         default="mathematics_dataset",
         metadata={
-            "help": "Name of the dataset. Allowed: mathematics_dataset, gsm8k, multiplication, expression"
+            "help": "Name of the dataset. Allowed: mathematics_dataset, gsm8k, multiplication, arithmetics_dataset"
         },
     )
diff --git a/src/ntl/run_language_modeling.py b/src/ntl/run_language_modeling.py
@@ -337,15 +337,17 @@ def run_language_modeling(model_args: ModelArguments, training_args: TrainingArg
         train_dataset = load_json_dataset(train_data_path)
         eval_dataset = load_json_dataset(eval_data_path)
         test_dataset = load_json_dataset(test_data_path)
-    elif dataset_args.dataset_name == "expression_dataset":
-        train_data_path = "data/expression-dataset/train.jsonl"
-        eval_data_path = "data/expression-dataset/val.jsonl"
-        test_data_path = "data/expression-dataset/test.jsonl"
+    elif dataset_args.dataset_name == "arithmetic":
+        train_data_path = "data/arithmetics_dataset/data/train.jsonl"
+        eval_data_path = "data/arithmetics_dataset/data/val.jsonl"
+        test_data_path = "data/arithmetics_dataset/data/test.jsonl"
         train_dataset = load_json_dataset(train_data_path)
         eval_dataset = load_json_dataset(eval_data_path)
         test_dataset = load_json_dataset(test_data_path)
     else:
-        raise ValueError(f"Unknown dataset: {dataset_args.dataset_name}. Allowed: gsm8k, mathematics_dataset, multiplication")
+        raise ValueError(
+            f"Unknown dataset: {dataset_args.dataset_name}. Allowed: gsm8k, mathematics_dataset, multiplication, arithmetic"
+        )
 
     num_params = sum(p.numel() for p in model.parameters() if p.requires_grad)
     logger.info(f"Number of parameters {num_params} of type {type(model)}")