From 3d83d7afa34edd87b0feb9653f4958a6c5eea29a Mon Sep 17 00:00:00 2001 From: Milutin-S Date: Thu, 18 Apr 2024 11:47:45 +0200 Subject: [PATCH 1/2] Add rename for missing library. --- src/evaluate/loading.py | 64 ++++++++++++++++++++++++++++++++--------- 1 file changed, 51 insertions(+), 13 deletions(-) diff --git a/src/evaluate/loading.py b/src/evaluate/loading.py index 2a3437681..01c1a8317 100644 --- a/src/evaluate/loading.py +++ b/src/evaluate/loading.py @@ -54,7 +54,8 @@ def init_dynamic_modules( - name: str = config.MODULE_NAME_FOR_DYNAMIC_MODULES, hf_modules_cache: Optional[Union[Path, str]] = None + name: str = config.MODULE_NAME_FOR_DYNAMIC_MODULES, + hf_modules_cache: Optional[Union[Path, str]] = None, ): """ Create a module with name `name` in which you can add dynamic modules @@ -71,7 +72,9 @@ def init_dynamic_modules( return dynamic_modules_path -def import_main_class(module_path) -> Optional[Union[Type[DatasetBuilder], Type[EvaluationModule]]]: +def import_main_class( + module_path, +) -> Optional[Union[Type[DatasetBuilder], Type[EvaluationModule]]]: """Import a module at module_path and return its main class, a Metric by default""" module = importlib.import_module(module_path) main_cls_type = EvaluationModule @@ -176,7 +179,11 @@ def get_imports(file_path: str) -> Tuple[str, str, str, str]: # not be added as required dependencies continue - match = re.match(r"^import\s+(\.?)([^\s\.]+)[^#\r\n]*(?:#\s+From:\s+)?([^\r\n]*)", line, flags=re.MULTILINE) + match = re.match( + r"^import\s+(\.?)([^\s\.]+)[^#\r\n]*(?:#\s+From:\s+)?([^\r\n]*)", + line, + flags=re.MULTILINE, + ) if match is None: match = re.match( r"^from\s+(\.?)([^\s\.]+)(?:[^\s]*)\s+import\s+[^#\r\n]*(?:#\s+From:\s+)?([^\r\n]*)", @@ -210,7 +217,10 @@ def get_imports(file_path: str) -> Tuple[str, str, str, str]: def _download_additional_modules( - name: str, base_path: str, imports: Tuple[str, str, str, str], download_config: Optional[DownloadConfig] + name: str, + base_path: str, + imports: Tuple[str, str, str, str], + download_config: Optional[DownloadConfig], ) -> List[Tuple[str, str]]: """ Download additional module for a module .py at URL (or local path) /.py @@ -260,6 +270,7 @@ def _download_additional_modules( lib = importlib.import_module(library_import_name) # noqa F841 except ImportError: library_import_name = "scikit-learn" if library_import_name == "sklearn" else library_import_name + library_import_name = "absl-py" if library_import_name == "absl" else library_import_name needs_to_be_installed.add((library_import_name, library_import_path)) if needs_to_be_installed: raise ImportError( @@ -329,7 +340,10 @@ def _copy_script_and_other_resources_in_importable_dir( # Record metadata associating original dataset path with local unique folder meta_path = importable_local_file.split(".py")[0] + ".json" if not os.path.exists(meta_path): - meta = {"original file path": original_local_path, "local file path": importable_local_file} + meta = { + "original file path": original_local_path, + "local file path": importable_local_file, + } # the filename is *.py in our case, so better rename to filenam.json instead of filename.py.json with open(meta_path, "w", encoding="utf-8") as meta_file: json.dump(meta, meta_file) @@ -381,7 +395,13 @@ def _create_importable_file( ) logger.debug(f"Created importable dataset file at {importable_local_file}") module_path = ".".join( - [os.path.basename(dynamic_modules_path), module_namespace, name.replace("/", "--"), hash, name.split("/")[-1]] + [ + os.path.basename(dynamic_modules_path), + module_namespace, + name.replace("/", "--"), + hash, + name.split("/")[-1], + ] ) return module_path, hash @@ -550,7 +570,13 @@ def _get_modification_time(module_hash): ) # make the new module to be noticed by the import system module_path = ".".join( - [os.path.basename(dynamic_modules_path), self.module_type, self.name, hash, self.name.split("--")[-1]] + [ + os.path.basename(dynamic_modules_path), + self.module_type, + self.name, + hash, + self.name.split("--")[-1], + ] ) importlib.invalidate_caches() return ImportableModule(module_path, hash) @@ -614,13 +640,17 @@ def evaluation_module_factory( if path.endswith(filename): if os.path.isfile(path): return LocalEvaluationModuleFactory( - path, download_mode=download_mode, dynamic_modules_path=dynamic_modules_path + path, + download_mode=download_mode, + dynamic_modules_path=dynamic_modules_path, ).get_module() else: raise FileNotFoundError(f"Couldn't find a metric script at {relative_to_absolute_path(path)}") elif os.path.isfile(combined_path): return LocalEvaluationModuleFactory( - combined_path, download_mode=download_mode, dynamic_modules_path=dynamic_modules_path + combined_path, + download_mode=download_mode, + dynamic_modules_path=dynamic_modules_path, ).get_module() elif is_relative_path(path) and path.count("/") <= 1 and not force_local_path: try: @@ -658,13 +688,16 @@ def evaluation_module_factory( download_mode=download_mode, dynamic_modules_path=dynamic_modules_path, ).get_module() - except Exception as e1: # noqa: all the attempts failed, before raising the error we should check if the module is already cached. + except ( + Exception + ) as e1: # noqa: all the attempts failed, before raising the error we should check if the module is already cached. # if it's a canonical module we need to check if it's any of the types if path.count("/") == 0: for current_type in ["metric", "comparison", "measurement"]: try: return CachedEvaluationModuleFactory( - f"evaluate-{current_type}--{path}", dynamic_modules_path=dynamic_modules_path + f"evaluate-{current_type}--{path}", + dynamic_modules_path=dynamic_modules_path, ).get_module() except Exception as e2: # noqa: if it's not in the cache, then it doesn't exist. pass @@ -672,7 +705,8 @@ def evaluation_module_factory( elif path.count("/") == 1: try: return CachedEvaluationModuleFactory( - path.replace("/", "--"), dynamic_modules_path=dynamic_modules_path + path.replace("/", "--"), + dynamic_modules_path=dynamic_modules_path, ).get_module() except Exception as e2: # noqa: if it's not in the cache, then it doesn't exist. pass @@ -746,7 +780,11 @@ def load( """ download_mode = DownloadMode(download_mode or DownloadMode.REUSE_DATASET_IF_EXISTS) evaluation_module = evaluation_module_factory( - path, module_type=module_type, revision=revision, download_config=download_config, download_mode=download_mode + path, + module_type=module_type, + revision=revision, + download_config=download_config, + download_mode=download_mode, ) evaluation_cls = import_main_class(evaluation_module.module_path) evaluation_instance = evaluation_cls( From 2bbd42ae68cec3706ba2b18458929aafd85cb201 Mon Sep 17 00:00:00 2001 From: Milutin-S Date: Thu, 18 Apr 2024 11:48:08 +0200 Subject: [PATCH 2/2] Modifications from 'make style' command. --- metrics/bleu/tokenizer_13a.py | 2 +- metrics/google_bleu/tokenizer_13a.py | 2 +- metrics/indic_glue/indic_glue.py | 16 +++++++++------ metrics/roc_auc/roc_auc.py | 20 ++++++++++--------- metrics/squad_v2/compute_score.py | 1 + metrics/super_glue/record_evaluation.py | 1 - .../evaluator/audio_classification.py | 1 - .../evaluator/image_classification.py | 1 - tests/test_evaluator.py | 8 +++++--- 9 files changed, 29 insertions(+), 23 deletions(-) diff --git a/metrics/bleu/tokenizer_13a.py b/metrics/bleu/tokenizer_13a.py index c7a1b3dbd..d589c606c 100644 --- a/metrics/bleu/tokenizer_13a.py +++ b/metrics/bleu/tokenizer_13a.py @@ -61,7 +61,7 @@ def __call__(self, line): :param line: a segment to tokenize :return: the tokenized line """ - for (_re, repl) in self._re: + for _re, repl in self._re: line = _re.sub(repl, line) # no leading or trailing spaces, single space within words diff --git a/metrics/google_bleu/tokenizer_13a.py b/metrics/google_bleu/tokenizer_13a.py index c7a1b3dbd..d589c606c 100644 --- a/metrics/google_bleu/tokenizer_13a.py +++ b/metrics/google_bleu/tokenizer_13a.py @@ -61,7 +61,7 @@ def __call__(self, line): :param line: a segment to tokenize :return: the tokenized line """ - for (_re, repl) in self._re: + for _re, repl in self._re: line = _re.sub(repl, line) # no leading or trailing spaces, single space within words diff --git a/metrics/indic_glue/indic_glue.py b/metrics/indic_glue/indic_glue.py index 03afd1700..e12595866 100644 --- a/metrics/indic_glue/indic_glue.py +++ b/metrics/indic_glue/indic_glue.py @@ -132,12 +132,16 @@ def _info(self): inputs_description=_KWARGS_DESCRIPTION, features=datasets.Features( { - "predictions": datasets.Value("int64") - if self.config_name != "cvit-mkb-clsr" - else datasets.Sequence(datasets.Value("float32")), - "references": datasets.Value("int64") - if self.config_name != "cvit-mkb-clsr" - else datasets.Sequence(datasets.Value("float32")), + "predictions": ( + datasets.Value("int64") + if self.config_name != "cvit-mkb-clsr" + else datasets.Sequence(datasets.Value("float32")) + ), + "references": ( + datasets.Value("int64") + if self.config_name != "cvit-mkb-clsr" + else datasets.Sequence(datasets.Value("float32")) + ), } ), codebase_urls=[], diff --git a/metrics/roc_auc/roc_auc.py b/metrics/roc_auc/roc_auc.py index 604f93ab3..c6c8b0088 100644 --- a/metrics/roc_auc/roc_auc.py +++ b/metrics/roc_auc/roc_auc.py @@ -155,15 +155,17 @@ def _info(self): "references": datasets.Value("int32"), } if self.config_name == "multiclass" - else { - "references": datasets.Sequence(datasets.Value("int32")), - "prediction_scores": datasets.Sequence(datasets.Value("float")), - } - if self.config_name == "multilabel" - else { - "references": datasets.Value("int32"), - "prediction_scores": datasets.Value("float"), - } + else ( + { + "references": datasets.Sequence(datasets.Value("int32")), + "prediction_scores": datasets.Sequence(datasets.Value("float")), + } + if self.config_name == "multilabel" + else { + "references": datasets.Value("int32"), + "prediction_scores": datasets.Value("float"), + } + ) ), reference_urls=["https://scikit-learn.org/stable/modules/generated/sklearn.metrics.roc_auc_score.html"], ) diff --git a/metrics/squad_v2/compute_score.py b/metrics/squad_v2/compute_score.py index 3b512ae92..2f8f2e110 100644 --- a/metrics/squad_v2/compute_score.py +++ b/metrics/squad_v2/compute_score.py @@ -5,6 +5,7 @@ This file is expected to map question ID's to the model's predicted probability that a question is unanswerable. """ + import argparse import collections import json diff --git a/metrics/super_glue/record_evaluation.py b/metrics/super_glue/record_evaluation.py index 396d7ac8a..e8a933614 100644 --- a/metrics/super_glue/record_evaluation.py +++ b/metrics/super_glue/record_evaluation.py @@ -3,7 +3,6 @@ (Some functions are adopted from the SQuAD evaluation script.) """ - import argparse import json import re diff --git a/src/evaluate/evaluator/audio_classification.py b/src/evaluate/evaluator/audio_classification.py index 685fb9fd8..85c6809aa 100644 --- a/src/evaluate/evaluator/audio_classification.py +++ b/src/evaluate/evaluator/audio_classification.py @@ -119,7 +119,6 @@ def compute( label_column: str = "label", label_mapping: Optional[Dict[str, Number]] = None, ) -> Tuple[Dict[str, float], Any]: - """ input_column (`str`, defaults to `"file"`): The name of the column containing either the audio files or a raw waveform, represented as a numpy array, in the dataset specified by `data`. diff --git a/src/evaluate/evaluator/image_classification.py b/src/evaluate/evaluator/image_classification.py index 82831458b..d71d5b920 100644 --- a/src/evaluate/evaluator/image_classification.py +++ b/src/evaluate/evaluator/image_classification.py @@ -87,7 +87,6 @@ def compute( label_column: str = "label", label_mapping: Optional[Dict[str, Number]] = None, ) -> Tuple[Dict[str, float], Any]: - """ input_column (`str`, defaults to `"image"`): The name of the column containing the images as PIL ImageFile in the dataset specified by `data`. diff --git a/tests/test_evaluator.py b/tests/test_evaluator.py index b0e4d5a6f..bbb215e66 100644 --- a/tests/test_evaluator.py +++ b/tests/test_evaluator.py @@ -95,9 +95,11 @@ def __init__(self, v2: bool): def __call__(self, question, context, **kwargs): if self.v2: return [ - {"score": 0.95, "start": 31, "end": 39, "answer": "Felix"} - if i % 2 == 0 - else {"score": 0.95, "start": 0, "end": 0, "answer": ""} + ( + {"score": 0.95, "start": 31, "end": 39, "answer": "Felix"} + if i % 2 == 0 + else {"score": 0.95, "start": 0, "end": 0, "answer": ""} + ) for i in range(len(question)) ] else: