From 7ce00aed610722658f7ab56d239b1c2624e59bf0 Mon Sep 17 00:00:00 2001 From: dannyjameswilliams Date: Thu, 26 Sep 2024 10:51:53 +0100 Subject: [PATCH 01/30] initial experiment setup --- chunked_pooling/mteb_chunked_eval.py | 89 ++++++++++- soft_token_boundary.py | 223 +++++++++++++++++++++++++++ 2 files changed, 306 insertions(+), 6 deletions(-) create mode 100644 soft_token_boundary.py diff --git a/chunked_pooling/mteb_chunked_eval.py b/chunked_pooling/mteb_chunked_eval.py index b119deb..f533354 100644 --- a/chunked_pooling/mteb_chunked_eval.py +++ b/chunked_pooling/mteb_chunked_eval.py @@ -27,6 +27,9 @@ def __init__( model_has_instructions: bool = False, embedding_model_name: Optional[str] = None, # for semantic chunking truncate_max_length: Optional[int] = 8192, + soft_boundary_embed_size: Optional[int] = 0, + soft_boundary_overlap_size: Optional[int] = 512, + hard_boundary_embed_size: Optional[int] = 0, **kwargs, ): super().__init__(**kwargs) @@ -51,6 +54,13 @@ def __init__( } self.truncate_max_length = truncate_max_length + if soft_boundary_embed_size > 0 and hard_boundary_embed_size > 0: + raise ValueError('Cannot use both soft and hard boundaries') + + self.soft_boundary_embed_size = soft_boundary_embed_size + self.soft_boundary_overlap_size = soft_boundary_overlap_size + self.hard_boundary_embed_size = hard_boundary_embed_size + def load_data(self, **kwargs): self.retrieval_task.load_data(**kwargs) self.corpus = self.retrieval_task.corpus @@ -111,9 +121,65 @@ def _truncate_documents(self, corpus): max_length=self.truncate_max_length, ) last_token_span = tokens.offset_mapping[-2] + if len(self.tokenizer(v["text"]).tokens()) > self.truncate_max_length: + print(f"Document {k} will be truncated to {self.truncate_max_length} tokens") v['text'] = v['text'][: last_token_span[1]] return corpus + def _embed_with_soft_boundary(self, model, model_inputs): + + tokens = model_inputs.tokens() + + if len(tokens) > self.soft_boundary_embed_size: + indices = [] + for i in range(0, len(tokens), self.soft_boundary_embed_size - self.soft_boundary_overlap_size): + start = i + end = min(i + self.soft_boundary_embed_size, len(tokens)) + indices.append((start, end)) + else: + indices = [(0, len(tokens))] + + outputs = [] + for start, end in indices: + + batch_inputs = {k: v[:, start:end] for k, v in model_inputs.items()} + + with torch.no_grad(): + model_output = model(**batch_inputs) + + if start > 0: + outputs.append(model_output[0][:, self.soft_boundary_overlap_size:]) + else: + outputs.append(model_output[0]) + + return torch.cat(outputs, dim=1).to(model.device) + + def _embed_with_hard_boundary(self, model, model_inputs): + + tokens = model_inputs.tokens() + + if len(tokens) > self.hard_boundary_embed_size: + indices = [] + for i in range(0, len(tokens), self.hard_boundary_embed_size): + start = i + end = min(i + self.hard_boundary_embed_size, len(tokens)) + indices.append((start, end)) + else: + indices = [(0, len(tokens))] + + outputs = [] + for start, end in indices: + + batch_inputs = {k: v[:, start:end] for k, v in model_inputs.items()} + + with torch.no_grad(): + model_output = model(**batch_inputs) + + outputs.append(model_output[0]) + + return torch.cat(outputs, dim=1).to(model.device) + + def _evaluate_monolingual( self, model, @@ -181,17 +247,28 @@ def _evaluate_monolingual( text_inputs, return_tensors='pt', padding=True, - truncation=True, - max_length=8192, + truncation=False ) if model.device.type == 'cuda': model_inputs = { k: v.to(model.device) for k, v in model_inputs.items() } - model_outputs = model(**model_inputs) - output_embs = chunked_pooling( - model_outputs, annotations, max_length=8192 - ) + + if self.soft_boundary_embed_size > 0: + model_outputs = self._embed_with_soft_boundary(model, model_inputs) + output_embs = chunked_pooling( + [model_outputs], annotations, max_length=None + ) + elif self.hard_boundary_embed_size > 0: + model_outputs = self._embed_with_hard_boundary(model, model_inputs) + output_embs = chunked_pooling( + [model_outputs], annotations, max_length=None + ) + else: + model_outputs = model(**model_inputs) + output_embs = chunked_pooling( + model_outputs, annotations, max_length=self.truncate_max_length + ) corpus_embs.extend(output_embs) max_chunks = max([len(x) for x in corpus_embs]) diff --git a/soft_token_boundary.py b/soft_token_boundary.py new file mode 100644 index 0000000..e77caa1 --- /dev/null +++ b/soft_token_boundary.py @@ -0,0 +1,223 @@ +# soft_token_boundary.py +# +# long context embedding models (like jina-embeddings-v2-base-en) have a limit of 8192 tokens +# for longer context documents, such as those in LongEmbed benchmarks, how do we embed them? +# +# method 1: truncate the document at the token boundary +# method 2: embed twice, before and after the truncation point as many times as needed +# method 3: same as method 2 but with overlap +# +# obviously method 1 is not great. any information after the 8192nd token is discarded. +# method 3 should perform the best, but how much better is it? +# +# and do we need to look into reducing the size of context window? +# +# let's use the WikimQA dataset to test these ideas, just method 2 and 3. +# +# let's also try to use the mteb benchmark to evaluate the results. + +from chunked_pooling.wrappers import load_model +from transformers import AutoModel, AutoTokenizer, pipeline +# from experiments.lib import ContextualRetrievalEmbedder, LateChunkingEmbedder + + +import click +import torch.cuda +from mteb import MTEB +from transformers import AutoModel, AutoTokenizer + +from chunked_pooling.chunked_eval_tasks import LEMBWikimQARetrievalChunked, LEMBNeedleRetrievalChunked +from chunked_pooling.wrappers import load_model + +DEFAULT_CHUNKING_STRATEGY = 'fixed' +DEFAULT_CHUNK_SIZE = 256 +DEFAULT_N_SENTENCES = 5 +BATCH_SIZE = 1 + + +@click.command() +@click.option( + '--model-name', + default='jinaai/jina-embeddings-v2-small-en', + help='The name of the model to use.', +) +@click.option( + '--strategy', + default=DEFAULT_CHUNKING_STRATEGY, + help='The chunking strategy to be applied.', +) +@click.option( + '--task-name', default='LEMBWikimQARetrievalChunked', help='The evaluation task to perform.' +) +@click.option( + '--eval-split', default='test', help='The name of the evaluation split in the task.' +) +@click.option( + '--chunking-model', + default=None, + required=False, + help='The name of the model used for semantic chunking.', +) +@click.option( + '--truncate-max-length', + default=8192, + type=int, + help='Maximum number of tokens; By default, no truncation is done.', +) +@click.option( + '--chunk-size', + default=DEFAULT_CHUNK_SIZE, + type=int, + help='Number of tokens per chunk for fixed strategy.', +) +@click.option( + '--n-sentences', + default=DEFAULT_N_SENTENCES, + type=int, + help='Number of sentences per chunk for sentence strategy.', +) +@click.option( + '--soft_boundary_embed_size', + default=8192, + type=int, + help='Token length of the embeddings that come before/after soft boundaries (i.e. overlapping embeddings). Above zero, soft boundaries are used.', +) +@click.option( + '--soft_boundary_overlap_size', + default=512, + type=int, + help='Overlap between the embeddings before and after the soft boundary, in tokens.', +) +@click.option( + '--hard_boundary_embed_size', + default=8192, + type=int, + help='Token length of the embeddings that come before/after hard boundaries. Above zero, hard boundaries are used.', +) + +def main( + model_name, + strategy, + task_name, + eval_split, + chunking_model, + truncate_max_length, + chunk_size, + n_sentences, + soft_boundary_embed_size, + soft_boundary_overlap_size, + hard_boundary_embed_size, +): + try: + task_cls = globals()[task_name] + except: + raise ValueError(f'Unknown task name: {task_name}') + + model, has_instructions = load_model(model_name) + + tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True) + + chunking_args = { + 'chunk_size': chunk_size, + 'n_sentences': n_sentences, + 'chunking_strategy': strategy, + 'model_has_instructions': has_instructions, + 'embedding_model_name': chunking_model if chunking_model else model_name, + } + + if torch.cuda.is_available(): + model = model.cuda() + + model.eval() + + # Evaluate with soft boundary + tasks = [ + task_cls( + chunked_pooling_enabled=True, + tokenizer=tokenizer, + prune_size=None, + truncate_max_length=0, + soft_boundary_embed_size=soft_boundary_embed_size, + soft_boundary_overlap_size=soft_boundary_overlap_size, + hard_boundary_embed_size=0, + **chunking_args, + ) + ] + + evaluation = MTEB( + tasks=tasks, + chunked_pooling_enabled=True, + tokenizer=tokenizer, + prune_size=None, + **chunking_args, + ) + evaluation.run( + model, + output_folder=f'results-soft-boundary/embed_size_{soft_boundary_embed_size}', + eval_splits=[eval_split], + overwrite_results=True, + batch_size=BATCH_SIZE, + encode_kwargs={'batch_size': BATCH_SIZE}, + ) + + + # Evaluate with hard boundary + tasks = [ + task_cls( + chunked_pooling_enabled=True, + tokenizer=tokenizer, + prune_size=None, + truncate_max_length=0, + soft_boundary_embed_size=0, + hard_boundary_embed_size=hard_boundary_embed_size, + **chunking_args, + ) + ] + + evaluation = MTEB( + tasks=tasks, + chunked_pooling_enabled=True, + tokenizer=tokenizer, + prune_size=None, + **chunking_args, + ) + evaluation.run( + model, + output_folder=f'results-hard-boundary/embed_size_{hard_boundary_embed_size}', + eval_splits=[eval_split], + overwrite_results=True, + batch_size=BATCH_SIZE, + encode_kwargs={'batch_size': BATCH_SIZE}, + ) + + # Evaluate with no boundary (truncation) + tasks = [ + task_cls( + chunked_pooling_enabled=True, + tokenizer=tokenizer, + prune_size=None, + truncate_max_length=truncate_max_length, + soft_boundary_embed_size=0, + hard_boundary_embed_size=0, + **chunking_args, + ) + ] + + evaluation = MTEB( + tasks=tasks, + chunked_pooling_enabled=True, + tokenizer=tokenizer, + prune_size=None, + **chunking_args, + ) + evaluation.run( + model, + output_folder=f'results-truncation/embed_size_{truncate_max_length}', + eval_splits=[eval_split], + overwrite_results=True, + batch_size=BATCH_SIZE, + encode_kwargs={'batch_size': BATCH_SIZE}, + ) + +if __name__ == '__main__': + main() From ffed829aee37a2e13aeb1fa5539472a3a2112651 Mon Sep 17 00:00:00 2001 From: dannyjameswilliams Date: Thu, 26 Sep 2024 11:42:49 +0100 Subject: [PATCH 02/30] changed len tokens --- chunked_pooling/mteb_chunked_eval.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/chunked_pooling/mteb_chunked_eval.py b/chunked_pooling/mteb_chunked_eval.py index f533354..62400bf 100644 --- a/chunked_pooling/mteb_chunked_eval.py +++ b/chunked_pooling/mteb_chunked_eval.py @@ -128,16 +128,16 @@ def _truncate_documents(self, corpus): def _embed_with_soft_boundary(self, model, model_inputs): - tokens = model_inputs.tokens() + len_tokens = len(model_inputs["input_ids"][0]) - if len(tokens) > self.soft_boundary_embed_size: + if len_tokens > self.soft_boundary_embed_size: indices = [] - for i in range(0, len(tokens), self.soft_boundary_embed_size - self.soft_boundary_overlap_size): + for i in range(0, len_tokens, self.soft_boundary_embed_size - self.soft_boundary_overlap_size): start = i - end = min(i + self.soft_boundary_embed_size, len(tokens)) + end = min(i + self.soft_boundary_embed_size, len_tokens) indices.append((start, end)) else: - indices = [(0, len(tokens))] + indices = [(0, len_tokens)] outputs = [] for start, end in indices: @@ -156,16 +156,16 @@ def _embed_with_soft_boundary(self, model, model_inputs): def _embed_with_hard_boundary(self, model, model_inputs): - tokens = model_inputs.tokens() + len_tokens = len(model_inputs["input_ids"][0]) - if len(tokens) > self.hard_boundary_embed_size: + if len_tokens > self.hard_boundary_embed_size: indices = [] - for i in range(0, len(tokens), self.hard_boundary_embed_size): + for i in range(0, len_tokens, self.hard_boundary_embed_size): start = i - end = min(i + self.hard_boundary_embed_size, len(tokens)) + end = min(i + self.hard_boundary_embed_size, len_tokens) indices.append((start, end)) else: - indices = [(0, len(tokens))] + indices = [(0, len_tokens)] outputs = [] for start, end in indices: From b4675752d9b66c2bbf8d682707bb780f1d71ac12 Mon Sep 17 00:00:00 2001 From: dannyjameswilliams Date: Thu, 26 Sep 2024 11:51:01 +0100 Subject: [PATCH 03/30] soft boundary correctly overlaps for later batches --- chunked_pooling/mteb_chunked_eval.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/chunked_pooling/mteb_chunked_eval.py b/chunked_pooling/mteb_chunked_eval.py index 62400bf..a6ac6c3 100644 --- a/chunked_pooling/mteb_chunked_eval.py +++ b/chunked_pooling/mteb_chunked_eval.py @@ -127,28 +127,28 @@ def _truncate_documents(self, corpus): return corpus def _embed_with_soft_boundary(self, model, model_inputs): - len_tokens = len(model_inputs["input_ids"][0]) if len_tokens > self.soft_boundary_embed_size: indices = [] - for i in range(0, len_tokens, self.soft_boundary_embed_size - self.soft_boundary_overlap_size): - start = i - end = min(i + self.soft_boundary_embed_size, len_tokens) + start = 0 + while start < len_tokens: + end = min(start + self.soft_boundary_embed_size, len_tokens) indices.append((start, end)) + start = end - self.soft_boundary_overlap_size else: indices = [(0, len_tokens)] outputs = [] - for start, end in indices: - + for i, (start, end) in enumerate(indices): batch_inputs = {k: v[:, start:end] for k, v in model_inputs.items()} with torch.no_grad(): model_output = model(**batch_inputs) - if start > 0: - outputs.append(model_output[0][:, self.soft_boundary_overlap_size:]) + if i > 0: + overlap = self.soft_boundary_overlap_size * i + outputs.append(model_output[0][:, overlap:]) else: outputs.append(model_output[0]) @@ -247,7 +247,8 @@ def _evaluate_monolingual( text_inputs, return_tensors='pt', padding=True, - truncation=False + truncation=True, + max_length=self.truncate_max_length, ) if model.device.type == 'cuda': model_inputs = { From d0ef88a0f977d5a52352b6f1dfd25f90eea1f8da Mon Sep 17 00:00:00 2001 From: dannyjameswilliams Date: Thu, 26 Sep 2024 11:58:03 +0100 Subject: [PATCH 04/30] removed incorrect soft --- chunked_pooling/mteb_chunked_eval.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/chunked_pooling/mteb_chunked_eval.py b/chunked_pooling/mteb_chunked_eval.py index a6ac6c3..719a0ed 100644 --- a/chunked_pooling/mteb_chunked_eval.py +++ b/chunked_pooling/mteb_chunked_eval.py @@ -127,28 +127,28 @@ def _truncate_documents(self, corpus): return corpus def _embed_with_soft_boundary(self, model, model_inputs): + len_tokens = len(model_inputs["input_ids"][0]) if len_tokens > self.soft_boundary_embed_size: indices = [] - start = 0 - while start < len_tokens: - end = min(start + self.soft_boundary_embed_size, len_tokens) + for pos, i in enumerate(range(0, len_tokens, self.soft_boundary_embed_size - self.soft_boundary_overlap_size)): + start = i + end = min(i + self.soft_boundary_embed_size, len_tokens) indices.append((start, end)) - start = end - self.soft_boundary_overlap_size else: indices = [(0, len_tokens)] outputs = [] - for i, (start, end) in enumerate(indices): + for start, end in indices: + batch_inputs = {k: v[:, start:end] for k, v in model_inputs.items()} with torch.no_grad(): model_output = model(**batch_inputs) - if i > 0: - overlap = self.soft_boundary_overlap_size * i - outputs.append(model_output[0][:, overlap:]) + if start > 0: + outputs.append(model_output[0][:, self.soft_boundary_overlap_size:]) else: outputs.append(model_output[0]) From 7985c6681c022383924133f6ef58cd993ffd6f69 Mon Sep 17 00:00:00 2001 From: dannyjameswilliams Date: Thu, 26 Sep 2024 12:03:09 +0100 Subject: [PATCH 05/30] added loop over overlap sizes --- chunked_pooling/mteb_chunked_eval.py | 4 +- soft_token_boundary.py | 59 +++++++++++++--------------- 2 files changed, 29 insertions(+), 34 deletions(-) diff --git a/chunked_pooling/mteb_chunked_eval.py b/chunked_pooling/mteb_chunked_eval.py index 719a0ed..4616124 100644 --- a/chunked_pooling/mteb_chunked_eval.py +++ b/chunked_pooling/mteb_chunked_eval.py @@ -121,8 +121,6 @@ def _truncate_documents(self, corpus): max_length=self.truncate_max_length, ) last_token_span = tokens.offset_mapping[-2] - if len(self.tokenizer(v["text"]).tokens()) > self.truncate_max_length: - print(f"Document {k} will be truncated to {self.truncate_max_length} tokens") v['text'] = v['text'][: last_token_span[1]] return corpus @@ -132,7 +130,7 @@ def _embed_with_soft_boundary(self, model, model_inputs): if len_tokens > self.soft_boundary_embed_size: indices = [] - for pos, i in enumerate(range(0, len_tokens, self.soft_boundary_embed_size - self.soft_boundary_overlap_size)): + for i in range(0, len_tokens, self.soft_boundary_embed_size - self.soft_boundary_overlap_size): start = i end = min(i + self.soft_boundary_embed_size, len_tokens) indices.append((start, end)) diff --git a/soft_token_boundary.py b/soft_token_boundary.py index e77caa1..88859e3 100644 --- a/soft_token_boundary.py +++ b/soft_token_boundary.py @@ -82,12 +82,6 @@ type=int, help='Token length of the embeddings that come before/after soft boundaries (i.e. overlapping embeddings). Above zero, soft boundaries are used.', ) -@click.option( - '--soft_boundary_overlap_size', - default=512, - type=int, - help='Overlap between the embeddings before and after the soft boundary, in tokens.', -) @click.option( '--hard_boundary_embed_size', default=8192, @@ -105,7 +99,6 @@ def main( chunk_size, n_sentences, soft_boundary_embed_size, - soft_boundary_overlap_size, hard_boundary_embed_size, ): try: @@ -130,35 +123,39 @@ def main( model.eval() - # Evaluate with soft boundary - tasks = [ - task_cls( + overlap_sizes = [32, 64, 128, 256, 512] + for overlap_size in overlap_sizes: + + + # Evaluate with soft boundary + tasks = [ + task_cls( + chunked_pooling_enabled=True, + tokenizer=tokenizer, + prune_size=None, + truncate_max_length=0, + soft_boundary_embed_size=soft_boundary_embed_size, + soft_boundary_overlap_size=overlap_size, + hard_boundary_embed_size=0, + **chunking_args, + ) + ] + + evaluation = MTEB( + tasks=tasks, chunked_pooling_enabled=True, tokenizer=tokenizer, prune_size=None, - truncate_max_length=0, - soft_boundary_embed_size=soft_boundary_embed_size, - soft_boundary_overlap_size=soft_boundary_overlap_size, - hard_boundary_embed_size=0, **chunking_args, ) - ] - - evaluation = MTEB( - tasks=tasks, - chunked_pooling_enabled=True, - tokenizer=tokenizer, - prune_size=None, - **chunking_args, - ) - evaluation.run( - model, - output_folder=f'results-soft-boundary/embed_size_{soft_boundary_embed_size}', - eval_splits=[eval_split], - overwrite_results=True, - batch_size=BATCH_SIZE, - encode_kwargs={'batch_size': BATCH_SIZE}, - ) + evaluation.run( + model, + output_folder=f'results-soft-boundary/embed_size_{soft_boundary_embed_size}/overlap_{overlap_size}', + eval_splits=[eval_split], + overwrite_results=True, + batch_size=BATCH_SIZE, + encode_kwargs={'batch_size': BATCH_SIZE}, + ) # Evaluate with hard boundary From 56a6a41759df054defe2c27b7d9a6002e1239ac3 Mon Sep 17 00:00:00 2001 From: dannyjameswilliams Date: Thu, 26 Sep 2024 13:25:43 +0100 Subject: [PATCH 06/30] added results --- parse_experiments.py | 85 +++++++++++++++++++ .../LEMBWikimQARetrievalChunked.json | 38 +++++++++ .../no_revision_available/model_meta.json | 1 + .../LEMBWikimQARetrievalChunked.json | 38 +++++++++ .../no_revision_available/model_meta.json | 1 + .../LEMBWikimQARetrievalChunked.json | 38 +++++++++ .../no_revision_available/model_meta.json | 1 + .../LEMBWikimQARetrievalChunked.json | 38 +++++++++ .../no_revision_available/model_meta.json | 1 + .../LEMBWikimQARetrievalChunked.json | 38 +++++++++ .../no_revision_available/model_meta.json | 1 + .../LEMBWikimQARetrievalChunked.json | 38 +++++++++ .../no_revision_available/model_meta.json | 1 + .../LEMBWikimQARetrievalChunked.json | 38 +++++++++ .../no_revision_available/model_meta.json | 1 + .../LEMBWikimQARetrievalChunked.json | 38 +++++++++ .../no_revision_available/model_meta.json | 1 + .../LEMBWikimQARetrievalChunked.json | 38 +++++++++ .../no_revision_available/model_meta.json | 1 + .../LEMBWikimQARetrievalChunked.json | 38 +++++++++ .../no_revision_available/model_meta.json | 1 + .../LEMBWikimQARetrievalChunked.json | 38 +++++++++ .../no_revision_available/model_meta.json | 1 + .../LEMBWikimQARetrievalChunked.json | 38 +++++++++ .../no_revision_available/model_meta.json | 1 + .../LEMBWikimQARetrievalChunked.json | 38 +++++++++ .../no_revision_available/model_meta.json | 1 + .../LEMBWikimQARetrievalChunked.json | 38 +++++++++ .../no_revision_available/model_meta.json | 1 + .../LEMBWikimQARetrievalChunked.json | 38 +++++++++ .../no_revision_available/model_meta.json | 1 + .../LEMBWikimQARetrievalChunked.json | 38 +++++++++ .../no_revision_available/model_meta.json | 1 + .../LEMBWikimQARetrievalChunked.json | 38 +++++++++ .../no_revision_available/model_meta.json | 1 + .../LEMBWikimQARetrievalChunked.json | 38 +++++++++ .../no_revision_available/model_meta.json | 1 + .../LEMBWikimQARetrievalChunked.json | 38 +++++++++ .../no_revision_available/model_meta.json | 1 + .../LEMBWikimQARetrievalChunked.json | 38 +++++++++ .../no_revision_available/model_meta.json | 1 + .../LEMBWikimQARetrievalChunked.json | 38 +++++++++ .../no_revision_available/model_meta.json | 1 + .../LEMBWikimQARetrievalChunked.json | 38 +++++++++ .../no_revision_available/model_meta.json | 1 + .../LEMBWikimQARetrievalChunked.json | 38 +++++++++ .../no_revision_available/model_meta.json | 1 + .../LEMBWikimQARetrievalChunked.json | 38 +++++++++ .../no_revision_available/model_meta.json | 1 + .../LEMBWikimQARetrievalChunked.json | 43 ++++++++++ .../no_revision_available/model_meta.json | 1 + 51 files changed, 1065 insertions(+) create mode 100644 parse_experiments.py create mode 100644 results-hard-boundary/embed_size_1024/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json create mode 100644 results-hard-boundary/embed_size_1024/no_model_name_available/no_revision_available/model_meta.json create mode 100644 results-hard-boundary/embed_size_2048/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json create mode 100644 results-hard-boundary/embed_size_2048/no_model_name_available/no_revision_available/model_meta.json create mode 100644 results-hard-boundary/embed_size_4096/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json create mode 100644 results-hard-boundary/embed_size_4096/no_model_name_available/no_revision_available/model_meta.json create mode 100644 results-hard-boundary/embed_size_8192/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json create mode 100644 results-hard-boundary/embed_size_8192/no_model_name_available/no_revision_available/model_meta.json create mode 100644 results-soft-boundary/embed_size_1024/overlap_128/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json create mode 100644 results-soft-boundary/embed_size_1024/overlap_128/no_model_name_available/no_revision_available/model_meta.json create mode 100644 results-soft-boundary/embed_size_1024/overlap_256/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json create mode 100644 results-soft-boundary/embed_size_1024/overlap_256/no_model_name_available/no_revision_available/model_meta.json create mode 100644 results-soft-boundary/embed_size_1024/overlap_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json create mode 100644 results-soft-boundary/embed_size_1024/overlap_32/no_model_name_available/no_revision_available/model_meta.json create mode 100644 results-soft-boundary/embed_size_1024/overlap_512/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json create mode 100644 results-soft-boundary/embed_size_1024/overlap_512/no_model_name_available/no_revision_available/model_meta.json create mode 100644 results-soft-boundary/embed_size_1024/overlap_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json create mode 100644 results-soft-boundary/embed_size_1024/overlap_64/no_model_name_available/no_revision_available/model_meta.json create mode 100644 results-soft-boundary/embed_size_2048/overlap_128/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json create mode 100644 results-soft-boundary/embed_size_2048/overlap_128/no_model_name_available/no_revision_available/model_meta.json create mode 100644 results-soft-boundary/embed_size_2048/overlap_256/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json create mode 100644 results-soft-boundary/embed_size_2048/overlap_256/no_model_name_available/no_revision_available/model_meta.json create mode 100644 results-soft-boundary/embed_size_2048/overlap_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json create mode 100644 results-soft-boundary/embed_size_2048/overlap_32/no_model_name_available/no_revision_available/model_meta.json create mode 100644 results-soft-boundary/embed_size_2048/overlap_512/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json create mode 100644 results-soft-boundary/embed_size_2048/overlap_512/no_model_name_available/no_revision_available/model_meta.json create mode 100644 results-soft-boundary/embed_size_2048/overlap_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json create mode 100644 results-soft-boundary/embed_size_2048/overlap_64/no_model_name_available/no_revision_available/model_meta.json create mode 100644 results-soft-boundary/embed_size_4096/overlap_128/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json create mode 100644 results-soft-boundary/embed_size_4096/overlap_128/no_model_name_available/no_revision_available/model_meta.json create mode 100644 results-soft-boundary/embed_size_4096/overlap_256/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json create mode 100644 results-soft-boundary/embed_size_4096/overlap_256/no_model_name_available/no_revision_available/model_meta.json create mode 100644 results-soft-boundary/embed_size_4096/overlap_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json create mode 100644 results-soft-boundary/embed_size_4096/overlap_32/no_model_name_available/no_revision_available/model_meta.json create mode 100644 results-soft-boundary/embed_size_4096/overlap_512/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json create mode 100644 results-soft-boundary/embed_size_4096/overlap_512/no_model_name_available/no_revision_available/model_meta.json create mode 100644 results-soft-boundary/embed_size_4096/overlap_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json create mode 100644 results-soft-boundary/embed_size_4096/overlap_64/no_model_name_available/no_revision_available/model_meta.json create mode 100644 results-soft-boundary/embed_size_8192/overlap_128/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json create mode 100644 results-soft-boundary/embed_size_8192/overlap_128/no_model_name_available/no_revision_available/model_meta.json create mode 100644 results-soft-boundary/embed_size_8192/overlap_256/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json create mode 100644 results-soft-boundary/embed_size_8192/overlap_256/no_model_name_available/no_revision_available/model_meta.json create mode 100644 results-soft-boundary/embed_size_8192/overlap_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json create mode 100644 results-soft-boundary/embed_size_8192/overlap_32/no_model_name_available/no_revision_available/model_meta.json create mode 100644 results-soft-boundary/embed_size_8192/overlap_512/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json create mode 100644 results-soft-boundary/embed_size_8192/overlap_512/no_model_name_available/no_revision_available/model_meta.json create mode 100644 results-soft-boundary/embed_size_8192/overlap_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json create mode 100644 results-soft-boundary/embed_size_8192/overlap_64/no_model_name_available/no_revision_available/model_meta.json create mode 100644 results-truncation/embed_size_8192/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json create mode 100644 results-truncation/embed_size_8192/no_model_name_available/no_revision_available/model_meta.json diff --git a/parse_experiments.py b/parse_experiments.py new file mode 100644 index 0000000..e7979f6 --- /dev/null +++ b/parse_experiments.py @@ -0,0 +1,85 @@ +# overlap size and soft boundary + +import numpy as np + +import matplotlib.pyplot as plt + +import json + + +import os + + + + + + + + + + +if __name__ == "__main__": + + task_name = "LEMBWikimQARetrievalChunked" + soft_boundary_dir = "results-soft-boundary" + hard_boundary_dir = "results-hard-boundary" + truncate_dir = "results-truncation" + + embed_sizes = [1024, 2048, 4096, 8192] + overlap_sizes = [32, 64, 128, 256, 512] + + soft_boundary_ndcgs = np.empty((len(embed_sizes), len(overlap_sizes))) + hard_boundary_ndcgs = np.empty(len(embed_sizes)) + + soft_boundary_maps = np.empty((len(embed_sizes), len(overlap_sizes))) + hard_boundary_maps = np.empty(len(embed_sizes)) + + for embed_i, embed_size in enumerate(embed_sizes): + + for overlap_i, overlap_size in enumerate(overlap_sizes): + + soft_dir = os.path.join(soft_boundary_dir, f"embed_size_{embed_size}", f"overlap_{overlap_size}", "no_model_name_available", "no_revision_available", f"{task_name}.json") + with open(soft_dir, "r") as f: + results = json.load(f) + + soft_boundary_ndcgs[embed_i, overlap_i] = results["scores"]["test"][0]["ndcg_at_10"] + soft_boundary_maps[embed_i, overlap_i] = results["scores"]["test"][0]["map_at_10"] + + soft_dir = os.path.join(hard_boundary_dir, f"embed_size_{embed_size}", "no_model_name_available", "no_revision_available", f"{task_name}.json") + with open(soft_dir, "r") as f: + results = json.load(f) + + hard_boundary_ndcgs[embed_i] = results["scores"]["test"][0]["ndcg_at_10"] + hard_boundary_maps[embed_i] = results["scores"]["test"][0]["map_at_10"] + + + # NDCG + fig, ax = plt.subplots(len(embed_sizes), 1, figsize=(12, 9)) + + for embed_i, embed_size in enumerate(embed_sizes): + + ax[embed_i].plot(overlap_sizes, soft_boundary_ndcgs[embed_i], label="Soft Boundary") + ax[embed_i].scatter(overlap_sizes, soft_boundary_ndcgs[embed_i]) + ax[embed_i].axhline(hard_boundary_ndcgs[embed_i], color="red", linestyle="--", label="Hard Boundary") + + ax[embed_i].set_title(f"Embed Size: {embed_size}") + ax[embed_i].set_xlabel("Overlap Size") + ax[embed_i].set_ylabel("nCDG@10") + + ax[embed_i].legend() + fig.tight_layout() + + # MAP + fig, ax = plt.subplots(len(embed_sizes), 1, figsize=(12, 9)) + for embed_i, embed_size in enumerate(embed_sizes): + + ax[embed_i].plot(overlap_sizes, soft_boundary_maps[embed_i], label="Soft Boundary") + ax[embed_i].scatter(overlap_sizes, soft_boundary_maps[embed_i]) + ax[embed_i].axhline(hard_boundary_maps[embed_i], color="red", linestyle="--", label="Hard Boundary") + + ax[embed_i].set_title(f"Embed Size: {embed_size}") + ax[embed_i].set_xlabel("Overlap Size") + ax[embed_i].set_ylabel("mAP@10") + + ax[embed_i].legend() + fig.tight_layout() \ No newline at end of file diff --git a/results-hard-boundary/embed_size_1024/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-hard-boundary/embed_size_1024/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json new file mode 100644 index 0000000..51eb6ed --- /dev/null +++ b/results-hard-boundary/embed_size_1024/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -0,0 +1,38 @@ +{ + "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", + "evaluation_time": 14.709291696548462, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.94987, + "map_at_1": 0.91333, + "map_at_10": 0.93901, + "map_at_3": 0.93389, + "map_at_5": 0.93772, + "mrr_at_1": 0.9133333333333333, + "mrr_at_10": 0.9390079365079366, + "mrr_at_3": 0.933888888888889, + "mrr_at_5": 0.9377222222222222, + "ndcg_at_1": 0.91333, + "ndcg_at_10": 0.94987, + "ndcg_at_3": 0.9398, + "ndcg_at_5": 0.94669, + "precision_at_1": 0.91333, + "precision_at_10": 0.09833, + "precision_at_3": 0.31889, + "precision_at_5": 0.19467, + "recall_at_1": 0.91333, + "recall_at_10": 0.98333, + "recall_at_3": 0.95667, + "recall_at_5": 0.97333 + } + ] + }, + "task_name": "LEMBWikimQARetrievalChunked" +} \ No newline at end of file diff --git a/results-hard-boundary/embed_size_1024/no_model_name_available/no_revision_available/model_meta.json b/results-hard-boundary/embed_size_1024/no_model_name_available/no_revision_available/model_meta.json new file mode 100644 index 0000000..40ff52f --- /dev/null +++ b/results-hard-boundary/embed_size_1024/no_model_name_available/no_revision_available/model_meta.json @@ -0,0 +1 @@ +{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-hard-boundary/embed_size_2048/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-hard-boundary/embed_size_2048/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json new file mode 100644 index 0000000..4d319ce --- /dev/null +++ b/results-hard-boundary/embed_size_2048/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -0,0 +1,38 @@ +{ + "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", + "evaluation_time": 19.393415451049805, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.94552, + "map_at_1": 0.91, + "map_at_10": 0.93434, + "map_at_3": 0.93, + "map_at_5": 0.93233, + "mrr_at_1": 0.91, + "mrr_at_10": 0.9343373015873015, + "mrr_at_3": 0.93, + "mrr_at_5": 0.9323333333333333, + "ndcg_at_1": 0.91, + "ndcg_at_10": 0.94552, + "ndcg_at_3": 0.93682, + "ndcg_at_5": 0.94099, + "precision_at_1": 0.91, + "precision_at_10": 0.098, + "precision_at_3": 0.31889, + "precision_at_5": 0.19333, + "recall_at_1": 0.91, + "recall_at_10": 0.98, + "recall_at_3": 0.95667, + "recall_at_5": 0.96667 + } + ] + }, + "task_name": "LEMBWikimQARetrievalChunked" +} \ No newline at end of file diff --git a/results-hard-boundary/embed_size_2048/no_model_name_available/no_revision_available/model_meta.json b/results-hard-boundary/embed_size_2048/no_model_name_available/no_revision_available/model_meta.json new file mode 100644 index 0000000..40ff52f --- /dev/null +++ b/results-hard-boundary/embed_size_2048/no_model_name_available/no_revision_available/model_meta.json @@ -0,0 +1 @@ +{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-hard-boundary/embed_size_4096/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-hard-boundary/embed_size_4096/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json new file mode 100644 index 0000000..4335934 --- /dev/null +++ b/results-hard-boundary/embed_size_4096/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -0,0 +1,38 @@ +{ + "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", + "evaluation_time": 26.35363245010376, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.94626, + "map_at_1": 0.90667, + "map_at_10": 0.9342, + "map_at_3": 0.93, + "map_at_5": 0.93283, + "mrr_at_1": 0.9066666666666666, + "mrr_at_10": 0.9341984126984126, + "mrr_at_3": 0.93, + "mrr_at_5": 0.9328333333333334, + "ndcg_at_1": 0.90667, + "ndcg_at_10": 0.94626, + "ndcg_at_3": 0.9377, + "ndcg_at_5": 0.943, + "precision_at_1": 0.90667, + "precision_at_10": 0.09833, + "precision_at_3": 0.32, + "precision_at_5": 0.19467, + "recall_at_1": 0.90667, + "recall_at_10": 0.98333, + "recall_at_3": 0.96, + "recall_at_5": 0.97333 + } + ] + }, + "task_name": "LEMBWikimQARetrievalChunked" +} \ No newline at end of file diff --git a/results-hard-boundary/embed_size_4096/no_model_name_available/no_revision_available/model_meta.json b/results-hard-boundary/embed_size_4096/no_model_name_available/no_revision_available/model_meta.json new file mode 100644 index 0000000..40ff52f --- /dev/null +++ b/results-hard-boundary/embed_size_4096/no_model_name_available/no_revision_available/model_meta.json @@ -0,0 +1 @@ +{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-hard-boundary/embed_size_8192/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-hard-boundary/embed_size_8192/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json new file mode 100644 index 0000000..78a2879 --- /dev/null +++ b/results-hard-boundary/embed_size_8192/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -0,0 +1,38 @@ +{ + "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", + "evaluation_time": 37.40705418586731, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.94608, + "map_at_1": 0.90667, + "map_at_10": 0.93403, + "map_at_3": 0.93, + "map_at_5": 0.93233, + "mrr_at_1": 0.9066666666666666, + "mrr_at_10": 0.9340317460317461, + "mrr_at_3": 0.93, + "mrr_at_5": 0.9323333333333333, + "ndcg_at_1": 0.90667, + "ndcg_at_10": 0.94608, + "ndcg_at_3": 0.9377, + "ndcg_at_5": 0.94186, + "precision_at_1": 0.90667, + "precision_at_10": 0.09833, + "precision_at_3": 0.32, + "precision_at_5": 0.194, + "recall_at_1": 0.90667, + "recall_at_10": 0.98333, + "recall_at_3": 0.96, + "recall_at_5": 0.97 + } + ] + }, + "task_name": "LEMBWikimQARetrievalChunked" +} \ No newline at end of file diff --git a/results-hard-boundary/embed_size_8192/no_model_name_available/no_revision_available/model_meta.json b/results-hard-boundary/embed_size_8192/no_model_name_available/no_revision_available/model_meta.json new file mode 100644 index 0000000..40ff52f --- /dev/null +++ b/results-hard-boundary/embed_size_8192/no_model_name_available/no_revision_available/model_meta.json @@ -0,0 +1 @@ +{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_1024/overlap_128/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-soft-boundary/embed_size_1024/overlap_128/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json new file mode 100644 index 0000000..b812dc9 --- /dev/null +++ b/results-soft-boundary/embed_size_1024/overlap_128/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -0,0 +1,38 @@ +{ + "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", + "evaluation_time": 15.548282861709595, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.94486, + "map_at_1": 0.90667, + "map_at_10": 0.93423, + "map_at_3": 0.93333, + "map_at_5": 0.93333, + "mrr_at_1": 0.9066666666666666, + "mrr_at_10": 0.9342261904761906, + "mrr_at_3": 0.9333333333333333, + "mrr_at_5": 0.9333333333333333, + "ndcg_at_1": 0.90667, + "ndcg_at_10": 0.94486, + "ndcg_at_3": 0.9427, + "ndcg_at_5": 0.9427, + "precision_at_1": 0.90667, + "precision_at_10": 0.09767, + "precision_at_3": 0.32333, + "precision_at_5": 0.194, + "recall_at_1": 0.90667, + "recall_at_10": 0.97667, + "recall_at_3": 0.97, + "recall_at_5": 0.97 + } + ] + }, + "task_name": "LEMBWikimQARetrievalChunked" +} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_1024/overlap_128/no_model_name_available/no_revision_available/model_meta.json b/results-soft-boundary/embed_size_1024/overlap_128/no_model_name_available/no_revision_available/model_meta.json new file mode 100644 index 0000000..40ff52f --- /dev/null +++ b/results-soft-boundary/embed_size_1024/overlap_128/no_model_name_available/no_revision_available/model_meta.json @@ -0,0 +1 @@ +{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_1024/overlap_256/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-soft-boundary/embed_size_1024/overlap_256/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json new file mode 100644 index 0000000..674accc --- /dev/null +++ b/results-soft-boundary/embed_size_1024/overlap_256/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -0,0 +1,38 @@ +{ + "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", + "evaluation_time": 16.896415948867798, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.9528, + "map_at_1": 0.92, + "map_at_10": 0.94381, + "map_at_3": 0.94167, + "map_at_5": 0.94233, + "mrr_at_1": 0.92, + "mrr_at_10": 0.9438148148148148, + "mrr_at_3": 0.9416666666666667, + "mrr_at_5": 0.9423333333333332, + "ndcg_at_1": 0.92, + "ndcg_at_10": 0.9528, + "ndcg_at_3": 0.94813, + "ndcg_at_5": 0.94942, + "precision_at_1": 0.92, + "precision_at_10": 0.098, + "precision_at_3": 0.32222, + "precision_at_5": 0.194, + "recall_at_1": 0.92, + "recall_at_10": 0.98, + "recall_at_3": 0.96667, + "recall_at_5": 0.97 + } + ] + }, + "task_name": "LEMBWikimQARetrievalChunked" +} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_1024/overlap_256/no_model_name_available/no_revision_available/model_meta.json b/results-soft-boundary/embed_size_1024/overlap_256/no_model_name_available/no_revision_available/model_meta.json new file mode 100644 index 0000000..40ff52f --- /dev/null +++ b/results-soft-boundary/embed_size_1024/overlap_256/no_model_name_available/no_revision_available/model_meta.json @@ -0,0 +1 @@ +{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_1024/overlap_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-soft-boundary/embed_size_1024/overlap_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json new file mode 100644 index 0000000..dc09724 --- /dev/null +++ b/results-soft-boundary/embed_size_1024/overlap_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -0,0 +1,38 @@ +{ + "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", + "evaluation_time": 14.820404767990112, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.9417, + "map_at_1": 0.90333, + "map_at_10": 0.93037, + "map_at_3": 0.92722, + "map_at_5": 0.92872, + "mrr_at_1": 0.9033333333333333, + "mrr_at_10": 0.9303650793650794, + "mrr_at_3": 0.9272222222222223, + "mrr_at_5": 0.9287222222222222, + "ndcg_at_1": 0.90333, + "ndcg_at_10": 0.9417, + "ndcg_at_3": 0.9348, + "ndcg_at_5": 0.93753, + "precision_at_1": 0.90333, + "precision_at_10": 0.09767, + "precision_at_3": 0.31889, + "precision_at_5": 0.19267, + "recall_at_1": 0.90333, + "recall_at_10": 0.97667, + "recall_at_3": 0.95667, + "recall_at_5": 0.96333 + } + ] + }, + "task_name": "LEMBWikimQARetrievalChunked" +} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_1024/overlap_32/no_model_name_available/no_revision_available/model_meta.json b/results-soft-boundary/embed_size_1024/overlap_32/no_model_name_available/no_revision_available/model_meta.json new file mode 100644 index 0000000..40ff52f --- /dev/null +++ b/results-soft-boundary/embed_size_1024/overlap_32/no_model_name_available/no_revision_available/model_meta.json @@ -0,0 +1 @@ +{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_1024/overlap_512/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-soft-boundary/embed_size_1024/overlap_512/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json new file mode 100644 index 0000000..5f6d37a --- /dev/null +++ b/results-soft-boundary/embed_size_1024/overlap_512/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -0,0 +1,38 @@ +{ + "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", + "evaluation_time": 21.096606016159058, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.95035, + "map_at_1": 0.91333, + "map_at_10": 0.94048, + "map_at_3": 0.93833, + "map_at_5": 0.93833, + "mrr_at_1": 0.9133333333333333, + "mrr_at_10": 0.9404761904761906, + "mrr_at_3": 0.9383333333333334, + "mrr_at_5": 0.9383333333333334, + "ndcg_at_1": 0.91333, + "ndcg_at_10": 0.95035, + "ndcg_at_3": 0.94567, + "ndcg_at_5": 0.94567, + "precision_at_1": 0.91333, + "precision_at_10": 0.098, + "precision_at_3": 0.32222, + "precision_at_5": 0.19333, + "recall_at_1": 0.91333, + "recall_at_10": 0.98, + "recall_at_3": 0.96667, + "recall_at_5": 0.96667 + } + ] + }, + "task_name": "LEMBWikimQARetrievalChunked" +} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_1024/overlap_512/no_model_name_available/no_revision_available/model_meta.json b/results-soft-boundary/embed_size_1024/overlap_512/no_model_name_available/no_revision_available/model_meta.json new file mode 100644 index 0000000..40ff52f --- /dev/null +++ b/results-soft-boundary/embed_size_1024/overlap_512/no_model_name_available/no_revision_available/model_meta.json @@ -0,0 +1 @@ +{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_1024/overlap_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-soft-boundary/embed_size_1024/overlap_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json new file mode 100644 index 0000000..ad117f0 --- /dev/null +++ b/results-soft-boundary/embed_size_1024/overlap_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -0,0 +1,38 @@ +{ + "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", + "evaluation_time": 15.033222198486328, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.94176, + "map_at_1": 0.89667, + "map_at_10": 0.92912, + "map_at_3": 0.92556, + "map_at_5": 0.92822, + "mrr_at_1": 0.8966666666666666, + "mrr_at_10": 0.9291150793650793, + "mrr_at_3": 0.9255555555555556, + "mrr_at_5": 0.9282222222222222, + "ndcg_at_1": 0.89667, + "ndcg_at_10": 0.94176, + "ndcg_at_3": 0.93444, + "ndcg_at_5": 0.9396, + "precision_at_1": 0.89667, + "precision_at_10": 0.098, + "precision_at_3": 0.32, + "precision_at_5": 0.19467, + "recall_at_1": 0.89667, + "recall_at_10": 0.98, + "recall_at_3": 0.96, + "recall_at_5": 0.97333 + } + ] + }, + "task_name": "LEMBWikimQARetrievalChunked" +} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_1024/overlap_64/no_model_name_available/no_revision_available/model_meta.json b/results-soft-boundary/embed_size_1024/overlap_64/no_model_name_available/no_revision_available/model_meta.json new file mode 100644 index 0000000..40ff52f --- /dev/null +++ b/results-soft-boundary/embed_size_1024/overlap_64/no_model_name_available/no_revision_available/model_meta.json @@ -0,0 +1 @@ +{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_2048/overlap_128/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-soft-boundary/embed_size_2048/overlap_128/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json new file mode 100644 index 0000000..b15a329 --- /dev/null +++ b/results-soft-boundary/embed_size_2048/overlap_128/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -0,0 +1,38 @@ +{ + "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", + "evaluation_time": 19.98063826560974, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.94518, + "map_at_1": 0.91, + "map_at_10": 0.93492, + "map_at_3": 0.93056, + "map_at_5": 0.93356, + "mrr_at_1": 0.91, + "mrr_at_10": 0.9349206349206349, + "mrr_at_3": 0.9305555555555556, + "mrr_at_5": 0.9335555555555556, + "ndcg_at_1": 0.91, + "ndcg_at_10": 0.94518, + "ndcg_at_3": 0.93647, + "ndcg_at_5": 0.94192, + "precision_at_1": 0.91, + "precision_at_10": 0.09767, + "precision_at_3": 0.31778, + "precision_at_5": 0.19333, + "recall_at_1": 0.91, + "recall_at_10": 0.97667, + "recall_at_3": 0.95333, + "recall_at_5": 0.96667 + } + ] + }, + "task_name": "LEMBWikimQARetrievalChunked" +} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_2048/overlap_128/no_model_name_available/no_revision_available/model_meta.json b/results-soft-boundary/embed_size_2048/overlap_128/no_model_name_available/no_revision_available/model_meta.json new file mode 100644 index 0000000..40ff52f --- /dev/null +++ b/results-soft-boundary/embed_size_2048/overlap_128/no_model_name_available/no_revision_available/model_meta.json @@ -0,0 +1 @@ +{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_2048/overlap_256/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-soft-boundary/embed_size_2048/overlap_256/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json new file mode 100644 index 0000000..519b1a4 --- /dev/null +++ b/results-soft-boundary/embed_size_2048/overlap_256/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -0,0 +1,38 @@ +{ + "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", + "evaluation_time": 20.697884798049927, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.94175, + "map_at_1": 0.9, + "map_at_10": 0.92928, + "map_at_3": 0.925, + "map_at_5": 0.9275, + "mrr_at_1": 0.9, + "mrr_at_10": 0.9292817460317461, + "mrr_at_3": 0.925, + "mrr_at_5": 0.9275, + "ndcg_at_1": 0.9, + "ndcg_at_10": 0.94175, + "ndcg_at_3": 0.93313, + "ndcg_at_5": 0.93744, + "precision_at_1": 0.9, + "precision_at_10": 0.098, + "precision_at_3": 0.31889, + "precision_at_5": 0.19333, + "recall_at_1": 0.9, + "recall_at_10": 0.98, + "recall_at_3": 0.95667, + "recall_at_5": 0.96667 + } + ] + }, + "task_name": "LEMBWikimQARetrievalChunked" +} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_2048/overlap_256/no_model_name_available/no_revision_available/model_meta.json b/results-soft-boundary/embed_size_2048/overlap_256/no_model_name_available/no_revision_available/model_meta.json new file mode 100644 index 0000000..40ff52f --- /dev/null +++ b/results-soft-boundary/embed_size_2048/overlap_256/no_model_name_available/no_revision_available/model_meta.json @@ -0,0 +1 @@ +{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_2048/overlap_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-soft-boundary/embed_size_2048/overlap_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json new file mode 100644 index 0000000..5bbf470 --- /dev/null +++ b/results-soft-boundary/embed_size_2048/overlap_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -0,0 +1,38 @@ +{ + "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", + "evaluation_time": 19.531440019607544, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.94103, + "map_at_1": 0.9, + "map_at_10": 0.92838, + "map_at_3": 0.92333, + "map_at_5": 0.92717, + "mrr_at_1": 0.9, + "mrr_at_10": 0.9283835978835979, + "mrr_at_3": 0.9233333333333333, + "mrr_at_5": 0.9271666666666666, + "ndcg_at_1": 0.9, + "ndcg_at_10": 0.94103, + "ndcg_at_3": 0.93103, + "ndcg_at_5": 0.93792, + "precision_at_1": 0.9, + "precision_at_10": 0.098, + "precision_at_3": 0.31778, + "precision_at_5": 0.194, + "recall_at_1": 0.9, + "recall_at_10": 0.98, + "recall_at_3": 0.95333, + "recall_at_5": 0.97 + } + ] + }, + "task_name": "LEMBWikimQARetrievalChunked" +} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_2048/overlap_32/no_model_name_available/no_revision_available/model_meta.json b/results-soft-boundary/embed_size_2048/overlap_32/no_model_name_available/no_revision_available/model_meta.json new file mode 100644 index 0000000..40ff52f --- /dev/null +++ b/results-soft-boundary/embed_size_2048/overlap_32/no_model_name_available/no_revision_available/model_meta.json @@ -0,0 +1 @@ +{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_2048/overlap_512/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-soft-boundary/embed_size_2048/overlap_512/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json new file mode 100644 index 0000000..b3cb768 --- /dev/null +++ b/results-soft-boundary/embed_size_2048/overlap_512/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -0,0 +1,38 @@ +{ + "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", + "evaluation_time": 22.640079736709595, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.9416, + "map_at_1": 0.89667, + "map_at_10": 0.92892, + "map_at_3": 0.92611, + "map_at_5": 0.92761, + "mrr_at_1": 0.8966666666666666, + "mrr_at_10": 0.9289166666666667, + "mrr_at_3": 0.926111111111111, + "mrr_at_5": 0.9276111111111112, + "ndcg_at_1": 0.89667, + "ndcg_at_10": 0.9416, + "ndcg_at_3": 0.93567, + "ndcg_at_5": 0.9384, + "precision_at_1": 0.89667, + "precision_at_10": 0.098, + "precision_at_3": 0.32111, + "precision_at_5": 0.194, + "recall_at_1": 0.89667, + "recall_at_10": 0.98, + "recall_at_3": 0.96333, + "recall_at_5": 0.97 + } + ] + }, + "task_name": "LEMBWikimQARetrievalChunked" +} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_2048/overlap_512/no_model_name_available/no_revision_available/model_meta.json b/results-soft-boundary/embed_size_2048/overlap_512/no_model_name_available/no_revision_available/model_meta.json new file mode 100644 index 0000000..40ff52f --- /dev/null +++ b/results-soft-boundary/embed_size_2048/overlap_512/no_model_name_available/no_revision_available/model_meta.json @@ -0,0 +1 @@ +{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_2048/overlap_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-soft-boundary/embed_size_2048/overlap_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json new file mode 100644 index 0000000..b299341 --- /dev/null +++ b/results-soft-boundary/embed_size_2048/overlap_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -0,0 +1,38 @@ +{ + "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", + "evaluation_time": 19.4959659576416, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.94172, + "map_at_1": 0.9, + "map_at_10": 0.92924, + "map_at_3": 0.925, + "map_at_5": 0.9275, + "mrr_at_1": 0.9, + "mrr_at_10": 0.9292394179894181, + "mrr_at_3": 0.925, + "mrr_at_5": 0.9275, + "ndcg_at_1": 0.9, + "ndcg_at_10": 0.94172, + "ndcg_at_3": 0.93313, + "ndcg_at_5": 0.93744, + "precision_at_1": 0.9, + "precision_at_10": 0.098, + "precision_at_3": 0.31889, + "precision_at_5": 0.19333, + "recall_at_1": 0.9, + "recall_at_10": 0.98, + "recall_at_3": 0.95667, + "recall_at_5": 0.96667 + } + ] + }, + "task_name": "LEMBWikimQARetrievalChunked" +} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_2048/overlap_64/no_model_name_available/no_revision_available/model_meta.json b/results-soft-boundary/embed_size_2048/overlap_64/no_model_name_available/no_revision_available/model_meta.json new file mode 100644 index 0000000..40ff52f --- /dev/null +++ b/results-soft-boundary/embed_size_2048/overlap_64/no_model_name_available/no_revision_available/model_meta.json @@ -0,0 +1 @@ +{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_4096/overlap_128/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-soft-boundary/embed_size_4096/overlap_128/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json new file mode 100644 index 0000000..2d201a9 --- /dev/null +++ b/results-soft-boundary/embed_size_4096/overlap_128/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -0,0 +1,38 @@ +{ + "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", + "evaluation_time": 26.74845862388611, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.9426, + "map_at_1": 0.90333, + "map_at_10": 0.93134, + "map_at_3": 0.92833, + "map_at_5": 0.92983, + "mrr_at_1": 0.9033333333333333, + "mrr_at_10": 0.9313412698412699, + "mrr_at_3": 0.9283333333333333, + "mrr_at_5": 0.9298333333333333, + "ndcg_at_1": 0.90333, + "ndcg_at_10": 0.9426, + "ndcg_at_3": 0.93647, + "ndcg_at_5": 0.93919, + "precision_at_1": 0.90333, + "precision_at_10": 0.09767, + "precision_at_3": 0.32, + "precision_at_5": 0.19333, + "recall_at_1": 0.90333, + "recall_at_10": 0.97667, + "recall_at_3": 0.96, + "recall_at_5": 0.96667 + } + ] + }, + "task_name": "LEMBWikimQARetrievalChunked" +} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_4096/overlap_128/no_model_name_available/no_revision_available/model_meta.json b/results-soft-boundary/embed_size_4096/overlap_128/no_model_name_available/no_revision_available/model_meta.json new file mode 100644 index 0000000..40ff52f --- /dev/null +++ b/results-soft-boundary/embed_size_4096/overlap_128/no_model_name_available/no_revision_available/model_meta.json @@ -0,0 +1 @@ +{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_4096/overlap_256/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-soft-boundary/embed_size_4096/overlap_256/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json new file mode 100644 index 0000000..aae5d39 --- /dev/null +++ b/results-soft-boundary/embed_size_4096/overlap_256/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -0,0 +1,38 @@ +{ + "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", + "evaluation_time": 27.24738645553589, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.94597, + "map_at_1": 0.91, + "map_at_10": 0.93492, + "map_at_3": 0.93056, + "map_at_5": 0.93356, + "mrr_at_1": 0.91, + "mrr_at_10": 0.9349206349206349, + "mrr_at_3": 0.9305555555555556, + "mrr_at_5": 0.9335555555555556, + "ndcg_at_1": 0.91, + "ndcg_at_10": 0.94597, + "ndcg_at_3": 0.93726, + "ndcg_at_5": 0.94271, + "precision_at_1": 0.91, + "precision_at_10": 0.098, + "precision_at_3": 0.31889, + "precision_at_5": 0.194, + "recall_at_1": 0.91, + "recall_at_10": 0.98, + "recall_at_3": 0.95667, + "recall_at_5": 0.97 + } + ] + }, + "task_name": "LEMBWikimQARetrievalChunked" +} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_4096/overlap_256/no_model_name_available/no_revision_available/model_meta.json b/results-soft-boundary/embed_size_4096/overlap_256/no_model_name_available/no_revision_available/model_meta.json new file mode 100644 index 0000000..40ff52f --- /dev/null +++ b/results-soft-boundary/embed_size_4096/overlap_256/no_model_name_available/no_revision_available/model_meta.json @@ -0,0 +1 @@ +{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_4096/overlap_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-soft-boundary/embed_size_4096/overlap_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json new file mode 100644 index 0000000..56212e1 --- /dev/null +++ b/results-soft-boundary/embed_size_4096/overlap_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -0,0 +1,38 @@ +{ + "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", + "evaluation_time": 26.33897829055786, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.94472, + "map_at_1": 0.90333, + "map_at_10": 0.9322, + "map_at_3": 0.92833, + "map_at_5": 0.9305, + "mrr_at_1": 0.9033333333333333, + "mrr_at_10": 0.932202380952381, + "mrr_at_3": 0.9283333333333333, + "mrr_at_5": 0.9304999999999999, + "ndcg_at_1": 0.90333, + "ndcg_at_10": 0.94472, + "ndcg_at_3": 0.93647, + "ndcg_at_5": 0.94048, + "precision_at_1": 0.90333, + "precision_at_10": 0.09833, + "precision_at_3": 0.32, + "precision_at_5": 0.194, + "recall_at_1": 0.90333, + "recall_at_10": 0.98333, + "recall_at_3": 0.96, + "recall_at_5": 0.97 + } + ] + }, + "task_name": "LEMBWikimQARetrievalChunked" +} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_4096/overlap_32/no_model_name_available/no_revision_available/model_meta.json b/results-soft-boundary/embed_size_4096/overlap_32/no_model_name_available/no_revision_available/model_meta.json new file mode 100644 index 0000000..40ff52f --- /dev/null +++ b/results-soft-boundary/embed_size_4096/overlap_32/no_model_name_available/no_revision_available/model_meta.json @@ -0,0 +1 @@ +{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_4096/overlap_512/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-soft-boundary/embed_size_4096/overlap_512/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json new file mode 100644 index 0000000..4ad6a0f --- /dev/null +++ b/results-soft-boundary/embed_size_4096/overlap_512/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -0,0 +1,38 @@ +{ + "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", + "evaluation_time": 28.279660940170288, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.94569, + "map_at_1": 0.90667, + "map_at_10": 0.93356, + "map_at_3": 0.92889, + "map_at_5": 0.93122, + "mrr_at_1": 0.9066666666666666, + "mrr_at_10": 0.9335595238095238, + "mrr_at_3": 0.928888888888889, + "mrr_at_5": 0.9312222222222223, + "ndcg_at_1": 0.90667, + "ndcg_at_10": 0.94569, + "ndcg_at_3": 0.93603, + "ndcg_at_5": 0.94019, + "precision_at_1": 0.90667, + "precision_at_10": 0.09833, + "precision_at_3": 0.31889, + "precision_at_5": 0.19333, + "recall_at_1": 0.90667, + "recall_at_10": 0.98333, + "recall_at_3": 0.95667, + "recall_at_5": 0.96667 + } + ] + }, + "task_name": "LEMBWikimQARetrievalChunked" +} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_4096/overlap_512/no_model_name_available/no_revision_available/model_meta.json b/results-soft-boundary/embed_size_4096/overlap_512/no_model_name_available/no_revision_available/model_meta.json new file mode 100644 index 0000000..40ff52f --- /dev/null +++ b/results-soft-boundary/embed_size_4096/overlap_512/no_model_name_available/no_revision_available/model_meta.json @@ -0,0 +1 @@ +{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_4096/overlap_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-soft-boundary/embed_size_4096/overlap_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json new file mode 100644 index 0000000..9a1156b --- /dev/null +++ b/results-soft-boundary/embed_size_4096/overlap_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -0,0 +1,38 @@ +{ + "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", + "evaluation_time": 26.438919067382812, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.94488, + "map_at_1": 0.90333, + "map_at_10": 0.93239, + "map_at_3": 0.92833, + "map_at_5": 0.93067, + "mrr_at_1": 0.9033333333333333, + "mrr_at_10": 0.9323888888888888, + "mrr_at_3": 0.9283333333333333, + "mrr_at_5": 0.9306666666666666, + "ndcg_at_1": 0.90333, + "ndcg_at_10": 0.94488, + "ndcg_at_3": 0.93647, + "ndcg_at_5": 0.94063, + "precision_at_1": 0.90333, + "precision_at_10": 0.09833, + "precision_at_3": 0.32, + "precision_at_5": 0.194, + "recall_at_1": 0.90333, + "recall_at_10": 0.98333, + "recall_at_3": 0.96, + "recall_at_5": 0.97 + } + ] + }, + "task_name": "LEMBWikimQARetrievalChunked" +} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_4096/overlap_64/no_model_name_available/no_revision_available/model_meta.json b/results-soft-boundary/embed_size_4096/overlap_64/no_model_name_available/no_revision_available/model_meta.json new file mode 100644 index 0000000..40ff52f --- /dev/null +++ b/results-soft-boundary/embed_size_4096/overlap_64/no_model_name_available/no_revision_available/model_meta.json @@ -0,0 +1 @@ +{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_8192/overlap_128/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-soft-boundary/embed_size_8192/overlap_128/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json new file mode 100644 index 0000000..8add32b --- /dev/null +++ b/results-soft-boundary/embed_size_8192/overlap_128/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -0,0 +1,38 @@ +{ + "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", + "evaluation_time": 37.05101680755615, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.94608, + "map_at_1": 0.90667, + "map_at_10": 0.93403, + "map_at_3": 0.93, + "map_at_5": 0.93233, + "mrr_at_1": 0.9066666666666666, + "mrr_at_10": 0.9340317460317461, + "mrr_at_3": 0.93, + "mrr_at_5": 0.9323333333333333, + "ndcg_at_1": 0.90667, + "ndcg_at_10": 0.94608, + "ndcg_at_3": 0.9377, + "ndcg_at_5": 0.94186, + "precision_at_1": 0.90667, + "precision_at_10": 0.09833, + "precision_at_3": 0.32, + "precision_at_5": 0.194, + "recall_at_1": 0.90667, + "recall_at_10": 0.98333, + "recall_at_3": 0.96, + "recall_at_5": 0.97 + } + ] + }, + "task_name": "LEMBWikimQARetrievalChunked" +} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_8192/overlap_128/no_model_name_available/no_revision_available/model_meta.json b/results-soft-boundary/embed_size_8192/overlap_128/no_model_name_available/no_revision_available/model_meta.json new file mode 100644 index 0000000..40ff52f --- /dev/null +++ b/results-soft-boundary/embed_size_8192/overlap_128/no_model_name_available/no_revision_available/model_meta.json @@ -0,0 +1 @@ +{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_8192/overlap_256/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-soft-boundary/embed_size_8192/overlap_256/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json new file mode 100644 index 0000000..b73c116 --- /dev/null +++ b/results-soft-boundary/embed_size_8192/overlap_256/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -0,0 +1,38 @@ +{ + "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", + "evaluation_time": 37.3258798122406, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.94608, + "map_at_1": 0.90667, + "map_at_10": 0.93403, + "map_at_3": 0.93, + "map_at_5": 0.93233, + "mrr_at_1": 0.9066666666666666, + "mrr_at_10": 0.9340317460317461, + "mrr_at_3": 0.93, + "mrr_at_5": 0.9323333333333333, + "ndcg_at_1": 0.90667, + "ndcg_at_10": 0.94608, + "ndcg_at_3": 0.9377, + "ndcg_at_5": 0.94186, + "precision_at_1": 0.90667, + "precision_at_10": 0.09833, + "precision_at_3": 0.32, + "precision_at_5": 0.194, + "recall_at_1": 0.90667, + "recall_at_10": 0.98333, + "recall_at_3": 0.96, + "recall_at_5": 0.97 + } + ] + }, + "task_name": "LEMBWikimQARetrievalChunked" +} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_8192/overlap_256/no_model_name_available/no_revision_available/model_meta.json b/results-soft-boundary/embed_size_8192/overlap_256/no_model_name_available/no_revision_available/model_meta.json new file mode 100644 index 0000000..40ff52f --- /dev/null +++ b/results-soft-boundary/embed_size_8192/overlap_256/no_model_name_available/no_revision_available/model_meta.json @@ -0,0 +1 @@ +{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_8192/overlap_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-soft-boundary/embed_size_8192/overlap_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json new file mode 100644 index 0000000..d870b79 --- /dev/null +++ b/results-soft-boundary/embed_size_8192/overlap_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -0,0 +1,38 @@ +{ + "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", + "evaluation_time": 37.429115772247314, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.94608, + "map_at_1": 0.90667, + "map_at_10": 0.93403, + "map_at_3": 0.93, + "map_at_5": 0.93233, + "mrr_at_1": 0.9066666666666666, + "mrr_at_10": 0.9340317460317461, + "mrr_at_3": 0.93, + "mrr_at_5": 0.9323333333333333, + "ndcg_at_1": 0.90667, + "ndcg_at_10": 0.94608, + "ndcg_at_3": 0.9377, + "ndcg_at_5": 0.94186, + "precision_at_1": 0.90667, + "precision_at_10": 0.09833, + "precision_at_3": 0.32, + "precision_at_5": 0.194, + "recall_at_1": 0.90667, + "recall_at_10": 0.98333, + "recall_at_3": 0.96, + "recall_at_5": 0.97 + } + ] + }, + "task_name": "LEMBWikimQARetrievalChunked" +} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_8192/overlap_32/no_model_name_available/no_revision_available/model_meta.json b/results-soft-boundary/embed_size_8192/overlap_32/no_model_name_available/no_revision_available/model_meta.json new file mode 100644 index 0000000..40ff52f --- /dev/null +++ b/results-soft-boundary/embed_size_8192/overlap_32/no_model_name_available/no_revision_available/model_meta.json @@ -0,0 +1 @@ +{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_8192/overlap_512/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-soft-boundary/embed_size_8192/overlap_512/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json new file mode 100644 index 0000000..cb29697 --- /dev/null +++ b/results-soft-boundary/embed_size_8192/overlap_512/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -0,0 +1,38 @@ +{ + "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", + "evaluation_time": 37.87094449996948, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.94608, + "map_at_1": 0.90667, + "map_at_10": 0.93403, + "map_at_3": 0.93, + "map_at_5": 0.93233, + "mrr_at_1": 0.9066666666666666, + "mrr_at_10": 0.9340317460317461, + "mrr_at_3": 0.93, + "mrr_at_5": 0.9323333333333333, + "ndcg_at_1": 0.90667, + "ndcg_at_10": 0.94608, + "ndcg_at_3": 0.9377, + "ndcg_at_5": 0.94186, + "precision_at_1": 0.90667, + "precision_at_10": 0.09833, + "precision_at_3": 0.32, + "precision_at_5": 0.194, + "recall_at_1": 0.90667, + "recall_at_10": 0.98333, + "recall_at_3": 0.96, + "recall_at_5": 0.97 + } + ] + }, + "task_name": "LEMBWikimQARetrievalChunked" +} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_8192/overlap_512/no_model_name_available/no_revision_available/model_meta.json b/results-soft-boundary/embed_size_8192/overlap_512/no_model_name_available/no_revision_available/model_meta.json new file mode 100644 index 0000000..40ff52f --- /dev/null +++ b/results-soft-boundary/embed_size_8192/overlap_512/no_model_name_available/no_revision_available/model_meta.json @@ -0,0 +1 @@ +{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_8192/overlap_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-soft-boundary/embed_size_8192/overlap_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json new file mode 100644 index 0000000..071adf3 --- /dev/null +++ b/results-soft-boundary/embed_size_8192/overlap_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -0,0 +1,38 @@ +{ + "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", + "evaluation_time": 36.87527394294739, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.94608, + "map_at_1": 0.90667, + "map_at_10": 0.93403, + "map_at_3": 0.93, + "map_at_5": 0.93233, + "mrr_at_1": 0.9066666666666666, + "mrr_at_10": 0.9340317460317461, + "mrr_at_3": 0.93, + "mrr_at_5": 0.9323333333333333, + "ndcg_at_1": 0.90667, + "ndcg_at_10": 0.94608, + "ndcg_at_3": 0.9377, + "ndcg_at_5": 0.94186, + "precision_at_1": 0.90667, + "precision_at_10": 0.09833, + "precision_at_3": 0.32, + "precision_at_5": 0.194, + "recall_at_1": 0.90667, + "recall_at_10": 0.98333, + "recall_at_3": 0.96, + "recall_at_5": 0.97 + } + ] + }, + "task_name": "LEMBWikimQARetrievalChunked" +} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_8192/overlap_64/no_model_name_available/no_revision_available/model_meta.json b/results-soft-boundary/embed_size_8192/overlap_64/no_model_name_available/no_revision_available/model_meta.json new file mode 100644 index 0000000..40ff52f --- /dev/null +++ b/results-soft-boundary/embed_size_8192/overlap_64/no_model_name_available/no_revision_available/model_meta.json @@ -0,0 +1 @@ +{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-truncation/embed_size_8192/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-truncation/embed_size_8192/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json new file mode 100644 index 0000000..f34aedb --- /dev/null +++ b/results-truncation/embed_size_8192/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -0,0 +1,43 @@ +{ + "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", + "evaluation_time": 32.949374198913574, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.88074, + "map_at_1": 0.83667, + "map_at_10": 0.86607, + "map_at_20": 0.86737, + "map_at_3": 0.85944, + "map_at_5": 0.86378, + "mrr_at_1": 0.8366666666666667, + "mrr_at_10": 0.8660687830687831, + "mrr_at_20": 0.8673737974987974, + "mrr_at_3": 0.8594444444444443, + "mrr_at_5": 0.8637777777777778, + "ndcg_at_1": 0.83667, + "ndcg_at_10": 0.88074, + "ndcg_at_20": 0.8857, + "ndcg_at_3": 0.86726, + "ndcg_at_5": 0.87529, + "precision_at_1": 0.83667, + "precision_at_10": 0.09267, + "precision_at_20": 0.04733, + "precision_at_3": 0.29667, + "precision_at_5": 0.182, + "recall_at_1": 0.83667, + "recall_at_10": 0.92667, + "recall_at_20": 0.94667, + "recall_at_3": 0.89, + "recall_at_5": 0.91 + } + ] + }, + "task_name": "LEMBWikimQARetrievalChunked" +} \ No newline at end of file diff --git a/results-truncation/embed_size_8192/no_model_name_available/no_revision_available/model_meta.json b/results-truncation/embed_size_8192/no_model_name_available/no_revision_available/model_meta.json new file mode 100644 index 0000000..40ff52f --- /dev/null +++ b/results-truncation/embed_size_8192/no_model_name_available/no_revision_available/model_meta.json @@ -0,0 +1 @@ +{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file From f3eb17d7b11eb3e145c091ee888dc8eee050fbf0 Mon Sep 17 00:00:00 2001 From: dannyjameswilliams Date: Thu, 26 Sep 2024 13:42:09 +0100 Subject: [PATCH 07/30] experiment for WikimQA --- parse_experiments.py | 70 +++++++++++++++++++++++++++----------------- 1 file changed, 43 insertions(+), 27 deletions(-) diff --git a/parse_experiments.py b/parse_experiments.py index e7979f6..ddc4d91 100644 --- a/parse_experiments.py +++ b/parse_experiments.py @@ -12,18 +12,12 @@ - - - - - - if __name__ == "__main__": task_name = "LEMBWikimQARetrievalChunked" soft_boundary_dir = "results-soft-boundary" hard_boundary_dir = "results-hard-boundary" - truncate_dir = "results-truncation" + truncated_boundary_dir = "results-truncation" embed_sizes = [1024, 2048, 4096, 8192] overlap_sizes = [32, 64, 128, 256, 512] @@ -52,34 +46,56 @@ hard_boundary_ndcgs[embed_i] = results["scores"]["test"][0]["ndcg_at_10"] hard_boundary_maps[embed_i] = results["scores"]["test"][0]["map_at_10"] + truncation_dir = os.path.join(truncated_boundary_dir, "no_model_name_available", "no_revision_available", f"{task_name}.json") + with open(truncation_dir, "r") as f: + results = json.load(f) + + truncated_boundary_ndcg = results["scores"]["test"][0]["ndcg_at_10"] + truncated_boundary_map = results["scores"]["test"][0]["map_at_10"] # NDCG - fig, ax = plt.subplots(len(embed_sizes), 1, figsize=(12, 9)) + plt.style.use('seaborn-v0_8-whitegrid') + plt.rcParams.update({'font.size': 14, 'font.family': 'serif'}) - for embed_i, embed_size in enumerate(embed_sizes): + fig, ax = plt.subplots(1, 1, figsize=(12, 9), sharex=True) - ax[embed_i].plot(overlap_sizes, soft_boundary_ndcgs[embed_i], label="Soft Boundary") - ax[embed_i].scatter(overlap_sizes, soft_boundary_ndcgs[embed_i]) - ax[embed_i].axhline(hard_boundary_ndcgs[embed_i], color="red", linestyle="--", label="Hard Boundary") + colors = plt.cm.get_cmap('plasma', len(embed_sizes))(np.linspace(0, 1, len(embed_sizes), endpoint=False)) + colors = ["red", "blue", "green", "orange"] - ax[embed_i].set_title(f"Embed Size: {embed_size}") - ax[embed_i].set_xlabel("Overlap Size") - ax[embed_i].set_ylabel("nCDG@10") + for embed_i, embed_size in enumerate(embed_sizes[:-1]): - ax[embed_i].legend() - fig.tight_layout() + ax.plot([o - embed_i*5 for o in overlap_sizes], soft_boundary_ndcgs[embed_i], label=f"Soft Boundary ({embed_size})", color=colors[embed_i], marker='o', linestyle='-', linewidth=2, markersize=6) + ax.axhline(hard_boundary_ndcgs[embed_i], linestyle="--", label=f"Hard Boundary ({embed_size})", linewidth=2, color=colors[embed_i]) + + ax.set_title(f"Embed Size: {embed_size}", fontsize=16) + ax.set_ylabel("nCDG@10", fontsize=14) + ax.legend(loc='best', fontsize=12) + + ax.axhline(truncated_boundary_ndcg, label=f"Truncated Boundary (at 8192)", color="k", linestyle='-.', linewidth=2) + ax.set_xlabel("Overlap Size", fontsize=14) + fig.suptitle("NDCG@10 for Different Embed Sizes and Overlap Sizes", fontsize=18) + fig.tight_layout(rect=[0, 0, 1, 0.96]) # MAP - fig, ax = plt.subplots(len(embed_sizes), 1, figsize=(12, 9)) - for embed_i, embed_size in enumerate(embed_sizes): + plt.style.use('seaborn-v0_8-whitegrid') + plt.rcParams.update({'font.size': 14, 'font.family': 'serif'}) + + fig, ax = plt.subplots(1, 1, figsize=(12, 9), sharex=True) + + colors = plt.cm.get_cmap('plasma', len(embed_sizes))(np.linspace(0, 1, len(embed_sizes), endpoint=False)) + colors = ["red", "blue", "green", "orange"] + + for embed_i, embed_size in enumerate(embed_sizes[:-1]): + + ax.plot([o - embed_i*5 for o in overlap_sizes], soft_boundary_maps[embed_i], label=f"Soft Boundary ({embed_size})", color=colors[embed_i], marker='o', linestyle='-', linewidth=2, markersize=6) + ax.axhline(hard_boundary_maps[embed_i], linestyle="--", label=f"Hard Boundary ({embed_size})", linewidth=2, color=colors[embed_i]) - ax[embed_i].plot(overlap_sizes, soft_boundary_maps[embed_i], label="Soft Boundary") - ax[embed_i].scatter(overlap_sizes, soft_boundary_maps[embed_i]) - ax[embed_i].axhline(hard_boundary_maps[embed_i], color="red", linestyle="--", label="Hard Boundary") + ax.set_title(f"Embed Size: {embed_size}", fontsize=16) + ax.set_ylabel("mAP@10", fontsize=14) + ax.legend(loc='best', fontsize=12) - ax[embed_i].set_title(f"Embed Size: {embed_size}") - ax[embed_i].set_xlabel("Overlap Size") - ax[embed_i].set_ylabel("mAP@10") - ax[embed_i].legend() - fig.tight_layout() \ No newline at end of file + ax.axhline(truncated_boundary_map, label=f"Truncated Boundary (at 8192)", color="k", linestyle='-.', linewidth=2) + ax.set_xlabel("Overlap Size", fontsize=14) + fig.suptitle("mAP@10 for Different Embed Sizes and Overlap Sizes", fontsize=18) + fig.tight_layout(rect=[0, 0, 1, 0.96]) \ No newline at end of file From 50b2b00892980d85dc5406c887ae8277bc97e126 Mon Sep 17 00:00:00 2001 From: dannyjameswilliams Date: Thu, 26 Sep 2024 14:00:14 +0100 Subject: [PATCH 08/30] added narrativeQA task --- chunked_pooling/chunked_eval_tasks.py | 80 ++++++++++++++++++++++++++- 1 file changed, 79 insertions(+), 1 deletion(-) diff --git a/chunked_pooling/chunked_eval_tasks.py b/chunked_pooling/chunked_eval_tasks.py index 23dbcbf..7a127e5 100644 --- a/chunked_pooling/chunked_eval_tasks.py +++ b/chunked_pooling/chunked_eval_tasks.py @@ -228,7 +228,7 @@ class LEMBWikimQARetrievalChunked(AbsTaskChunkedRetrieval): name="LEMBWikimQARetrievalChunked", dataset={ "path": "dwzhu/LongEmbed", - "revision": "6e346642246bfb4928c560ee08640dc84d074e8c", + "revision": "10039a580487dacecf79db69166e17ace3ede392", "name": "LEMBWikimQARetrieval", }, reference="https://huggingface.co/datasets/dwzhu/LongEmbed", @@ -296,6 +296,84 @@ def load_data(self, **kwargs): self.data_loaded = True +class LEMBNarrativeQARetrievalChunked(AbsTaskChunkedRetrieval): + """ + modified from https://github.com/embeddings-benchmark/mteb/blob/main/mteb/tasks/Retrieval/eng/LEMBWikimQARetrieval.py + """ + + _EVAL_SPLIT = "test" + + metadata = TaskMetadata( + name="LEMBNarrativeQARetrievalChunked", + dataset={ + "path": "dwzhu/LongEmbed", + "revision": "6e346642246bfb4928c560ee08640dc84d074e8c", + "name": "LEMBNarrativeQARetrieval", + }, + reference="https://huggingface.co/datasets/dwzhu/LongEmbed", + description=("narrativeqa subset of dwzhu/LongEmbed dataset."), + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=[_EVAL_SPLIT], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=("1950-01-01", "2019-12-31"), + domains=None, + socioeconomic_status=None, + n_samples=None, + avg_character_length=None, + form=None, + text_creation=None, + task_subtypes=["Article retrieval"], + license="not specified", + annotations_creators="derived", + dialect=[], + sample_creation="found", + bibtex_citation=""" + @inproceedings{ho2020constructing, + title={Constructing A Multi-hop QA Dataset for Comprehensive Evaluation of Reasoning Steps}, + author={Ho, Xanh and Nguyen, Anh-Khoa Duong and Sugawara, Saku and Aizawa, Akiko}, + booktitle={Proceedings of the 28th International Conference on Computational Linguistics}, + pages={6609--6625}, + year={2020} + } + """, + descriptive_stats={ + "n_samples": {_EVAL_SPLIT: 500}, + "avg_character_length": { + "test": { + "average_document_length": 326753.5323943662, + "average_query_length": 47.89453536223562, + "num_documents": 300, + "num_queries": 300, + "average_relevant_docs_per_query": 1.0, + } + }, + }, + ) + + def load_data(self, **kwargs): + if self.data_loaded: + return + + dataset_dict = {**self.metadata.dataset} + dataset_dict['name'] = 'narrativeqa' + + query_list = datasets.load_dataset(**dataset_dict)["queries"] + queries = {row["qid"]: row["text"] for row in query_list} + + corpus_list = datasets.load_dataset(**dataset_dict)["corpus"] + corpus = {row["doc_id"]: {"text": row["text"]} for row in corpus_list} + + qrels_list = datasets.load_dataset(**dataset_dict)["qrels"] + qrels = {row["qid"]: {row["doc_id"]: 1} for row in qrels_list} + + self.corpus = {self._EVAL_SPLIT: corpus} + self.queries = {self._EVAL_SPLIT: queries} + self.relevant_docs = {self._EVAL_SPLIT: qrels} + + self.data_loaded = True class LEMBNeedleRetrievalChunked(AbsTaskChunkedRetrieval): """ From 3f226a5550274538ca6a314719351ac012faa74a Mon Sep 17 00:00:00 2001 From: dannyjameswilliams Date: Thu, 26 Sep 2024 14:10:53 +0100 Subject: [PATCH 09/30] added remainder of longembed datasets --- chunked_pooling/chunked_eval_tasks.py | 167 +++++++++++++++++++++++++- 1 file changed, 164 insertions(+), 3 deletions(-) diff --git a/chunked_pooling/chunked_eval_tasks.py b/chunked_pooling/chunked_eval_tasks.py index 7a127e5..62d41e4 100644 --- a/chunked_pooling/chunked_eval_tasks.py +++ b/chunked_pooling/chunked_eval_tasks.py @@ -296,6 +296,7 @@ def load_data(self, **kwargs): self.data_loaded = True + class LEMBNarrativeQARetrievalChunked(AbsTaskChunkedRetrieval): """ modified from https://github.com/embeddings-benchmark/mteb/blob/main/mteb/tasks/Retrieval/eng/LEMBWikimQARetrieval.py @@ -307,7 +308,7 @@ class LEMBNarrativeQARetrievalChunked(AbsTaskChunkedRetrieval): name="LEMBNarrativeQARetrievalChunked", dataset={ "path": "dwzhu/LongEmbed", - "revision": "6e346642246bfb4928c560ee08640dc84d074e8c", + "revision": "10039a580487dacecf79db69166e17ace3ede392", "name": "LEMBNarrativeQARetrieval", }, reference="https://huggingface.co/datasets/dwzhu/LongEmbed", @@ -343,8 +344,8 @@ class LEMBNarrativeQARetrievalChunked(AbsTaskChunkedRetrieval): "n_samples": {_EVAL_SPLIT: 500}, "avg_character_length": { "test": { - "average_document_length": 326753.5323943662, - "average_query_length": 47.89453536223562, + "average_document_length": 326753.53, + "average_query_length": 47.89, "num_documents": 300, "num_queries": 300, "average_relevant_docs_per_query": 1.0, @@ -375,6 +376,166 @@ def load_data(self, **kwargs): self.data_loaded = True +class LEMBSummScreenFDRetrievalChunked(AbsTaskChunkedRetrieval): + """ + modified from https://github.com/embeddings-benchmark/mteb/blob/main/mteb/tasks/Retrieval/eng/LEMBWikimQARetrieval.py + """ + + _EVAL_SPLIT = "test" + + metadata = TaskMetadata( + name="LEMBSummScreenFDRetrievalChunked", + dataset={ + "path": "dwzhu/LongEmbed", + "revision": "10039a580487dacecf79db69166e17ace3ede392", + "name": "LEMBSummScreenFDRetrieval", + }, + reference="https://huggingface.co/datasets/dwzhu/LongEmbed", + description=("summ_screen_fd subset of dwzhu/LongEmbed dataset."), + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=[_EVAL_SPLIT], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=("1950-01-01", "2019-12-31"), + domains=None, + socioeconomic_status=None, + n_samples=None, + avg_character_length=None, + form=None, + text_creation=None, + task_subtypes=["Article retrieval"], + license="not specified", + annotations_creators="derived", + dialect=[], + sample_creation="found", + bibtex_citation=""" + @inproceedings{ho2020constructing, + title={Constructing A Multi-hop QA Dataset for Comprehensive Evaluation of Reasoning Steps}, + author={Ho, Xanh and Nguyen, Anh-Khoa Duong and Sugawara, Saku and Aizawa, Akiko}, + booktitle={Proceedings of the 28th International Conference on Computational Linguistics}, + pages={6609--6625}, + year={2020} + } + """, + descriptive_stats={ + "n_samples": {_EVAL_SPLIT: 500}, + "avg_character_length": { + "test": { + "average_document_length": 30854.327, + "average_query_length": 591.49, + "num_documents": 300, + "num_queries": 300, + "average_relevant_docs_per_query": 1.0, + } + }, + }, + ) + + def load_data(self, **kwargs): + if self.data_loaded: + return + + dataset_dict = {**self.metadata.dataset} + dataset_dict['name'] = 'summ_screen_fd' + + query_list = datasets.load_dataset(**dataset_dict)["queries"] + queries = {row["qid"]: row["text"] for row in query_list} + + corpus_list = datasets.load_dataset(**dataset_dict)["corpus"] + corpus = {row["doc_id"]: {"text": row["text"]} for row in corpus_list} + + qrels_list = datasets.load_dataset(**dataset_dict)["qrels"] + qrels = {row["qid"]: {row["doc_id"]: 1} for row in qrels_list} + + self.corpus = {self._EVAL_SPLIT: corpus} + self.queries = {self._EVAL_SPLIT: queries} + self.relevant_docs = {self._EVAL_SPLIT: qrels} + + self.data_loaded = True + + +class LEMBQMSumRetrievalRetrievalChunked(AbsTaskChunkedRetrieval): + """ + modified from https://github.com/embeddings-benchmark/mteb/blob/main/mteb/tasks/Retrieval/eng/LEMBWikimQARetrieval.py + """ + + _EVAL_SPLIT = "test" + + metadata = TaskMetadata( + name="LEMBQMSumRetrievalRetrievalChunked", + dataset={ + "path": "dwzhu/LongEmbed", + "revision": "10039a580487dacecf79db69166e17ace3ede392", + "name": "LEMBQMSumRetrievalRetrieval", + }, + reference="https://huggingface.co/datasets/dwzhu/LongEmbed", + description=("qmsum subset of dwzhu/LongEmbed dataset."), + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=[_EVAL_SPLIT], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=("1950-01-01", "2019-12-31"), + domains=None, + socioeconomic_status=None, + n_samples=None, + avg_character_length=None, + form=None, + text_creation=None, + task_subtypes=["Article retrieval"], + license="not specified", + annotations_creators="derived", + dialect=[], + sample_creation="found", + bibtex_citation=""" + @inproceedings{ho2020constructing, + title={Constructing A Multi-hop QA Dataset for Comprehensive Evaluation of Reasoning Steps}, + author={Ho, Xanh and Nguyen, Anh-Khoa Duong and Sugawara, Saku and Aizawa, Akiko}, + booktitle={Proceedings of the 28th International Conference on Computational Linguistics}, + pages={6609--6625}, + year={2020} + } + """, + descriptive_stats={ + "n_samples": {_EVAL_SPLIT: 500}, + "avg_character_length": { + "test": { + "average_document_length": 53335.817, + "average_query_length": 433.50, + "num_documents": 300, + "num_queries": 300, + "average_relevant_docs_per_query": 1.0, + } + }, + }, + ) + + def load_data(self, **kwargs): + if self.data_loaded: + return + + dataset_dict = {**self.metadata.dataset} + dataset_dict['name'] = 'qmsum' + + query_list = datasets.load_dataset(**dataset_dict)["queries"] + queries = {row["qid"]: row["text"] for row in query_list} + + corpus_list = datasets.load_dataset(**dataset_dict)["corpus"] + corpus = {row["doc_id"]: {"text": row["text"]} for row in corpus_list} + + qrels_list = datasets.load_dataset(**dataset_dict)["qrels"] + qrels = {row["qid"]: {row["doc_id"]: 1} for row in qrels_list} + + self.corpus = {self._EVAL_SPLIT: corpus} + self.queries = {self._EVAL_SPLIT: queries} + self.relevant_docs = {self._EVAL_SPLIT: qrels} + + self.data_loaded = True + + class LEMBNeedleRetrievalChunked(AbsTaskChunkedRetrieval): """ modified from https://github.com/embeddings-benchmark/mteb/blob/main/mteb/tasks/Retrieval/eng/LEMBNeedleRetrieval.py From 19ab752e58aecd867265a153e5c814b2a0c256a4 Mon Sep 17 00:00:00 2001 From: dannyjameswilliams Date: Thu, 26 Sep 2024 14:15:53 +0100 Subject: [PATCH 10/30] typo --- chunked_pooling/chunked_eval_tasks.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/chunked_pooling/chunked_eval_tasks.py b/chunked_pooling/chunked_eval_tasks.py index 62d41e4..45ff1d1 100644 --- a/chunked_pooling/chunked_eval_tasks.py +++ b/chunked_pooling/chunked_eval_tasks.py @@ -456,7 +456,7 @@ def load_data(self, **kwargs): self.data_loaded = True -class LEMBQMSumRetrievalRetrievalChunked(AbsTaskChunkedRetrieval): +class LEMBQMSumRetrievalChunked(AbsTaskChunkedRetrieval): """ modified from https://github.com/embeddings-benchmark/mteb/blob/main/mteb/tasks/Retrieval/eng/LEMBWikimQARetrieval.py """ @@ -464,11 +464,11 @@ class LEMBQMSumRetrievalRetrievalChunked(AbsTaskChunkedRetrieval): _EVAL_SPLIT = "test" metadata = TaskMetadata( - name="LEMBQMSumRetrievalRetrievalChunked", + name="LEMBQMSumRetrievalChunked", dataset={ "path": "dwzhu/LongEmbed", "revision": "10039a580487dacecf79db69166e17ace3ede392", - "name": "LEMBQMSumRetrievalRetrieval", + "name": "LEMBQMSumRetrieval", }, reference="https://huggingface.co/datasets/dwzhu/LongEmbed", description=("qmsum subset of dwzhu/LongEmbed dataset."), From 57cc8bac94ccc59f7e5cfb1c2fe1083816d511ea Mon Sep 17 00:00:00 2001 From: dannyjameswilliams Date: Thu, 26 Sep 2024 16:37:32 +0100 Subject: [PATCH 11/30] more tasks for soft/hard --- parse_experiments.py | 78 +++++++++++-------- .../LEMBQMSumRetrievalChunked.json | 43 ++++++++++ .../LEMBSummScreenFDRetrievalChunked.json | 38 +++++++++ .../LEMBWikimQARetrievalChunked.json | 4 +- .../LEMBQMSumRetrievalChunked.json | 43 ++++++++++ .../LEMBSummScreenFDRetrievalChunked.json | 38 +++++++++ .../LEMBWikimQARetrievalChunked.json | 4 +- .../LEMBQMSumRetrievalChunked.json | 43 ++++++++++ .../LEMBSummScreenFDRetrievalChunked.json | 38 +++++++++ .../LEMBWikimQARetrievalChunked.json | 4 +- .../LEMBQMSumRetrievalChunked.json | 43 ++++++++++ .../LEMBSummScreenFDRetrievalChunked.json | 38 +++++++++ .../LEMBWikimQARetrievalChunked.json | 4 +- .../LEMBQMSumRetrievalChunked.json | 43 ++++++++++ .../LEMBSummScreenFDRetrievalChunked.json | 38 +++++++++ .../LEMBWikimQARetrievalChunked.json | 4 +- .../LEMBQMSumRetrievalChunked.json | 43 ++++++++++ .../LEMBSummScreenFDRetrievalChunked.json | 38 +++++++++ .../LEMBWikimQARetrievalChunked.json | 4 +- .../LEMBQMSumRetrievalChunked.json | 43 ++++++++++ .../LEMBSummScreenFDRetrievalChunked.json | 38 +++++++++ .../LEMBWikimQARetrievalChunked.json | 4 +- .../LEMBQMSumRetrievalChunked.json | 43 ++++++++++ .../LEMBSummScreenFDRetrievalChunked.json | 38 +++++++++ .../LEMBWikimQARetrievalChunked.json | 4 +- .../LEMBQMSumRetrievalChunked.json | 43 ++++++++++ .../LEMBSummScreenFDRetrievalChunked.json | 38 +++++++++ .../LEMBWikimQARetrievalChunked.json | 4 +- .../LEMBQMSumRetrievalChunked.json | 43 ++++++++++ .../LEMBSummScreenFDRetrievalChunked.json | 38 +++++++++ .../LEMBWikimQARetrievalChunked.json | 4 +- .../LEMBQMSumRetrievalChunked.json | 43 ++++++++++ .../LEMBSummScreenFDRetrievalChunked.json | 38 +++++++++ .../LEMBWikimQARetrievalChunked.json | 4 +- .../LEMBQMSumRetrievalChunked.json | 43 ++++++++++ .../LEMBSummScreenFDRetrievalChunked.json | 38 +++++++++ .../LEMBWikimQARetrievalChunked.json | 4 +- .../LEMBQMSumRetrievalChunked.json | 43 ++++++++++ .../LEMBSummScreenFDRetrievalChunked.json | 38 +++++++++ .../LEMBWikimQARetrievalChunked.json | 4 +- .../LEMBQMSumRetrievalChunked.json | 43 ++++++++++ .../LEMBSummScreenFDRetrievalChunked.json | 38 +++++++++ .../LEMBWikimQARetrievalChunked.json | 4 +- .../LEMBQMSumRetrievalChunked.json | 43 ++++++++++ .../LEMBSummScreenFDRetrievalChunked.json | 38 +++++++++ .../LEMBWikimQARetrievalChunked.json | 4 +- .../LEMBQMSumRetrievalChunked.json | 43 ++++++++++ .../LEMBSummScreenFDRetrievalChunked.json | 38 +++++++++ .../LEMBWikimQARetrievalChunked.json | 4 +- .../LEMBQMSumRetrievalChunked.json | 43 ++++++++++ .../LEMBSummScreenFDRetrievalChunked.json | 38 +++++++++ .../LEMBWikimQARetrievalChunked.json | 4 +- .../LEMBQMSumRetrievalChunked.json | 43 ++++++++++ .../LEMBSummScreenFDRetrievalChunked.json | 38 +++++++++ .../LEMBWikimQARetrievalChunked.json | 4 +- .../LEMBQMSumRetrievalChunked.json | 43 ++++++++++ .../LEMBSummScreenFDRetrievalChunked.json | 38 +++++++++ .../LEMBWikimQARetrievalChunked.json | 4 +- .../LEMBQMSumRetrievalChunked.json | 43 ++++++++++ .../no_revision_available/model_meta.json | 1 + .../LEMBQMSumRetrievalChunked.json | 43 ++++++++++ .../no_revision_available/model_meta.json | 1 + .../LEMBQMSumRetrievalChunked.json | 43 ++++++++++ .../no_revision_available/model_meta.json | 1 + .../no_revision_available/model_meta.json | 1 + .../LEMBQMSumRetrievalChunked.json | 43 ++++++++++ .../no_revision_available/model_meta.json | 1 + .../LEMBQMSumRetrievalChunked.json | 43 ++++++++++ .../LEMBSummScreenFDRetrievalChunked.json | 38 +++++++++ .../LEMBWikimQARetrievalChunked.json | 4 +- .../LEMBQMSumRetrievalChunked.json | 43 ++++++++++ .../LEMBSummScreenFDRetrievalChunked.json | 38 +++++++++ .../LEMBWikimQARetrievalChunked.json | 4 +- .../LEMBQMSumRetrievalChunked.json | 43 ++++++++++ .../LEMBSummScreenFDRetrievalChunked.json | 38 +++++++++ .../LEMBWikimQARetrievalChunked.json | 4 +- .../LEMBQMSumRetrievalChunked.json | 43 ++++++++++ .../LEMBSummScreenFDRetrievalChunked.json | 38 +++++++++ .../LEMBWikimQARetrievalChunked.json | 4 +- .../LEMBQMSumRetrievalChunked.json | 43 ++++++++++ .../LEMBSummScreenFDRetrievalChunked.json | 38 +++++++++ .../LEMBWikimQARetrievalChunked.json | 4 +- .../LEMBQMSumRetrievalChunked.json | 43 ++++++++++ .../LEMBSummScreenFDRetrievalChunked.json | 43 ++++++++++ .../LEMBWikimQARetrievalChunked.json | 4 +- 85 files changed, 2303 insertions(+), 82 deletions(-) create mode 100644 results-hard-boundary/embed_size_1024/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json create mode 100644 results-hard-boundary/embed_size_1024/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json create mode 100644 results-hard-boundary/embed_size_2048/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json create mode 100644 results-hard-boundary/embed_size_2048/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json create mode 100644 results-hard-boundary/embed_size_4096/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json create mode 100644 results-hard-boundary/embed_size_4096/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json create mode 100644 results-hard-boundary/embed_size_8192/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json create mode 100644 results-hard-boundary/embed_size_8192/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json create mode 100644 results-soft-boundary/embed_size_1024/overlap_128/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json create mode 100644 results-soft-boundary/embed_size_1024/overlap_128/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json create mode 100644 results-soft-boundary/embed_size_1024/overlap_256/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json create mode 100644 results-soft-boundary/embed_size_1024/overlap_256/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json create mode 100644 results-soft-boundary/embed_size_1024/overlap_32/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json create mode 100644 results-soft-boundary/embed_size_1024/overlap_32/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json create mode 100644 results-soft-boundary/embed_size_1024/overlap_512/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json create mode 100644 results-soft-boundary/embed_size_1024/overlap_512/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json create mode 100644 results-soft-boundary/embed_size_1024/overlap_64/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json create mode 100644 results-soft-boundary/embed_size_1024/overlap_64/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json create mode 100644 results-soft-boundary/embed_size_2048/overlap_128/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json create mode 100644 results-soft-boundary/embed_size_2048/overlap_128/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json create mode 100644 results-soft-boundary/embed_size_2048/overlap_256/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json create mode 100644 results-soft-boundary/embed_size_2048/overlap_256/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json create mode 100644 results-soft-boundary/embed_size_2048/overlap_32/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json create mode 100644 results-soft-boundary/embed_size_2048/overlap_32/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json create mode 100644 results-soft-boundary/embed_size_2048/overlap_512/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json create mode 100644 results-soft-boundary/embed_size_2048/overlap_512/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json create mode 100644 results-soft-boundary/embed_size_2048/overlap_64/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json create mode 100644 results-soft-boundary/embed_size_2048/overlap_64/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json create mode 100644 results-soft-boundary/embed_size_4096/overlap_128/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json create mode 100644 results-soft-boundary/embed_size_4096/overlap_128/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json create mode 100644 results-soft-boundary/embed_size_4096/overlap_256/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json create mode 100644 results-soft-boundary/embed_size_4096/overlap_256/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json create mode 100644 results-soft-boundary/embed_size_4096/overlap_32/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json create mode 100644 results-soft-boundary/embed_size_4096/overlap_32/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json create mode 100644 results-soft-boundary/embed_size_4096/overlap_512/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json create mode 100644 results-soft-boundary/embed_size_4096/overlap_512/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json create mode 100644 results-soft-boundary/embed_size_4096/overlap_64/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json create mode 100644 results-soft-boundary/embed_size_4096/overlap_64/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json create mode 100644 results-soft-boundary/embed_size_512/overlap_128/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json create mode 100644 results-soft-boundary/embed_size_512/overlap_128/no_model_name_available/no_revision_available/model_meta.json create mode 100644 results-soft-boundary/embed_size_512/overlap_256/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json create mode 100644 results-soft-boundary/embed_size_512/overlap_256/no_model_name_available/no_revision_available/model_meta.json create mode 100644 results-soft-boundary/embed_size_512/overlap_32/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json create mode 100644 results-soft-boundary/embed_size_512/overlap_32/no_model_name_available/no_revision_available/model_meta.json create mode 100644 results-soft-boundary/embed_size_512/overlap_512/no_model_name_available/no_revision_available/model_meta.json create mode 100644 results-soft-boundary/embed_size_512/overlap_64/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json create mode 100644 results-soft-boundary/embed_size_512/overlap_64/no_model_name_available/no_revision_available/model_meta.json create mode 100644 results-soft-boundary/embed_size_8192/overlap_128/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json create mode 100644 results-soft-boundary/embed_size_8192/overlap_128/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json create mode 100644 results-soft-boundary/embed_size_8192/overlap_256/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json create mode 100644 results-soft-boundary/embed_size_8192/overlap_256/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json create mode 100644 results-soft-boundary/embed_size_8192/overlap_32/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json create mode 100644 results-soft-boundary/embed_size_8192/overlap_32/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json create mode 100644 results-soft-boundary/embed_size_8192/overlap_512/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json create mode 100644 results-soft-boundary/embed_size_8192/overlap_512/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json create mode 100644 results-soft-boundary/embed_size_8192/overlap_64/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json create mode 100644 results-soft-boundary/embed_size_8192/overlap_64/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json create mode 100644 results-truncation/embed_size_8192/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json create mode 100644 results-truncation/embed_size_8192/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json diff --git a/parse_experiments.py b/parse_experiments.py index ddc4d91..02a793e 100644 --- a/parse_experiments.py +++ b/parse_experiments.py @@ -19,60 +19,74 @@ hard_boundary_dir = "results-hard-boundary" truncated_boundary_dir = "results-truncation" + # == Load results + embed_sizes = [1024, 2048, 4096, 8192] overlap_sizes = [32, 64, 128, 256, 512] + task_names = ["QMSum", "SummScreenFD", "WikimQA"] - soft_boundary_ndcgs = np.empty((len(embed_sizes), len(overlap_sizes))) - hard_boundary_ndcgs = np.empty(len(embed_sizes)) + soft_boundary_ndcgs = np.empty((len(task_names), len(embed_sizes), len(overlap_sizes))) + hard_boundary_ndcgs = np.empty((len(task_names), len(embed_sizes))) - soft_boundary_maps = np.empty((len(embed_sizes), len(overlap_sizes))) - hard_boundary_maps = np.empty(len(embed_sizes)) + soft_boundary_maps = np.empty((len(task_names), len(embed_sizes), len(overlap_sizes))) + hard_boundary_maps = np.empty((len(task_names), len(embed_sizes))) - for embed_i, embed_size in enumerate(embed_sizes): + for task_i, task_name in enumerate(task_names): - for overlap_i, overlap_size in enumerate(overlap_sizes): - - soft_dir = os.path.join(soft_boundary_dir, f"embed_size_{embed_size}", f"overlap_{overlap_size}", "no_model_name_available", "no_revision_available", f"{task_name}.json") + task_name = f"LEMB{task_name}RetrievalChunked" + + for embed_i, embed_size in enumerate(embed_sizes): + + for overlap_i, overlap_size in enumerate(overlap_sizes): + + soft_dir = os.path.join(soft_boundary_dir, f"embed_size_{embed_size}", f"overlap_{overlap_size}", "no_model_name_available", "no_revision_available", f"{task_name}.json") + with open(soft_dir, "r") as f: + results = json.load(f) + + soft_boundary_ndcgs[task_i, embed_i, overlap_i] = results["scores"]["test"][0]["ndcg_at_10"] + soft_boundary_maps[task_i, embed_i, overlap_i] = results["scores"]["test"][0]["map_at_10"] + + soft_dir = os.path.join(hard_boundary_dir, f"embed_size_{embed_size}", "no_model_name_available", "no_revision_available", f"{task_name}.json") with open(soft_dir, "r") as f: results = json.load(f) + + hard_boundary_ndcgs[task_i, embed_i] = results["scores"]["test"][0]["ndcg_at_10"] + hard_boundary_maps[task_i, embed_i] = results["scores"]["test"][0]["map_at_10"] - soft_boundary_ndcgs[embed_i, overlap_i] = results["scores"]["test"][0]["ndcg_at_10"] - soft_boundary_maps[embed_i, overlap_i] = results["scores"]["test"][0]["map_at_10"] + # truncation_dir = os.path.join(truncated_boundary_dir, "no_model_name_available", "no_revision_available", f"{task_name}.json") + # with open(truncation_dir, "r") as f: + # results = json.load(f) + + # truncated_boundary_ndcg = results["scores"]["test"][0]["ndcg_at_10"] + # truncated_boundary_map = results["scores"]["test"][0]["map_at_10"] - soft_dir = os.path.join(hard_boundary_dir, f"embed_size_{embed_size}", "no_model_name_available", "no_revision_available", f"{task_name}.json") - with open(soft_dir, "r") as f: - results = json.load(f) - - hard_boundary_ndcgs[embed_i] = results["scores"]["test"][0]["ndcg_at_10"] - hard_boundary_maps[embed_i] = results["scores"]["test"][0]["map_at_10"] - truncation_dir = os.path.join(truncated_boundary_dir, "no_model_name_available", "no_revision_available", f"{task_name}.json") - with open(truncation_dir, "r") as f: - results = json.load(f) - - truncated_boundary_ndcg = results["scores"]["test"][0]["ndcg_at_10"] - truncated_boundary_map = results["scores"]["test"][0]["map_at_10"] + # == Plot (by task separately) # NDCG plt.style.use('seaborn-v0_8-whitegrid') plt.rcParams.update({'font.size': 14, 'font.family': 'serif'}) - fig, ax = plt.subplots(1, 1, figsize=(12, 9), sharex=True) + fig, ax = plt.subplots(len(task_names), 1, figsize=(12, 3*len(task_names)), sharex=True) - colors = plt.cm.get_cmap('plasma', len(embed_sizes))(np.linspace(0, 1, len(embed_sizes), endpoint=False)) + # colors = plt.cm.get_cmap('plasma', len(embed_sizes))(np.linspace(0, 1, len(embed_sizes), endpoint=False)) colors = ["red", "blue", "green", "orange"] - for embed_i, embed_size in enumerate(embed_sizes[:-1]): + for task_i in range(len(task_names)): + for embed_i, embed_size in enumerate(embed_sizes[:-1]): - ax.plot([o - embed_i*5 for o in overlap_sizes], soft_boundary_ndcgs[embed_i], label=f"Soft Boundary ({embed_size})", color=colors[embed_i], marker='o', linestyle='-', linewidth=2, markersize=6) - ax.axhline(hard_boundary_ndcgs[embed_i], linestyle="--", label=f"Hard Boundary ({embed_size})", linewidth=2, color=colors[embed_i]) + ax[task_i].plot([o - embed_i*5 for o in overlap_sizes], soft_boundary_ndcgs[task_i, embed_i], label=f"Soft Boundary ({embed_size})", color=colors[embed_i], marker='o', linestyle='-', linewidth=2, markersize=6) + ax[task_i].axhline(hard_boundary_ndcgs[task_i, embed_i], linestyle="--", label=f"Hard Boundary ({embed_size})", linewidth=2, color=colors[embed_i]) - ax.set_title(f"Embed Size: {embed_size}", fontsize=16) - ax.set_ylabel("nCDG@10", fontsize=14) - ax.legend(loc='best', fontsize=12) + ax[task_i].set_title(f"Embed Size: {embed_size}", fontsize=16) + ax[task_i].set_ylabel("nCDG@10", fontsize=14) + ax[task_i].legend(loc='best', fontsize=12) + + ax[task_i].set_title("Task: " + task_names[task_i], fontsize=16) + + # ax.axhline(truncated_boundary_ndcg, label=f"Truncated Boundary (at 8192)", color="k", linestyle='-.', linewidth=2) + ax[task_i].set_xlabel("Overlap Size", fontsize=14) - ax.axhline(truncated_boundary_ndcg, label=f"Truncated Boundary (at 8192)", color="k", linestyle='-.', linewidth=2) - ax.set_xlabel("Overlap Size", fontsize=14) fig.suptitle("NDCG@10 for Different Embed Sizes and Overlap Sizes", fontsize=18) fig.tight_layout(rect=[0, 0, 1, 0.96]) diff --git a/results-hard-boundary/embed_size_1024/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json b/results-hard-boundary/embed_size_1024/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json new file mode 100644 index 0000000..7ecc33f --- /dev/null +++ b/results-hard-boundary/embed_size_1024/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json @@ -0,0 +1,43 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 34.07488179206848, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.56988, + "map_at_1": 0.42698, + "map_at_10": 0.52032, + "map_at_20": 0.52581, + "map_at_3": 0.49356, + "map_at_5": 0.50839, + "mrr_at_1": 0.426981008513425, + "mrr_at_10": 0.520315277387969, + "mrr_at_20": 0.5258095925001836, + "mrr_at_3": 0.49356035800043657, + "mrr_at_5": 0.5083933638943462, + "ndcg_at_1": 0.42698, + "ndcg_at_10": 0.56988, + "ndcg_at_20": 0.59002, + "ndcg_at_3": 0.51484, + "ndcg_at_5": 0.54142, + "precision_at_1": 0.42698, + "precision_at_10": 0.07276, + "precision_at_20": 0.04037, + "precision_at_3": 0.1921, + "precision_at_5": 0.12809, + "recall_at_1": 0.42698, + "recall_at_10": 0.72757, + "recall_at_20": 0.80747, + "recall_at_3": 0.57629, + "recall_at_5": 0.64047 + } + ] + }, + "task_name": "LEMBQMSumRetrievalChunked" +} \ No newline at end of file diff --git a/results-hard-boundary/embed_size_1024/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-hard-boundary/embed_size_1024/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json new file mode 100644 index 0000000..3b7fadd --- /dev/null +++ b/results-hard-boundary/embed_size_1024/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json @@ -0,0 +1,38 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 16.640787363052368, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.87932, + "map_at_1": 0.77083, + "map_at_10": 0.84754, + "map_at_3": 0.83631, + "map_at_5": 0.84375, + "mrr_at_1": 0.7708333333333334, + "mrr_at_10": 0.847535194633409, + "mrr_at_3": 0.8363095238095238, + "mrr_at_5": 0.84375, + "ndcg_at_1": 0.77083, + "ndcg_at_10": 0.87932, + "ndcg_at_3": 0.857, + "ndcg_at_5": 0.87045, + "precision_at_1": 0.77083, + "precision_at_10": 0.09762, + "precision_at_3": 0.30556, + "precision_at_5": 0.18988, + "recall_at_1": 0.77083, + "recall_at_10": 0.97619, + "recall_at_3": 0.91667, + "recall_at_5": 0.9494 + } + ] + }, + "task_name": "LEMBSummScreenFDRetrievalChunked" +} \ No newline at end of file diff --git a/results-hard-boundary/embed_size_1024/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-hard-boundary/embed_size_1024/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json index 51eb6ed..b8b7910 100644 --- a/results-hard-boundary/embed_size_1024/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ b/results-hard-boundary/embed_size_1024/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -1,6 +1,6 @@ { - "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", - "evaluation_time": 14.709291696548462, + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 15.625593185424805, "kg_co2_emissions": null, "mteb_version": "1.14.20", "scores": { diff --git a/results-hard-boundary/embed_size_2048/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json b/results-hard-boundary/embed_size_2048/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json new file mode 100644 index 0000000..c75facf --- /dev/null +++ b/results-hard-boundary/embed_size_2048/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json @@ -0,0 +1,43 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 38.28924012184143, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.57083, + "map_at_1": 0.42895, + "map_at_10": 0.52134, + "map_at_20": 0.52685, + "map_at_3": 0.49552, + "map_at_5": 0.5101, + "mrr_at_1": 0.4289456450556647, + "mrr_at_10": 0.5213446325921768, + "mrr_at_20": 0.5268485590590946, + "mrr_at_3": 0.49552499454267623, + "mrr_at_5": 0.5100960488976206, + "ndcg_at_1": 0.42895, + "ndcg_at_10": 0.57083, + "ndcg_at_20": 0.59108, + "ndcg_at_3": 0.51743, + "ndcg_at_5": 0.54378, + "precision_at_1": 0.42895, + "precision_at_10": 0.07282, + "precision_at_20": 0.04044, + "precision_at_3": 0.19363, + "precision_at_5": 0.12901, + "recall_at_1": 0.42895, + "recall_at_10": 0.72823, + "recall_at_20": 0.80878, + "recall_at_3": 0.58088, + "recall_at_5": 0.64506 + } + ] + }, + "task_name": "LEMBQMSumRetrievalChunked" +} \ No newline at end of file diff --git a/results-hard-boundary/embed_size_2048/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-hard-boundary/embed_size_2048/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json new file mode 100644 index 0000000..0653cf7 --- /dev/null +++ b/results-hard-boundary/embed_size_2048/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json @@ -0,0 +1,38 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 21.060544967651367, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.89083, + "map_at_1": 0.79762, + "map_at_10": 0.86375, + "map_at_3": 0.85417, + "map_at_5": 0.86027, + "mrr_at_1": 0.7976190476190477, + "mrr_at_10": 0.8637518896447467, + "mrr_at_3": 0.8541666666666666, + "mrr_at_5": 0.8602678571428571, + "ndcg_at_1": 0.79762, + "ndcg_at_10": 0.89083, + "ndcg_at_3": 0.87181, + "ndcg_at_5": 0.88282, + "precision_at_1": 0.79762, + "precision_at_10": 0.09732, + "precision_at_3": 0.30754, + "precision_at_5": 0.18988, + "recall_at_1": 0.79762, + "recall_at_10": 0.97321, + "recall_at_3": 0.92262, + "recall_at_5": 0.9494 + } + ] + }, + "task_name": "LEMBSummScreenFDRetrievalChunked" +} \ No newline at end of file diff --git a/results-hard-boundary/embed_size_2048/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-hard-boundary/embed_size_2048/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json index 4d319ce..7f50882 100644 --- a/results-hard-boundary/embed_size_2048/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ b/results-hard-boundary/embed_size_2048/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -1,6 +1,6 @@ { - "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", - "evaluation_time": 19.393415451049805, + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 19.39264702796936, "kg_co2_emissions": null, "mteb_version": "1.14.20", "scores": { diff --git a/results-hard-boundary/embed_size_4096/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json b/results-hard-boundary/embed_size_4096/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json new file mode 100644 index 0000000..36e918f --- /dev/null +++ b/results-hard-boundary/embed_size_4096/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json @@ -0,0 +1,43 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 45.90134239196777, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.571, + "map_at_1": 0.42829, + "map_at_10": 0.52151, + "map_at_20": 0.52705, + "map_at_3": 0.49596, + "map_at_5": 0.5103, + "mrr_at_1": 0.42829076620825146, + "mrr_at_10": 0.5215057535784451, + "mrr_at_20": 0.5270534163370235, + "mrr_at_3": 0.4959615804409518, + "mrr_at_5": 0.5103034271993014, + "ndcg_at_1": 0.42829, + "ndcg_at_10": 0.571, + "ndcg_at_20": 0.59141, + "ndcg_at_3": 0.51826, + "ndcg_at_5": 0.54413, + "precision_at_1": 0.42829, + "precision_at_10": 0.07282, + "precision_at_20": 0.04047, + "precision_at_3": 0.19428, + "precision_at_5": 0.12914, + "recall_at_1": 0.42829, + "recall_at_10": 0.72823, + "recall_at_20": 0.80943, + "recall_at_3": 0.58284, + "recall_at_5": 0.64571 + } + ] + }, + "task_name": "LEMBQMSumRetrievalChunked" +} \ No newline at end of file diff --git a/results-hard-boundary/embed_size_4096/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-hard-boundary/embed_size_4096/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json new file mode 100644 index 0000000..9506068 --- /dev/null +++ b/results-hard-boundary/embed_size_4096/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json @@ -0,0 +1,38 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 28.929830312728882, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.888, + "map_at_1": 0.78869, + "map_at_10": 0.85985, + "map_at_3": 0.85069, + "map_at_5": 0.85694, + "mrr_at_1": 0.7886904761904762, + "mrr_at_10": 0.8598462301587302, + "mrr_at_3": 0.8506944444444444, + "mrr_at_5": 0.8569444444444445, + "ndcg_at_1": 0.78869, + "ndcg_at_10": 0.888, + "ndcg_at_3": 0.87, + "ndcg_at_5": 0.88115, + "precision_at_1": 0.78869, + "precision_at_10": 0.09732, + "precision_at_3": 0.30853, + "precision_at_5": 0.19048, + "recall_at_1": 0.78869, + "recall_at_10": 0.97321, + "recall_at_3": 0.9256, + "recall_at_5": 0.95238 + } + ] + }, + "task_name": "LEMBSummScreenFDRetrievalChunked" +} \ No newline at end of file diff --git a/results-hard-boundary/embed_size_4096/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-hard-boundary/embed_size_4096/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json index 4335934..aa1f560 100644 --- a/results-hard-boundary/embed_size_4096/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ b/results-hard-boundary/embed_size_4096/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -1,6 +1,6 @@ { - "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", - "evaluation_time": 26.35363245010376, + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 26.394963026046753, "kg_co2_emissions": null, "mteb_version": "1.14.20", "scores": { diff --git a/results-hard-boundary/embed_size_8192/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json b/results-hard-boundary/embed_size_8192/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json new file mode 100644 index 0000000..6ab0b4c --- /dev/null +++ b/results-hard-boundary/embed_size_8192/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json @@ -0,0 +1,43 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 59.748815298080444, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.57042, + "map_at_1": 0.42698, + "map_at_10": 0.52075, + "map_at_20": 0.52611, + "map_at_3": 0.49509, + "map_at_5": 0.50989, + "mrr_at_1": 0.426981008513425, + "mrr_at_10": 0.5207518632862445, + "mrr_at_20": 0.526107118718291, + "mrr_at_3": 0.4950884086444008, + "mrr_at_5": 0.5098886705959398, + "ndcg_at_1": 0.42698, + "ndcg_at_10": 0.57042, + "ndcg_at_20": 0.59027, + "ndcg_at_3": 0.51745, + "ndcg_at_5": 0.54413, + "precision_at_1": 0.42698, + "precision_at_10": 0.07282, + "precision_at_20": 0.04037, + "precision_at_3": 0.19406, + "precision_at_5": 0.1294, + "recall_at_1": 0.42698, + "recall_at_10": 0.72823, + "recall_at_20": 0.80747, + "recall_at_3": 0.58219, + "recall_at_5": 0.64702 + } + ] + }, + "task_name": "LEMBQMSumRetrievalChunked" +} \ No newline at end of file diff --git a/results-hard-boundary/embed_size_8192/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-hard-boundary/embed_size_8192/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json new file mode 100644 index 0000000..69b97ae --- /dev/null +++ b/results-hard-boundary/embed_size_8192/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json @@ -0,0 +1,38 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 41.61167931556702, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.88999, + "map_at_1": 0.79464, + "map_at_10": 0.86334, + "map_at_3": 0.85516, + "map_at_5": 0.86081, + "mrr_at_1": 0.7946428571428571, + "mrr_at_10": 0.863343253968254, + "mrr_at_3": 0.8551587301587301, + "mrr_at_5": 0.8608134920634921, + "ndcg_at_1": 0.79464, + "ndcg_at_10": 0.88999, + "ndcg_at_3": 0.87408, + "ndcg_at_5": 0.88407, + "precision_at_1": 0.79464, + "precision_at_10": 0.09702, + "precision_at_3": 0.30952, + "precision_at_5": 0.19048, + "recall_at_1": 0.79464, + "recall_at_10": 0.97024, + "recall_at_3": 0.92857, + "recall_at_5": 0.95238 + } + ] + }, + "task_name": "LEMBSummScreenFDRetrievalChunked" +} \ No newline at end of file diff --git a/results-hard-boundary/embed_size_8192/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-hard-boundary/embed_size_8192/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json index 78a2879..fa8bab3 100644 --- a/results-hard-boundary/embed_size_8192/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ b/results-hard-boundary/embed_size_8192/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -1,6 +1,6 @@ { - "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", - "evaluation_time": 37.40705418586731, + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 37.43858456611633, "kg_co2_emissions": null, "mteb_version": "1.14.20", "scores": { diff --git a/results-soft-boundary/embed_size_1024/overlap_128/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json b/results-soft-boundary/embed_size_1024/overlap_128/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json new file mode 100644 index 0000000..a7363fe --- /dev/null +++ b/results-soft-boundary/embed_size_1024/overlap_128/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json @@ -0,0 +1,43 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 35.60650682449341, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.56849, + "map_at_1": 0.42829, + "map_at_10": 0.51994, + "map_at_20": 0.52575, + "map_at_3": 0.4952, + "map_at_5": 0.50849, + "mrr_at_1": 0.42829076620825146, + "mrr_at_10": 0.5199366430702383, + "mrr_at_20": 0.5257458272212815, + "mrr_at_3": 0.4951975551189696, + "mrr_at_5": 0.5084915957214582, + "ndcg_at_1": 0.42829, + "ndcg_at_10": 0.56849, + "ndcg_at_20": 0.58968, + "ndcg_at_3": 0.51703, + "ndcg_at_5": 0.54088, + "precision_at_1": 0.42829, + "precision_at_10": 0.0723, + "precision_at_20": 0.04034, + "precision_at_3": 0.19341, + "precision_at_5": 0.12757, + "recall_at_1": 0.42829, + "recall_at_10": 0.72299, + "recall_at_20": 0.80681, + "recall_at_3": 0.58022, + "recall_at_5": 0.63785 + } + ] + }, + "task_name": "LEMBQMSumRetrievalChunked" +} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_1024/overlap_128/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-soft-boundary/embed_size_1024/overlap_128/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json new file mode 100644 index 0000000..9984c8c --- /dev/null +++ b/results-soft-boundary/embed_size_1024/overlap_128/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json @@ -0,0 +1,38 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 17.512223482131958, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.88919, + "map_at_1": 0.78869, + "map_at_10": 0.86056, + "map_at_3": 0.85268, + "map_at_5": 0.85699, + "mrr_at_1": 0.7886904761904762, + "mrr_at_10": 0.860557208994709, + "mrr_at_3": 0.8526785714285714, + "mrr_at_5": 0.8569940476190476, + "ndcg_at_1": 0.78869, + "ndcg_at_10": 0.88919, + "ndcg_at_3": 0.87298, + "ndcg_at_5": 0.88054, + "precision_at_1": 0.78869, + "precision_at_10": 0.09762, + "precision_at_3": 0.31052, + "precision_at_5": 0.18988, + "recall_at_1": 0.78869, + "recall_at_10": 0.97619, + "recall_at_3": 0.93155, + "recall_at_5": 0.9494 + } + ] + }, + "task_name": "LEMBSummScreenFDRetrievalChunked" +} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_1024/overlap_128/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-soft-boundary/embed_size_1024/overlap_128/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json index b812dc9..a3bfb03 100644 --- a/results-soft-boundary/embed_size_1024/overlap_128/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ b/results-soft-boundary/embed_size_1024/overlap_128/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -1,6 +1,6 @@ { - "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", - "evaluation_time": 15.548282861709595, + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 16.536237955093384, "kg_co2_emissions": null, "mteb_version": "1.14.20", "scores": { diff --git a/results-soft-boundary/embed_size_1024/overlap_256/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json b/results-soft-boundary/embed_size_1024/overlap_256/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json new file mode 100644 index 0000000..0a722a2 --- /dev/null +++ b/results-soft-boundary/embed_size_1024/overlap_256/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json @@ -0,0 +1,43 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 36.81988477706909, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.57045, + "map_at_1": 0.42764, + "map_at_10": 0.52085, + "map_at_20": 0.5265, + "map_at_3": 0.49574, + "map_at_5": 0.50986, + "mrr_at_1": 0.42763588736083824, + "mrr_at_10": 0.5208480161328884, + "mrr_at_20": 0.5264964381197267, + "mrr_at_3": 0.495743287491814, + "mrr_at_5": 0.509855926653569, + "ndcg_at_1": 0.42764, + "ndcg_at_10": 0.57045, + "ndcg_at_20": 0.591, + "ndcg_at_3": 0.51826, + "ndcg_at_5": 0.54379, + "precision_at_1": 0.42764, + "precision_at_10": 0.07282, + "precision_at_20": 0.04047, + "precision_at_3": 0.1945, + "precision_at_5": 0.12914, + "recall_at_1": 0.42764, + "recall_at_10": 0.72823, + "recall_at_20": 0.80943, + "recall_at_3": 0.5835, + "recall_at_5": 0.64571 + } + ] + }, + "task_name": "LEMBQMSumRetrievalChunked" +} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_1024/overlap_256/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-soft-boundary/embed_size_1024/overlap_256/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json new file mode 100644 index 0000000..bccd972 --- /dev/null +++ b/results-soft-boundary/embed_size_1024/overlap_256/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json @@ -0,0 +1,38 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 19.22271418571472, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.88196, + "map_at_1": 0.78274, + "map_at_10": 0.85208, + "map_at_3": 0.84276, + "map_at_5": 0.84826, + "mrr_at_1": 0.7827380952380952, + "mrr_at_10": 0.8520833333333334, + "mrr_at_3": 0.8427579365079365, + "mrr_at_5": 0.8482638888888888, + "ndcg_at_1": 0.78274, + "ndcg_at_10": 0.88196, + "ndcg_at_3": 0.8632, + "ndcg_at_5": 0.87306, + "precision_at_1": 0.78274, + "precision_at_10": 0.09732, + "precision_at_3": 0.30754, + "precision_at_5": 0.18929, + "recall_at_1": 0.78274, + "recall_at_10": 0.97321, + "recall_at_3": 0.92262, + "recall_at_5": 0.94643 + } + ] + }, + "task_name": "LEMBSummScreenFDRetrievalChunked" +} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_1024/overlap_256/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-soft-boundary/embed_size_1024/overlap_256/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json index 674accc..9de7121 100644 --- a/results-soft-boundary/embed_size_1024/overlap_256/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ b/results-soft-boundary/embed_size_1024/overlap_256/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -1,6 +1,6 @@ { - "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", - "evaluation_time": 16.896415948867798, + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 18.07439875602722, "kg_co2_emissions": null, "mteb_version": "1.14.20", "scores": { diff --git a/results-soft-boundary/embed_size_1024/overlap_32/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json b/results-soft-boundary/embed_size_1024/overlap_32/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json new file mode 100644 index 0000000..5f77527 --- /dev/null +++ b/results-soft-boundary/embed_size_1024/overlap_32/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json @@ -0,0 +1,43 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 34.675989389419556, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.57228, + "map_at_1": 0.43484, + "map_at_10": 0.52436, + "map_at_20": 0.5299, + "map_at_3": 0.49793, + "map_at_5": 0.51325, + "mrr_at_1": 0.4348395546823838, + "mrr_at_10": 0.5243643517219155, + "mrr_at_20": 0.5298984772829408, + "mrr_at_3": 0.49792621698319145, + "mrr_at_5": 0.513250382012661, + "ndcg_at_1": 0.43484, + "ndcg_at_10": 0.57228, + "ndcg_at_20": 0.59247, + "ndcg_at_3": 0.51793, + "ndcg_at_5": 0.54549, + "precision_at_1": 0.43484, + "precision_at_10": 0.0725, + "precision_at_20": 0.04024, + "precision_at_3": 0.19188, + "precision_at_5": 0.12849, + "recall_at_1": 0.43484, + "recall_at_10": 0.72495, + "recall_at_20": 0.80485, + "recall_at_3": 0.57564, + "recall_at_5": 0.64244 + } + ] + }, + "task_name": "LEMBQMSumRetrievalChunked" +} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_1024/overlap_32/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-soft-boundary/embed_size_1024/overlap_32/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json new file mode 100644 index 0000000..656231d --- /dev/null +++ b/results-soft-boundary/embed_size_1024/overlap_32/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json @@ -0,0 +1,38 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 16.89709448814392, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.88061, + "map_at_1": 0.78274, + "map_at_10": 0.85128, + "map_at_3": 0.83879, + "map_at_5": 0.84846, + "mrr_at_1": 0.7827380952380952, + "mrr_at_10": 0.8512849584278156, + "mrr_at_3": 0.8387896825396824, + "mrr_at_5": 0.8484623015873015, + "ndcg_at_1": 0.78274, + "ndcg_at_10": 0.88061, + "ndcg_at_3": 0.85654, + "ndcg_at_5": 0.87383, + "precision_at_1": 0.78274, + "precision_at_10": 0.09702, + "precision_at_3": 0.30258, + "precision_at_5": 0.18988, + "recall_at_1": 0.78274, + "recall_at_10": 0.97024, + "recall_at_3": 0.90774, + "recall_at_5": 0.9494 + } + ] + }, + "task_name": "LEMBSummScreenFDRetrievalChunked" +} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_1024/overlap_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-soft-boundary/embed_size_1024/overlap_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json index dc09724..2a9e0d2 100644 --- a/results-soft-boundary/embed_size_1024/overlap_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ b/results-soft-boundary/embed_size_1024/overlap_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -1,6 +1,6 @@ { - "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", - "evaluation_time": 14.820404767990112, + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 15.907702445983887, "kg_co2_emissions": null, "mteb_version": "1.14.20", "scores": { diff --git a/results-soft-boundary/embed_size_1024/overlap_512/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json b/results-soft-boundary/embed_size_1024/overlap_512/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json new file mode 100644 index 0000000..6aff1db --- /dev/null +++ b/results-soft-boundary/embed_size_1024/overlap_512/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json @@ -0,0 +1,43 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 42.45911240577698, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.56841, + "map_at_1": 0.43091, + "map_at_10": 0.51941, + "map_at_20": 0.5254, + "map_at_3": 0.4928, + "map_at_5": 0.5076, + "mrr_at_1": 0.4309102815979044, + "mrr_at_10": 0.5194078024136963, + "mrr_at_20": 0.5253965671643295, + "mrr_at_3": 0.49279633267845446, + "mrr_at_5": 0.5075965946299935, + "ndcg_at_1": 0.43091, + "ndcg_at_10": 0.56841, + "ndcg_at_20": 0.58993, + "ndcg_at_3": 0.51337, + "ndcg_at_5": 0.54006, + "precision_at_1": 0.43091, + "precision_at_10": 0.0725, + "precision_at_20": 0.04047, + "precision_at_3": 0.19101, + "precision_at_5": 0.12757, + "recall_at_1": 0.43091, + "recall_at_10": 0.72495, + "recall_at_20": 0.80943, + "recall_at_3": 0.57302, + "recall_at_5": 0.63785 + } + ] + }, + "task_name": "LEMBQMSumRetrievalChunked" +} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_1024/overlap_512/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-soft-boundary/embed_size_1024/overlap_512/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json new file mode 100644 index 0000000..32c7514 --- /dev/null +++ b/results-soft-boundary/embed_size_1024/overlap_512/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json @@ -0,0 +1,38 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 24.504801988601685, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.8796, + "map_at_1": 0.78274, + "map_at_10": 0.85008, + "map_at_3": 0.83879, + "map_at_5": 0.84697, + "mrr_at_1": 0.7827380952380952, + "mrr_at_10": 0.8500767668178382, + "mrr_at_3": 0.8387896825396824, + "mrr_at_5": 0.8469742063492063, + "ndcg_at_1": 0.78274, + "ndcg_at_10": 0.8796, + "ndcg_at_3": 0.85725, + "ndcg_at_5": 0.87198, + "precision_at_1": 0.78274, + "precision_at_10": 0.09702, + "precision_at_3": 0.30357, + "precision_at_5": 0.18929, + "recall_at_1": 0.78274, + "recall_at_10": 0.97024, + "recall_at_3": 0.91071, + "recall_at_5": 0.94643 + } + ] + }, + "task_name": "LEMBSummScreenFDRetrievalChunked" +} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_1024/overlap_512/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-soft-boundary/embed_size_1024/overlap_512/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json index 5f6d37a..77b9950 100644 --- a/results-soft-boundary/embed_size_1024/overlap_512/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ b/results-soft-boundary/embed_size_1024/overlap_512/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -1,6 +1,6 @@ { - "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", - "evaluation_time": 21.096606016159058, + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 22.70527148246765, "kg_co2_emissions": null, "mteb_version": "1.14.20", "scores": { diff --git a/results-soft-boundary/embed_size_1024/overlap_64/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json b/results-soft-boundary/embed_size_1024/overlap_64/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json new file mode 100644 index 0000000..ad3d20f --- /dev/null +++ b/results-soft-boundary/embed_size_1024/overlap_64/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json @@ -0,0 +1,43 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 34.64872407913208, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.56738, + "map_at_1": 0.42764, + "map_at_10": 0.51932, + "map_at_20": 0.52511, + "map_at_3": 0.49432, + "map_at_5": 0.5087, + "mrr_at_1": 0.42763588736083824, + "mrr_at_10": 0.5193168470182223, + "mrr_at_20": 0.5251082902831308, + "mrr_at_3": 0.49432438332241874, + "mrr_at_5": 0.508698974023139, + "ndcg_at_1": 0.42764, + "ndcg_at_10": 0.56738, + "ndcg_at_20": 0.58832, + "ndcg_at_3": 0.51605, + "ndcg_at_5": 0.54209, + "precision_at_1": 0.42764, + "precision_at_10": 0.07197, + "precision_at_20": 0.04011, + "precision_at_3": 0.19297, + "precision_at_5": 0.12849, + "recall_at_1": 0.42764, + "recall_at_10": 0.71971, + "recall_at_20": 0.80223, + "recall_at_3": 0.57891, + "recall_at_5": 0.64244 + } + ] + }, + "task_name": "LEMBQMSumRetrievalChunked" +} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_1024/overlap_64/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-soft-boundary/embed_size_1024/overlap_64/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json new file mode 100644 index 0000000..0955202 --- /dev/null +++ b/results-soft-boundary/embed_size_1024/overlap_64/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json @@ -0,0 +1,38 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 17.055257558822632, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.89068, + "map_at_1": 0.8006, + "map_at_10": 0.86288, + "map_at_3": 0.85218, + "map_at_5": 0.85873, + "mrr_at_1": 0.8005952380952381, + "mrr_at_10": 0.8628838340891912, + "mrr_at_3": 0.8521825396825397, + "mrr_at_5": 0.8587301587301588, + "ndcg_at_1": 0.8006, + "ndcg_at_10": 0.89068, + "ndcg_at_3": 0.86947, + "ndcg_at_5": 0.88088, + "precision_at_1": 0.8006, + "precision_at_10": 0.09762, + "precision_at_3": 0.30655, + "precision_at_5": 0.18929, + "recall_at_1": 0.8006, + "recall_at_10": 0.97619, + "recall_at_3": 0.91964, + "recall_at_5": 0.94643 + } + ] + }, + "task_name": "LEMBSummScreenFDRetrievalChunked" +} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_1024/overlap_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-soft-boundary/embed_size_1024/overlap_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json index ad117f0..93de53d 100644 --- a/results-soft-boundary/embed_size_1024/overlap_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ b/results-soft-boundary/embed_size_1024/overlap_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -1,6 +1,6 @@ { - "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", - "evaluation_time": 15.033222198486328, + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 15.988184928894043, "kg_co2_emissions": null, "mteb_version": "1.14.20", "scores": { diff --git a/results-soft-boundary/embed_size_2048/overlap_128/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json b/results-soft-boundary/embed_size_2048/overlap_128/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json new file mode 100644 index 0000000..c178da9 --- /dev/null +++ b/results-soft-boundary/embed_size_2048/overlap_128/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json @@ -0,0 +1,43 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 38.877949237823486, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.57196, + "map_at_1": 0.4296, + "map_at_10": 0.52294, + "map_at_20": 0.5287, + "map_at_3": 0.49705, + "map_at_5": 0.51179, + "mrr_at_1": 0.42960052390307796, + "mrr_at_10": 0.5229368717580898, + "mrr_at_20": 0.5287004998766055, + "mrr_at_3": 0.49705304518664045, + "mrr_at_5": 0.5117878192534381, + "ndcg_at_1": 0.4296, + "ndcg_at_10": 0.57196, + "ndcg_at_20": 0.59319, + "ndcg_at_3": 0.51877, + "ndcg_at_5": 0.54526, + "precision_at_1": 0.4296, + "precision_at_10": 0.07276, + "precision_at_20": 0.0406, + "precision_at_3": 0.19384, + "precision_at_5": 0.12914, + "recall_at_1": 0.4296, + "recall_at_10": 0.72757, + "recall_at_20": 0.81205, + "recall_at_3": 0.58153, + "recall_at_5": 0.64571 + } + ] + }, + "task_name": "LEMBQMSumRetrievalChunked" +} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_2048/overlap_128/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-soft-boundary/embed_size_2048/overlap_128/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json new file mode 100644 index 0000000..70d6bc7 --- /dev/null +++ b/results-soft-boundary/embed_size_2048/overlap_128/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json @@ -0,0 +1,38 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 21.771485567092896, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.88959, + "map_at_1": 0.79762, + "map_at_10": 0.86372, + "map_at_3": 0.85665, + "map_at_5": 0.86156, + "mrr_at_1": 0.7976190476190477, + "mrr_at_10": 0.8637188208616781, + "mrr_at_3": 0.8566468253968254, + "mrr_at_5": 0.8615575396825397, + "ndcg_at_1": 0.79762, + "ndcg_at_10": 0.88959, + "ndcg_at_3": 0.87588, + "ndcg_at_5": 0.8846, + "precision_at_1": 0.79762, + "precision_at_10": 0.09673, + "precision_at_3": 0.31052, + "precision_at_5": 0.19048, + "recall_at_1": 0.79762, + "recall_at_10": 0.96726, + "recall_at_3": 0.93155, + "recall_at_5": 0.95238 + } + ] + }, + "task_name": "LEMBSummScreenFDRetrievalChunked" +} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_2048/overlap_128/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-soft-boundary/embed_size_2048/overlap_128/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json index b15a329..7476616 100644 --- a/results-soft-boundary/embed_size_2048/overlap_128/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ b/results-soft-boundary/embed_size_2048/overlap_128/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -1,6 +1,6 @@ { - "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", - "evaluation_time": 19.98063826560974, + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 19.88721489906311, "kg_co2_emissions": null, "mteb_version": "1.14.20", "scores": { diff --git a/results-soft-boundary/embed_size_2048/overlap_256/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json b/results-soft-boundary/embed_size_2048/overlap_256/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json new file mode 100644 index 0000000..213743d --- /dev/null +++ b/results-soft-boundary/embed_size_2048/overlap_256/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json @@ -0,0 +1,43 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 39.476019859313965, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.57129, + "map_at_1": 0.42895, + "map_at_10": 0.52168, + "map_at_20": 0.52722, + "map_at_3": 0.49596, + "map_at_5": 0.51083, + "mrr_at_1": 0.4289456450556647, + "mrr_at_10": 0.5216759701042609, + "mrr_at_20": 0.5272231690823965, + "mrr_at_3": 0.4959615804409518, + "mrr_at_5": 0.510827330277232, + "ndcg_at_1": 0.42895, + "ndcg_at_10": 0.57129, + "ndcg_at_20": 0.5916, + "ndcg_at_3": 0.5184, + "ndcg_at_5": 0.54514, + "precision_at_1": 0.42895, + "precision_at_10": 0.07289, + "precision_at_20": 0.04047, + "precision_at_3": 0.1945, + "precision_at_5": 0.12967, + "recall_at_1": 0.42895, + "recall_at_10": 0.72888, + "recall_at_20": 0.80943, + "recall_at_3": 0.5835, + "recall_at_5": 0.64833 + } + ] + }, + "task_name": "LEMBQMSumRetrievalChunked" +} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_2048/overlap_256/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-soft-boundary/embed_size_2048/overlap_256/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json new file mode 100644 index 0000000..2a989ce --- /dev/null +++ b/results-soft-boundary/embed_size_2048/overlap_256/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json @@ -0,0 +1,38 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 22.649596452713013, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.8917, + "map_at_1": 0.79762, + "map_at_10": 0.86487, + "map_at_3": 0.85665, + "map_at_5": 0.86171, + "mrr_at_1": 0.7976190476190477, + "mrr_at_10": 0.8648679610733181, + "mrr_at_3": 0.8566468253968254, + "mrr_at_5": 0.8617063492063493, + "ndcg_at_1": 0.79762, + "ndcg_at_10": 0.8917, + "ndcg_at_3": 0.87518, + "ndcg_at_5": 0.88402, + "precision_at_1": 0.79762, + "precision_at_10": 0.09732, + "precision_at_3": 0.30952, + "precision_at_5": 0.18988, + "recall_at_1": 0.79762, + "recall_at_10": 0.97321, + "recall_at_3": 0.92857, + "recall_at_5": 0.9494 + } + ] + }, + "task_name": "LEMBSummScreenFDRetrievalChunked" +} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_2048/overlap_256/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-soft-boundary/embed_size_2048/overlap_256/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json index 519b1a4..286d43a 100644 --- a/results-soft-boundary/embed_size_2048/overlap_256/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ b/results-soft-boundary/embed_size_2048/overlap_256/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -1,6 +1,6 @@ { - "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", - "evaluation_time": 20.697884798049927, + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 20.78504467010498, "kg_co2_emissions": null, "mteb_version": "1.14.20", "scores": { diff --git a/results-soft-boundary/embed_size_2048/overlap_32/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json b/results-soft-boundary/embed_size_2048/overlap_32/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json new file mode 100644 index 0000000..9fee106 --- /dev/null +++ b/results-soft-boundary/embed_size_2048/overlap_32/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json @@ -0,0 +1,43 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 38.34771919250488, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.57008, + "map_at_1": 0.42371, + "map_at_10": 0.51964, + "map_at_20": 0.52517, + "map_at_3": 0.49345, + "map_at_5": 0.50779, + "mrr_at_1": 0.42370661427635886, + "mrr_at_10": 0.5196406482261099, + "mrr_at_20": 0.5251709363866996, + "mrr_at_3": 0.49345121152586774, + "mrr_at_5": 0.5077930582842174, + "ndcg_at_1": 0.42371, + "ndcg_at_10": 0.57008, + "ndcg_at_20": 0.5908, + "ndcg_at_3": 0.51608, + "ndcg_at_5": 0.54181, + "precision_at_1": 0.42371, + "precision_at_10": 0.07302, + "precision_at_20": 0.04067, + "precision_at_3": 0.19384, + "precision_at_5": 0.12875, + "recall_at_1": 0.42371, + "recall_at_10": 0.73019, + "recall_at_20": 0.81336, + "recall_at_3": 0.58153, + "recall_at_5": 0.64375 + } + ] + }, + "task_name": "LEMBQMSumRetrievalChunked" +} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_2048/overlap_32/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-soft-boundary/embed_size_2048/overlap_32/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json new file mode 100644 index 0000000..ce84efa --- /dev/null +++ b/results-soft-boundary/embed_size_2048/overlap_32/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json @@ -0,0 +1,38 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 21.172362804412842, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.89001, + "map_at_1": 0.8006, + "map_at_10": 0.86365, + "map_at_3": 0.85367, + "map_at_5": 0.86066, + "mrr_at_1": 0.8005952380952381, + "mrr_at_10": 0.8636526832955405, + "mrr_at_3": 0.8536706349206349, + "mrr_at_5": 0.8606646825396825, + "ndcg_at_1": 0.8006, + "ndcg_at_10": 0.89001, + "ndcg_at_3": 0.87064, + "ndcg_at_5": 0.88307, + "precision_at_1": 0.8006, + "precision_at_10": 0.09702, + "precision_at_3": 0.30655, + "precision_at_5": 0.18988, + "recall_at_1": 0.8006, + "recall_at_10": 0.97024, + "recall_at_3": 0.91964, + "recall_at_5": 0.9494 + } + ] + }, + "task_name": "LEMBSummScreenFDRetrievalChunked" +} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_2048/overlap_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-soft-boundary/embed_size_2048/overlap_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json index 5bbf470..a4cbcb3 100644 --- a/results-soft-boundary/embed_size_2048/overlap_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ b/results-soft-boundary/embed_size_2048/overlap_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -1,6 +1,6 @@ { - "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", - "evaluation_time": 19.531440019607544, + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 19.420426607131958, "kg_co2_emissions": null, "mteb_version": "1.14.20", "scores": { diff --git a/results-soft-boundary/embed_size_2048/overlap_512/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json b/results-soft-boundary/embed_size_2048/overlap_512/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json new file mode 100644 index 0000000..9836433 --- /dev/null +++ b/results-soft-boundary/embed_size_2048/overlap_512/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json @@ -0,0 +1,43 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 41.91482329368591, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.56813, + "map_at_1": 0.42633, + "map_at_10": 0.51894, + "map_at_20": 0.5249, + "map_at_3": 0.49301, + "map_at_5": 0.50831, + "mrr_at_1": 0.4263261296660118, + "mrr_at_10": 0.5189382127004917, + "mrr_at_20": 0.5249046808790312, + "mrr_at_3": 0.49301462562759224, + "mrr_at_5": 0.508306046714691, + "ndcg_at_1": 0.42633, + "ndcg_at_10": 0.56813, + "ndcg_at_20": 0.59005, + "ndcg_at_3": 0.51505, + "ndcg_at_5": 0.54245, + "precision_at_1": 0.42633, + "precision_at_10": 0.07243, + "precision_at_20": 0.04057, + "precision_at_3": 0.19297, + "precision_at_5": 0.12901, + "recall_at_1": 0.42633, + "recall_at_10": 0.7243, + "recall_at_20": 0.81139, + "recall_at_3": 0.57891, + "recall_at_5": 0.64506 + } + ] + }, + "task_name": "LEMBQMSumRetrievalChunked" +} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_2048/overlap_512/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-soft-boundary/embed_size_2048/overlap_512/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json new file mode 100644 index 0000000..9d5cdc0 --- /dev/null +++ b/results-soft-boundary/embed_size_2048/overlap_512/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json @@ -0,0 +1,38 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 24.755677223205566, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.88511, + "map_at_1": 0.79167, + "map_at_10": 0.858, + "map_at_3": 0.8502, + "map_at_5": 0.85556, + "mrr_at_1": 0.7916666666666666, + "mrr_at_10": 0.8579967403628117, + "mrr_at_3": 0.8501984126984128, + "mrr_at_5": 0.8555555555555555, + "ndcg_at_1": 0.79167, + "ndcg_at_10": 0.88511, + "ndcg_at_3": 0.86954, + "ndcg_at_5": 0.87927, + "precision_at_1": 0.79167, + "precision_at_10": 0.09673, + "precision_at_3": 0.30853, + "precision_at_5": 0.18988, + "recall_at_1": 0.79167, + "recall_at_10": 0.96726, + "recall_at_3": 0.9256, + "recall_at_5": 0.9494 + } + ] + }, + "task_name": "LEMBSummScreenFDRetrievalChunked" +} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_2048/overlap_512/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-soft-boundary/embed_size_2048/overlap_512/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json index b3cb768..a100aaa 100644 --- a/results-soft-boundary/embed_size_2048/overlap_512/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ b/results-soft-boundary/embed_size_2048/overlap_512/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -1,6 +1,6 @@ { - "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", - "evaluation_time": 22.640079736709595, + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 22.70358395576477, "kg_co2_emissions": null, "mteb_version": "1.14.20", "scores": { diff --git a/results-soft-boundary/embed_size_2048/overlap_64/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json b/results-soft-boundary/embed_size_2048/overlap_64/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json new file mode 100644 index 0000000..5b277b4 --- /dev/null +++ b/results-soft-boundary/embed_size_2048/overlap_64/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json @@ -0,0 +1,43 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 38.49483585357666, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.57013, + "map_at_1": 0.42829, + "map_at_10": 0.52047, + "map_at_20": 0.5263, + "map_at_3": 0.49422, + "map_at_5": 0.50898, + "mrr_at_1": 0.42829076620825146, + "mrr_at_10": 0.5204712009230673, + "mrr_at_20": 0.5263036291241446, + "mrr_at_3": 0.4942152368478498, + "mrr_at_5": 0.5089827548570182, + "ndcg_at_1": 0.42829, + "ndcg_at_10": 0.57013, + "ndcg_at_20": 0.59187, + "ndcg_at_3": 0.51595, + "ndcg_at_5": 0.54247, + "precision_at_1": 0.42829, + "precision_at_10": 0.07282, + "precision_at_20": 0.04077, + "precision_at_3": 0.19297, + "precision_at_5": 0.12862, + "recall_at_1": 0.42829, + "recall_at_10": 0.72823, + "recall_at_20": 0.81532, + "recall_at_3": 0.57891, + "recall_at_5": 0.64309 + } + ] + }, + "task_name": "LEMBQMSumRetrievalChunked" +} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_2048/overlap_64/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-soft-boundary/embed_size_2048/overlap_64/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json new file mode 100644 index 0000000..f15ccfa --- /dev/null +++ b/results-soft-boundary/embed_size_2048/overlap_64/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json @@ -0,0 +1,38 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 21.254602670669556, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.89022, + "map_at_1": 0.8006, + "map_at_10": 0.8639, + "map_at_3": 0.85367, + "map_at_5": 0.86096, + "mrr_at_1": 0.8005952380952381, + "mrr_at_10": 0.8638971560846561, + "mrr_at_3": 0.8536706349206349, + "mrr_at_5": 0.8609623015873017, + "ndcg_at_1": 0.8006, + "ndcg_at_10": 0.89022, + "ndcg_at_3": 0.87064, + "ndcg_at_5": 0.88333, + "precision_at_1": 0.8006, + "precision_at_10": 0.09702, + "precision_at_3": 0.30655, + "precision_at_5": 0.18988, + "recall_at_1": 0.8006, + "recall_at_10": 0.97024, + "recall_at_3": 0.91964, + "recall_at_5": 0.9494 + } + ] + }, + "task_name": "LEMBSummScreenFDRetrievalChunked" +} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_2048/overlap_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-soft-boundary/embed_size_2048/overlap_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json index b299341..9cdca16 100644 --- a/results-soft-boundary/embed_size_2048/overlap_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ b/results-soft-boundary/embed_size_2048/overlap_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -1,6 +1,6 @@ { - "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", - "evaluation_time": 19.4959659576416, + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 19.476801872253418, "kg_co2_emissions": null, "mteb_version": "1.14.20", "scores": { diff --git a/results-soft-boundary/embed_size_4096/overlap_128/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json b/results-soft-boundary/embed_size_4096/overlap_128/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json new file mode 100644 index 0000000..0fe9929 --- /dev/null +++ b/results-soft-boundary/embed_size_4096/overlap_128/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json @@ -0,0 +1,43 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 46.40877437591553, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.57158, + "map_at_1": 0.42895, + "map_at_10": 0.52188, + "map_at_20": 0.52728, + "map_at_3": 0.49629, + "map_at_5": 0.51119, + "mrr_at_1": 0.4289456450556647, + "mrr_at_10": 0.5218815292980322, + "mrr_at_20": 0.5272814518486844, + "mrr_at_3": 0.4962890198646584, + "mrr_at_5": 0.5111875136433094, + "ndcg_at_1": 0.42895, + "ndcg_at_10": 0.57158, + "ndcg_at_20": 0.59159, + "ndcg_at_3": 0.51865, + "ndcg_at_5": 0.54571, + "precision_at_1": 0.42895, + "precision_at_10": 0.07295, + "precision_at_20": 0.04047, + "precision_at_3": 0.1945, + "precision_at_5": 0.12993, + "recall_at_1": 0.42895, + "recall_at_10": 0.72954, + "recall_at_20": 0.80943, + "recall_at_3": 0.5835, + "recall_at_5": 0.64964 + } + ] + }, + "task_name": "LEMBQMSumRetrievalChunked" +} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_4096/overlap_128/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-soft-boundary/embed_size_4096/overlap_128/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json new file mode 100644 index 0000000..6a74bfb --- /dev/null +++ b/results-soft-boundary/embed_size_4096/overlap_128/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json @@ -0,0 +1,38 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 29.33760666847229, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.89081, + "map_at_1": 0.79762, + "map_at_10": 0.86367, + "map_at_3": 0.85516, + "map_at_5": 0.86081, + "mrr_at_1": 0.7976190476190477, + "mrr_at_10": 0.8636739417989419, + "mrr_at_3": 0.8551587301587301, + "mrr_at_5": 0.8608134920634921, + "ndcg_at_1": 0.79762, + "ndcg_at_10": 0.89081, + "ndcg_at_3": 0.87401, + "ndcg_at_5": 0.884, + "precision_at_1": 0.79762, + "precision_at_10": 0.09732, + "precision_at_3": 0.30952, + "precision_at_5": 0.19048, + "recall_at_1": 0.79762, + "recall_at_10": 0.97321, + "recall_at_3": 0.92857, + "recall_at_5": 0.95238 + } + ] + }, + "task_name": "LEMBSummScreenFDRetrievalChunked" +} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_4096/overlap_128/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-soft-boundary/embed_size_4096/overlap_128/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json index 2d201a9..88e4ab0 100644 --- a/results-soft-boundary/embed_size_4096/overlap_128/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ b/results-soft-boundary/embed_size_4096/overlap_128/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -1,6 +1,6 @@ { - "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", - "evaluation_time": 26.74845862388611, + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 26.896676778793335, "kg_co2_emissions": null, "mteb_version": "1.14.20", "scores": { diff --git a/results-soft-boundary/embed_size_4096/overlap_256/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json b/results-soft-boundary/embed_size_4096/overlap_256/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json new file mode 100644 index 0000000..94f5a97 --- /dev/null +++ b/results-soft-boundary/embed_size_4096/overlap_256/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json @@ -0,0 +1,43 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 46.92707061767578, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.57072, + "map_at_1": 0.42436, + "map_at_10": 0.52029, + "map_at_20": 0.52563, + "map_at_3": 0.49454, + "map_at_5": 0.50911, + "mrr_at_1": 0.4243614931237721, + "mrr_at_10": 0.5202918888577042, + "mrr_at_20": 0.5256342949085654, + "mrr_at_3": 0.4945426762715564, + "mrr_at_5": 0.5091137306265008, + "ndcg_at_1": 0.42436, + "ndcg_at_10": 0.57072, + "ndcg_at_20": 0.59055, + "ndcg_at_3": 0.51739, + "ndcg_at_5": 0.54374, + "precision_at_1": 0.42436, + "precision_at_10": 0.07308, + "precision_at_20": 0.0405, + "precision_at_3": 0.1945, + "precision_at_5": 0.12954, + "recall_at_1": 0.42436, + "recall_at_10": 0.73084, + "recall_at_20": 0.81009, + "recall_at_3": 0.5835, + "recall_at_5": 0.64768 + } + ] + }, + "task_name": "LEMBQMSumRetrievalChunked" +} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_4096/overlap_256/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-soft-boundary/embed_size_4096/overlap_256/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json new file mode 100644 index 0000000..28e31ed --- /dev/null +++ b/results-soft-boundary/embed_size_4096/overlap_256/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json @@ -0,0 +1,38 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 29.846808195114136, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.89165, + "map_at_1": 0.79762, + "map_at_10": 0.86473, + "map_at_3": 0.85615, + "map_at_5": 0.8624, + "mrr_at_1": 0.7976190476190477, + "mrr_at_10": 0.8647321428571428, + "mrr_at_3": 0.8561507936507937, + "mrr_at_5": 0.8624007936507936, + "ndcg_at_1": 0.79762, + "ndcg_at_10": 0.89165, + "ndcg_at_3": 0.87479, + "ndcg_at_5": 0.88593, + "precision_at_1": 0.79762, + "precision_at_10": 0.09732, + "precision_at_3": 0.30952, + "precision_at_5": 0.19107, + "recall_at_1": 0.79762, + "recall_at_10": 0.97321, + "recall_at_3": 0.92857, + "recall_at_5": 0.95536 + } + ] + }, + "task_name": "LEMBSummScreenFDRetrievalChunked" +} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_4096/overlap_256/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-soft-boundary/embed_size_4096/overlap_256/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json index aae5d39..a37303b 100644 --- a/results-soft-boundary/embed_size_4096/overlap_256/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ b/results-soft-boundary/embed_size_4096/overlap_256/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -1,6 +1,6 @@ { - "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", - "evaluation_time": 27.24738645553589, + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 27.351439714431763, "kg_co2_emissions": null, "mteb_version": "1.14.20", "scores": { diff --git a/results-soft-boundary/embed_size_4096/overlap_32/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json b/results-soft-boundary/embed_size_4096/overlap_32/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json new file mode 100644 index 0000000..8f0b56e --- /dev/null +++ b/results-soft-boundary/embed_size_4096/overlap_32/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json @@ -0,0 +1,43 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 45.77136516571045, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.57235, + "map_at_1": 0.43091, + "map_at_10": 0.52309, + "map_at_20": 0.52872, + "map_at_3": 0.49793, + "map_at_5": 0.51197, + "mrr_at_1": 0.4309102815979044, + "mrr_at_10": 0.5230935749108637, + "mrr_at_20": 0.5287177084175539, + "mrr_at_3": 0.49792621698319145, + "mrr_at_5": 0.5119733682602052, + "ndcg_at_1": 0.43091, + "ndcg_at_10": 0.57235, + "ndcg_at_20": 0.59307, + "ndcg_at_3": 0.52021, + "ndcg_at_5": 0.54568, + "precision_at_1": 0.43091, + "precision_at_10": 0.07289, + "precision_at_20": 0.04057, + "precision_at_3": 0.19494, + "precision_at_5": 0.1294, + "recall_at_1": 0.43091, + "recall_at_10": 0.72888, + "recall_at_20": 0.81139, + "recall_at_3": 0.58481, + "recall_at_5": 0.64702 + } + ] + }, + "task_name": "LEMBQMSumRetrievalChunked" +} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_4096/overlap_32/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-soft-boundary/embed_size_4096/overlap_32/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json new file mode 100644 index 0000000..573b8e4 --- /dev/null +++ b/results-soft-boundary/embed_size_4096/overlap_32/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json @@ -0,0 +1,38 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 28.918544054031372, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.8896, + "map_at_1": 0.79464, + "map_at_10": 0.86285, + "map_at_3": 0.85466, + "map_at_5": 0.86032, + "mrr_at_1": 0.7946428571428571, + "mrr_at_10": 0.8628472222222223, + "mrr_at_3": 0.8546626984126985, + "mrr_at_5": 0.8603174603174603, + "ndcg_at_1": 0.79464, + "ndcg_at_10": 0.8896, + "ndcg_at_3": 0.87369, + "ndcg_at_5": 0.88368, + "precision_at_1": 0.79464, + "precision_at_10": 0.09702, + "precision_at_3": 0.30952, + "precision_at_5": 0.19048, + "recall_at_1": 0.79464, + "recall_at_10": 0.97024, + "recall_at_3": 0.92857, + "recall_at_5": 0.95238 + } + ] + }, + "task_name": "LEMBSummScreenFDRetrievalChunked" +} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_4096/overlap_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-soft-boundary/embed_size_4096/overlap_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json index 56212e1..ccec7ca 100644 --- a/results-soft-boundary/embed_size_4096/overlap_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ b/results-soft-boundary/embed_size_4096/overlap_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -1,6 +1,6 @@ { - "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", - "evaluation_time": 26.33897829055786, + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 26.577144145965576, "kg_co2_emissions": null, "mteb_version": "1.14.20", "scores": { diff --git a/results-soft-boundary/embed_size_4096/overlap_512/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json b/results-soft-boundary/embed_size_4096/overlap_512/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json new file mode 100644 index 0000000..7c51255 --- /dev/null +++ b/results-soft-boundary/embed_size_4096/overlap_512/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json @@ -0,0 +1,43 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 48.13499999046326, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.5719, + "map_at_1": 0.42633, + "map_at_10": 0.52184, + "map_at_20": 0.52697, + "map_at_3": 0.49673, + "map_at_5": 0.51051, + "mrr_at_1": 0.4263261296660118, + "mrr_at_10": 0.5218412490514651, + "mrr_at_20": 0.5269714581870543, + "mrr_at_3": 0.49672560576293384, + "mrr_at_5": 0.5105108055009823, + "ndcg_at_1": 0.42633, + "ndcg_at_10": 0.5719, + "ndcg_at_20": 0.59093, + "ndcg_at_3": 0.51968, + "ndcg_at_5": 0.54465, + "precision_at_1": 0.42633, + "precision_at_10": 0.07308, + "precision_at_20": 0.04034, + "precision_at_3": 0.19537, + "precision_at_5": 0.1294, + "recall_at_1": 0.42633, + "recall_at_10": 0.73084, + "recall_at_20": 0.80681, + "recall_at_3": 0.58612, + "recall_at_5": 0.64702 + } + ] + }, + "task_name": "LEMBQMSumRetrievalChunked" +} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_4096/overlap_512/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-soft-boundary/embed_size_4096/overlap_512/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json new file mode 100644 index 0000000..6c915d7 --- /dev/null +++ b/results-soft-boundary/embed_size_4096/overlap_512/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json @@ -0,0 +1,38 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 31.02289605140686, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.89229, + "map_at_1": 0.8006, + "map_at_10": 0.8657, + "map_at_3": 0.85665, + "map_at_5": 0.86245, + "mrr_at_1": 0.8005952380952381, + "mrr_at_10": 0.8657029478458049, + "mrr_at_3": 0.8566468253968254, + "mrr_at_5": 0.8624503968253969, + "ndcg_at_1": 0.8006, + "ndcg_at_10": 0.89229, + "ndcg_at_3": 0.8744, + "ndcg_at_5": 0.88452, + "precision_at_1": 0.8006, + "precision_at_10": 0.09732, + "precision_at_3": 0.30853, + "precision_at_5": 0.18988, + "recall_at_1": 0.8006, + "recall_at_10": 0.97321, + "recall_at_3": 0.9256, + "recall_at_5": 0.9494 + } + ] + }, + "task_name": "LEMBSummScreenFDRetrievalChunked" +} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_4096/overlap_512/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-soft-boundary/embed_size_4096/overlap_512/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json index 4ad6a0f..c7d214f 100644 --- a/results-soft-boundary/embed_size_4096/overlap_512/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ b/results-soft-boundary/embed_size_4096/overlap_512/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -1,6 +1,6 @@ { - "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", - "evaluation_time": 28.279660940170288, + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 28.48093295097351, "kg_co2_emissions": null, "mteb_version": "1.14.20", "scores": { diff --git a/results-soft-boundary/embed_size_4096/overlap_64/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json b/results-soft-boundary/embed_size_4096/overlap_64/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json new file mode 100644 index 0000000..ddca10d --- /dev/null +++ b/results-soft-boundary/embed_size_4096/overlap_64/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json @@ -0,0 +1,43 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 46.11063885688782, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.57256, + "map_at_1": 0.4296, + "map_at_10": 0.52266, + "map_at_20": 0.52806, + "map_at_3": 0.49705, + "map_at_5": 0.511, + "mrr_at_1": 0.42960052390307796, + "mrr_at_10": 0.5226562093949128, + "mrr_at_20": 0.5280638645171482, + "mrr_at_3": 0.49705304518664045, + "mrr_at_5": 0.5110019646365422, + "ndcg_at_1": 0.4296, + "ndcg_at_10": 0.57256, + "ndcg_at_20": 0.59269, + "ndcg_at_3": 0.51924, + "ndcg_at_5": 0.54449, + "precision_at_1": 0.4296, + "precision_at_10": 0.07315, + "precision_at_20": 0.0406, + "precision_at_3": 0.1945, + "precision_at_5": 0.12901, + "recall_at_1": 0.4296, + "recall_at_10": 0.7315, + "recall_at_20": 0.81205, + "recall_at_3": 0.5835, + "recall_at_5": 0.64506 + } + ] + }, + "task_name": "LEMBQMSumRetrievalChunked" +} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_4096/overlap_64/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-soft-boundary/embed_size_4096/overlap_64/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json new file mode 100644 index 0000000..b379c91 --- /dev/null +++ b/results-soft-boundary/embed_size_4096/overlap_64/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json @@ -0,0 +1,38 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 28.984849214553833, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.89135, + "map_at_1": 0.79762, + "map_at_10": 0.86438, + "map_at_3": 0.85516, + "map_at_5": 0.86156, + "mrr_at_1": 0.7976190476190477, + "mrr_at_10": 0.8643849206349207, + "mrr_at_3": 0.8551587301587301, + "mrr_at_5": 0.8615575396825397, + "ndcg_at_1": 0.79762, + "ndcg_at_10": 0.89135, + "ndcg_at_3": 0.8733, + "ndcg_at_5": 0.88457, + "precision_at_1": 0.79762, + "precision_at_10": 0.09732, + "precision_at_3": 0.30853, + "precision_at_5": 0.19048, + "recall_at_1": 0.79762, + "recall_at_10": 0.97321, + "recall_at_3": 0.9256, + "recall_at_5": 0.95238 + } + ] + }, + "task_name": "LEMBSummScreenFDRetrievalChunked" +} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_4096/overlap_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-soft-boundary/embed_size_4096/overlap_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json index 9a1156b..8ff2087 100644 --- a/results-soft-boundary/embed_size_4096/overlap_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ b/results-soft-boundary/embed_size_4096/overlap_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -1,6 +1,6 @@ { - "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", - "evaluation_time": 26.438919067382812, + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 26.565731525421143, "kg_co2_emissions": null, "mteb_version": "1.14.20", "scores": { diff --git a/results-soft-boundary/embed_size_512/overlap_128/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json b/results-soft-boundary/embed_size_512/overlap_128/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json new file mode 100644 index 0000000..0752ec3 --- /dev/null +++ b/results-soft-boundary/embed_size_512/overlap_128/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json @@ -0,0 +1,43 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 33.797311544418335, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.56433, + "map_at_1": 0.42371, + "map_at_10": 0.51527, + "map_at_20": 0.52098, + "map_at_3": 0.48985, + "map_at_5": 0.50403, + "mrr_at_1": 0.42370661427635886, + "mrr_at_10": 0.5152693319196265, + "mrr_at_20": 0.5209754242570668, + "mrr_at_3": 0.48984937786509497, + "mrr_at_5": 0.5040275049115913, + "ndcg_at_1": 0.42371, + "ndcg_at_10": 0.56433, + "ndcg_at_20": 0.58561, + "ndcg_at_3": 0.51107, + "ndcg_at_5": 0.5368, + "precision_at_1": 0.42371, + "precision_at_10": 0.0721, + "precision_at_20": 0.04031, + "precision_at_3": 0.19079, + "precision_at_5": 0.12705, + "recall_at_1": 0.42371, + "recall_at_10": 0.72102, + "recall_at_20": 0.80616, + "recall_at_3": 0.57236, + "recall_at_5": 0.63523 + } + ] + }, + "task_name": "LEMBQMSumRetrievalChunked" +} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_512/overlap_128/no_model_name_available/no_revision_available/model_meta.json b/results-soft-boundary/embed_size_512/overlap_128/no_model_name_available/no_revision_available/model_meta.json new file mode 100644 index 0000000..40ff52f --- /dev/null +++ b/results-soft-boundary/embed_size_512/overlap_128/no_model_name_available/no_revision_available/model_meta.json @@ -0,0 +1 @@ +{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_512/overlap_256/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json b/results-soft-boundary/embed_size_512/overlap_256/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json new file mode 100644 index 0000000..78ce41e --- /dev/null +++ b/results-soft-boundary/embed_size_512/overlap_256/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json @@ -0,0 +1,43 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 38.087034463882446, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.56557, + "map_at_1": 0.42698, + "map_at_10": 0.51611, + "map_at_20": 0.52222, + "map_at_3": 0.48941, + "map_at_5": 0.50372, + "mrr_at_1": 0.426981008513425, + "mrr_at_10": 0.516108720283573, + "mrr_at_20": 0.5222164663592797, + "mrr_at_3": 0.48941279196681947, + "mrr_at_5": 0.5037218947827985, + "ndcg_at_1": 0.42698, + "ndcg_at_10": 0.56557, + "ndcg_at_20": 0.58811, + "ndcg_at_3": 0.51018, + "ndcg_at_5": 0.53602, + "precision_at_1": 0.42698, + "precision_at_10": 0.07236, + "precision_at_20": 0.04067, + "precision_at_3": 0.19013, + "precision_at_5": 0.12665, + "recall_at_1": 0.42698, + "recall_at_10": 0.72364, + "recall_at_20": 0.81336, + "recall_at_3": 0.5704, + "recall_at_5": 0.63327 + } + ] + }, + "task_name": "LEMBQMSumRetrievalChunked" +} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_512/overlap_256/no_model_name_available/no_revision_available/model_meta.json b/results-soft-boundary/embed_size_512/overlap_256/no_model_name_available/no_revision_available/model_meta.json new file mode 100644 index 0000000..40ff52f --- /dev/null +++ b/results-soft-boundary/embed_size_512/overlap_256/no_model_name_available/no_revision_available/model_meta.json @@ -0,0 +1 @@ +{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_512/overlap_32/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json b/results-soft-boundary/embed_size_512/overlap_32/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json new file mode 100644 index 0000000..89fb91f --- /dev/null +++ b/results-soft-boundary/embed_size_512/overlap_32/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json @@ -0,0 +1,43 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 32.38495421409607, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.56145, + "map_at_1": 0.41912, + "map_at_10": 0.51173, + "map_at_20": 0.51745, + "map_at_3": 0.4845, + "map_at_5": 0.50084, + "mrr_at_1": 0.41912246234446626, + "mrr_at_10": 0.5117296078003347, + "mrr_at_20": 0.5174488508122417, + "mrr_at_3": 0.48450120061122026, + "mrr_at_5": 0.5008404278541803, + "ndcg_at_1": 0.41912, + "ndcg_at_10": 0.56145, + "ndcg_at_20": 0.58241, + "ndcg_at_3": 0.50573, + "ndcg_at_5": 0.53501, + "precision_at_1": 0.41912, + "precision_at_10": 0.07197, + "precision_at_20": 0.04014, + "precision_at_3": 0.18904, + "precision_at_5": 0.12757, + "recall_at_1": 0.41912, + "recall_at_10": 0.71971, + "recall_at_20": 0.80288, + "recall_at_3": 0.56713, + "recall_at_5": 0.63785 + } + ] + }, + "task_name": "LEMBQMSumRetrievalChunked" +} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_512/overlap_32/no_model_name_available/no_revision_available/model_meta.json b/results-soft-boundary/embed_size_512/overlap_32/no_model_name_available/no_revision_available/model_meta.json new file mode 100644 index 0000000..40ff52f --- /dev/null +++ b/results-soft-boundary/embed_size_512/overlap_32/no_model_name_available/no_revision_available/model_meta.json @@ -0,0 +1 @@ +{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_512/overlap_512/no_model_name_available/no_revision_available/model_meta.json b/results-soft-boundary/embed_size_512/overlap_512/no_model_name_available/no_revision_available/model_meta.json new file mode 100644 index 0000000..40ff52f --- /dev/null +++ b/results-soft-boundary/embed_size_512/overlap_512/no_model_name_available/no_revision_available/model_meta.json @@ -0,0 +1 @@ +{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_512/overlap_64/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json b/results-soft-boundary/embed_size_512/overlap_64/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json new file mode 100644 index 0000000..2aed848 --- /dev/null +++ b/results-soft-boundary/embed_size_512/overlap_64/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json @@ -0,0 +1,43 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 32.886000633239746, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.56817, + "map_at_1": 0.43353, + "map_at_10": 0.52101, + "map_at_20": 0.52707, + "map_at_3": 0.49509, + "map_at_5": 0.51113, + "mrr_at_1": 0.4335297969875573, + "mrr_at_10": 0.5210093969917152, + "mrr_at_20": 0.5270617882207564, + "mrr_at_3": 0.4950884086444008, + "mrr_at_5": 0.5111329404060249, + "ndcg_at_1": 0.43353, + "ndcg_at_10": 0.56817, + "ndcg_at_20": 0.59042, + "ndcg_at_3": 0.51511, + "ndcg_at_5": 0.54399, + "precision_at_1": 0.43353, + "precision_at_10": 0.07184, + "precision_at_20": 0.04034, + "precision_at_3": 0.19101, + "precision_at_5": 0.12862, + "recall_at_1": 0.43353, + "recall_at_10": 0.7184, + "recall_at_20": 0.80681, + "recall_at_3": 0.57302, + "recall_at_5": 0.64309 + } + ] + }, + "task_name": "LEMBQMSumRetrievalChunked" +} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_512/overlap_64/no_model_name_available/no_revision_available/model_meta.json b/results-soft-boundary/embed_size_512/overlap_64/no_model_name_available/no_revision_available/model_meta.json new file mode 100644 index 0000000..40ff52f --- /dev/null +++ b/results-soft-boundary/embed_size_512/overlap_64/no_model_name_available/no_revision_available/model_meta.json @@ -0,0 +1 @@ +{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_8192/overlap_128/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json b/results-soft-boundary/embed_size_8192/overlap_128/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json new file mode 100644 index 0000000..510ed14 --- /dev/null +++ b/results-soft-boundary/embed_size_8192/overlap_128/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json @@ -0,0 +1,43 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 59.90270757675171, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.57159, + "map_at_1": 0.42895, + "map_at_10": 0.5221, + "map_at_20": 0.52746, + "map_at_3": 0.49629, + "map_at_5": 0.51142, + "mrr_at_1": 0.4289456450556647, + "mrr_at_10": 0.5220993025020529, + "mrr_at_20": 0.5274577033575282, + "mrr_at_3": 0.4962890198646584, + "mrr_at_5": 0.5114167212399039, + "ndcg_at_1": 0.42895, + "ndcg_at_10": 0.57159, + "ndcg_at_20": 0.59144, + "ndcg_at_3": 0.51834, + "ndcg_at_5": 0.54573, + "precision_at_1": 0.42895, + "precision_at_10": 0.07289, + "precision_at_20": 0.04041, + "precision_at_3": 0.19406, + "precision_at_5": 0.1298, + "recall_at_1": 0.42895, + "recall_at_10": 0.72888, + "recall_at_20": 0.80812, + "recall_at_3": 0.58219, + "recall_at_5": 0.64898 + } + ] + }, + "task_name": "LEMBQMSumRetrievalChunked" +} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_8192/overlap_128/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-soft-boundary/embed_size_8192/overlap_128/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json new file mode 100644 index 0000000..e07c115 --- /dev/null +++ b/results-soft-boundary/embed_size_8192/overlap_128/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json @@ -0,0 +1,38 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 41.60118389129639, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.8896, + "map_at_1": 0.79464, + "map_at_10": 0.86285, + "map_at_3": 0.85466, + "map_at_5": 0.86032, + "mrr_at_1": 0.7946428571428571, + "mrr_at_10": 0.8628472222222223, + "mrr_at_3": 0.8546626984126985, + "mrr_at_5": 0.8603174603174603, + "ndcg_at_1": 0.79464, + "ndcg_at_10": 0.8896, + "ndcg_at_3": 0.87369, + "ndcg_at_5": 0.88368, + "precision_at_1": 0.79464, + "precision_at_10": 0.09702, + "precision_at_3": 0.30952, + "precision_at_5": 0.19048, + "recall_at_1": 0.79464, + "recall_at_10": 0.97024, + "recall_at_3": 0.92857, + "recall_at_5": 0.95238 + } + ] + }, + "task_name": "LEMBSummScreenFDRetrievalChunked" +} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_8192/overlap_128/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-soft-boundary/embed_size_8192/overlap_128/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json index 8add32b..62f0c08 100644 --- a/results-soft-boundary/embed_size_8192/overlap_128/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ b/results-soft-boundary/embed_size_8192/overlap_128/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -1,6 +1,6 @@ { - "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", - "evaluation_time": 37.05101680755615, + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 37.52750086784363, "kg_co2_emissions": null, "mteb_version": "1.14.20", "scores": { diff --git a/results-soft-boundary/embed_size_8192/overlap_256/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json b/results-soft-boundary/embed_size_8192/overlap_256/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json new file mode 100644 index 0000000..70910a4 --- /dev/null +++ b/results-soft-boundary/embed_size_8192/overlap_256/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json @@ -0,0 +1,43 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 60.42517876625061, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.5714, + "map_at_1": 0.42829, + "map_at_10": 0.52166, + "map_at_20": 0.52697, + "map_at_3": 0.49552, + "map_at_5": 0.51095, + "mrr_at_1": 0.42829076620825146, + "mrr_at_10": 0.521663496221453, + "mrr_at_20": 0.5269704423103457, + "mrr_at_3": 0.49552499454267623, + "mrr_at_5": 0.5109473913992578, + "ndcg_at_1": 0.42829, + "ndcg_at_10": 0.5714, + "ndcg_at_20": 0.59108, + "ndcg_at_3": 0.5176, + "ndcg_at_5": 0.54539, + "precision_at_1": 0.42829, + "precision_at_10": 0.07295, + "precision_at_20": 0.04041, + "precision_at_3": 0.19384, + "precision_at_5": 0.1298, + "recall_at_1": 0.42829, + "recall_at_10": 0.72954, + "recall_at_20": 0.80812, + "recall_at_3": 0.58153, + "recall_at_5": 0.64898 + } + ] + }, + "task_name": "LEMBQMSumRetrievalChunked" +} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_8192/overlap_256/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-soft-boundary/embed_size_8192/overlap_256/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json new file mode 100644 index 0000000..55d3863 --- /dev/null +++ b/results-soft-boundary/embed_size_8192/overlap_256/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json @@ -0,0 +1,38 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 42.03281593322754, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.8896, + "map_at_1": 0.79464, + "map_at_10": 0.86285, + "map_at_3": 0.85466, + "map_at_5": 0.86032, + "mrr_at_1": 0.7946428571428571, + "mrr_at_10": 0.8628472222222223, + "mrr_at_3": 0.8546626984126985, + "mrr_at_5": 0.8603174603174603, + "ndcg_at_1": 0.79464, + "ndcg_at_10": 0.8896, + "ndcg_at_3": 0.87369, + "ndcg_at_5": 0.88368, + "precision_at_1": 0.79464, + "precision_at_10": 0.09702, + "precision_at_3": 0.30952, + "precision_at_5": 0.19048, + "recall_at_1": 0.79464, + "recall_at_10": 0.97024, + "recall_at_3": 0.92857, + "recall_at_5": 0.95238 + } + ] + }, + "task_name": "LEMBSummScreenFDRetrievalChunked" +} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_8192/overlap_256/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-soft-boundary/embed_size_8192/overlap_256/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json index b73c116..3ac7a2e 100644 --- a/results-soft-boundary/embed_size_8192/overlap_256/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ b/results-soft-boundary/embed_size_8192/overlap_256/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -1,6 +1,6 @@ { - "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", - "evaluation_time": 37.3258798122406, + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 37.988229513168335, "kg_co2_emissions": null, "mteb_version": "1.14.20", "scores": { diff --git a/results-soft-boundary/embed_size_8192/overlap_32/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json b/results-soft-boundary/embed_size_8192/overlap_32/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json new file mode 100644 index 0000000..fb2986d --- /dev/null +++ b/results-soft-boundary/embed_size_8192/overlap_32/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json @@ -0,0 +1,43 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 59.76712894439697, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.57078, + "map_at_1": 0.42829, + "map_at_10": 0.52124, + "map_at_20": 0.52661, + "map_at_3": 0.4952, + "map_at_5": 0.51055, + "mrr_at_1": 0.42829076620825146, + "mrr_at_10": 0.5212370453529589, + "mrr_at_20": 0.5266118548320644, + "mrr_at_3": 0.4951975551189696, + "mrr_at_5": 0.5105544640908098, + "ndcg_at_1": 0.42829, + "ndcg_at_10": 0.57078, + "ndcg_at_20": 0.59065, + "ndcg_at_3": 0.51719, + "ndcg_at_5": 0.54492, + "precision_at_1": 0.42829, + "precision_at_10": 0.07282, + "precision_at_20": 0.04037, + "precision_at_3": 0.19363, + "precision_at_5": 0.12967, + "recall_at_1": 0.42829, + "recall_at_10": 0.72823, + "recall_at_20": 0.80747, + "recall_at_3": 0.58088, + "recall_at_5": 0.64833 + } + ] + }, + "task_name": "LEMBQMSumRetrievalChunked" +} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_8192/overlap_32/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-soft-boundary/embed_size_8192/overlap_32/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json new file mode 100644 index 0000000..e28ae82 --- /dev/null +++ b/results-soft-boundary/embed_size_8192/overlap_32/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json @@ -0,0 +1,38 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 41.46819353103638, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.8896, + "map_at_1": 0.79464, + "map_at_10": 0.86285, + "map_at_3": 0.85466, + "map_at_5": 0.86032, + "mrr_at_1": 0.7946428571428571, + "mrr_at_10": 0.8628472222222223, + "mrr_at_3": 0.8546626984126985, + "mrr_at_5": 0.8603174603174603, + "ndcg_at_1": 0.79464, + "ndcg_at_10": 0.8896, + "ndcg_at_3": 0.87369, + "ndcg_at_5": 0.88368, + "precision_at_1": 0.79464, + "precision_at_10": 0.09702, + "precision_at_3": 0.30952, + "precision_at_5": 0.19048, + "recall_at_1": 0.79464, + "recall_at_10": 0.97024, + "recall_at_3": 0.92857, + "recall_at_5": 0.95238 + } + ] + }, + "task_name": "LEMBSummScreenFDRetrievalChunked" +} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_8192/overlap_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-soft-boundary/embed_size_8192/overlap_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json index d870b79..c66dd56 100644 --- a/results-soft-boundary/embed_size_8192/overlap_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ b/results-soft-boundary/embed_size_8192/overlap_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -1,6 +1,6 @@ { - "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", - "evaluation_time": 37.429115772247314, + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 37.43460035324097, "kg_co2_emissions": null, "mteb_version": "1.14.20", "scores": { diff --git a/results-soft-boundary/embed_size_8192/overlap_512/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json b/results-soft-boundary/embed_size_8192/overlap_512/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json new file mode 100644 index 0000000..ac77fe1 --- /dev/null +++ b/results-soft-boundary/embed_size_8192/overlap_512/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json @@ -0,0 +1,43 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 61.178690671920776, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.57081, + "map_at_1": 0.42764, + "map_at_10": 0.52091, + "map_at_20": 0.52621, + "map_at_3": 0.49476, + "map_at_5": 0.51012, + "mrr_at_1": 0.42763588736083824, + "mrr_at_10": 0.5209119447822788, + "mrr_at_20": 0.5262128450125312, + "mrr_at_3": 0.4947609692206942, + "mrr_at_5": 0.5101178781925344, + "ndcg_at_1": 0.42764, + "ndcg_at_10": 0.57081, + "ndcg_at_20": 0.59048, + "ndcg_at_3": 0.51686, + "ndcg_at_5": 0.54459, + "precision_at_1": 0.42764, + "precision_at_10": 0.07295, + "precision_at_20": 0.04041, + "precision_at_3": 0.19363, + "precision_at_5": 0.12967, + "recall_at_1": 0.42764, + "recall_at_10": 0.72954, + "recall_at_20": 0.80812, + "recall_at_3": 0.58088, + "recall_at_5": 0.64833 + } + ] + }, + "task_name": "LEMBQMSumRetrievalChunked" +} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_8192/overlap_512/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-soft-boundary/embed_size_8192/overlap_512/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json new file mode 100644 index 0000000..8496845 --- /dev/null +++ b/results-soft-boundary/embed_size_8192/overlap_512/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json @@ -0,0 +1,38 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 42.5116925239563, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.8896, + "map_at_1": 0.79464, + "map_at_10": 0.86285, + "map_at_3": 0.85466, + "map_at_5": 0.86032, + "mrr_at_1": 0.7946428571428571, + "mrr_at_10": 0.8628472222222223, + "mrr_at_3": 0.8546626984126985, + "mrr_at_5": 0.8603174603174603, + "ndcg_at_1": 0.79464, + "ndcg_at_10": 0.8896, + "ndcg_at_3": 0.87369, + "ndcg_at_5": 0.88368, + "precision_at_1": 0.79464, + "precision_at_10": 0.09702, + "precision_at_3": 0.30952, + "precision_at_5": 0.19048, + "recall_at_1": 0.79464, + "recall_at_10": 0.97024, + "recall_at_3": 0.92857, + "recall_at_5": 0.95238 + } + ] + }, + "task_name": "LEMBSummScreenFDRetrievalChunked" +} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_8192/overlap_512/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-soft-boundary/embed_size_8192/overlap_512/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json index cb29697..3c8224f 100644 --- a/results-soft-boundary/embed_size_8192/overlap_512/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ b/results-soft-boundary/embed_size_8192/overlap_512/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -1,6 +1,6 @@ { - "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", - "evaluation_time": 37.87094449996948, + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 38.475874185562134, "kg_co2_emissions": null, "mteb_version": "1.14.20", "scores": { diff --git a/results-soft-boundary/embed_size_8192/overlap_64/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json b/results-soft-boundary/embed_size_8192/overlap_64/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json new file mode 100644 index 0000000..c0e2dd5 --- /dev/null +++ b/results-soft-boundary/embed_size_8192/overlap_64/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json @@ -0,0 +1,43 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 59.825618267059326, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.5716, + "map_at_1": 0.43026, + "map_at_10": 0.52234, + "map_at_20": 0.52777, + "map_at_3": 0.49651, + "map_at_5": 0.51164, + "mrr_at_1": 0.4302554027504912, + "mrr_at_10": 0.5223448820698329, + "mrr_at_20": 0.5277702261925865, + "mrr_at_3": 0.49650731281379606, + "mrr_at_5": 0.5116350141890417, + "ndcg_at_1": 0.43026, + "ndcg_at_10": 0.5716, + "ndcg_at_20": 0.59165, + "ndcg_at_3": 0.51833, + "ndcg_at_5": 0.54572, + "precision_at_1": 0.43026, + "precision_at_10": 0.07282, + "precision_at_20": 0.04041, + "precision_at_3": 0.19384, + "precision_at_5": 0.12967, + "recall_at_1": 0.43026, + "recall_at_10": 0.72823, + "recall_at_20": 0.80812, + "recall_at_3": 0.58153, + "recall_at_5": 0.64833 + } + ] + }, + "task_name": "LEMBQMSumRetrievalChunked" +} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_8192/overlap_64/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-soft-boundary/embed_size_8192/overlap_64/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json new file mode 100644 index 0000000..d244a55 --- /dev/null +++ b/results-soft-boundary/embed_size_8192/overlap_64/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json @@ -0,0 +1,38 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 41.537845849990845, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.8896, + "map_at_1": 0.79464, + "map_at_10": 0.86285, + "map_at_3": 0.85466, + "map_at_5": 0.86032, + "mrr_at_1": 0.7946428571428571, + "mrr_at_10": 0.8628472222222223, + "mrr_at_3": 0.8546626984126985, + "mrr_at_5": 0.8603174603174603, + "ndcg_at_1": 0.79464, + "ndcg_at_10": 0.8896, + "ndcg_at_3": 0.87369, + "ndcg_at_5": 0.88368, + "precision_at_1": 0.79464, + "precision_at_10": 0.09702, + "precision_at_3": 0.30952, + "precision_at_5": 0.19048, + "recall_at_1": 0.79464, + "recall_at_10": 0.97024, + "recall_at_3": 0.92857, + "recall_at_5": 0.95238 + } + ] + }, + "task_name": "LEMBSummScreenFDRetrievalChunked" +} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_8192/overlap_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-soft-boundary/embed_size_8192/overlap_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json index 071adf3..6f86397 100644 --- a/results-soft-boundary/embed_size_8192/overlap_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ b/results-soft-boundary/embed_size_8192/overlap_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -1,6 +1,6 @@ { - "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", - "evaluation_time": 36.87527394294739, + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 37.48687982559204, "kg_co2_emissions": null, "mteb_version": "1.14.20", "scores": { diff --git a/results-truncation/embed_size_8192/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json b/results-truncation/embed_size_8192/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json new file mode 100644 index 0000000..eb44b65 --- /dev/null +++ b/results-truncation/embed_size_8192/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json @@ -0,0 +1,43 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 33.62538528442383, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.48383, + "map_at_1": 0.32547, + "map_at_10": 0.42617, + "map_at_20": 0.43314, + "map_at_3": 0.39413, + "map_at_5": 0.41246, + "mrr_at_1": 0.3254747871643746, + "mrr_at_10": 0.4261722851113814, + "mrr_at_20": 0.43314067378880616, + "mrr_at_3": 0.3941279196681947, + "mrr_at_5": 0.41246452739576517, + "ndcg_at_1": 0.32547, + "ndcg_at_10": 0.48383, + "ndcg_at_20": 0.5096, + "ndcg_at_3": 0.41762, + "ndcg_at_5": 0.45059, + "precision_at_1": 0.32547, + "precision_at_10": 0.06686, + "precision_at_20": 0.03857, + "precision_at_3": 0.16197, + "precision_at_5": 0.11316, + "recall_at_1": 0.32547, + "recall_at_10": 0.66863, + "recall_at_20": 0.77145, + "recall_at_3": 0.48592, + "recall_at_5": 0.56582 + } + ] + }, + "task_name": "LEMBQMSumRetrievalChunked" +} \ No newline at end of file diff --git a/results-truncation/embed_size_8192/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-truncation/embed_size_8192/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json new file mode 100644 index 0000000..ec08f1f --- /dev/null +++ b/results-truncation/embed_size_8192/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json @@ -0,0 +1,43 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 42.96391987800598, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.88557, + "map_at_1": 0.78571, + "map_at_10": 0.85666, + "map_at_20": 0.85776, + "map_at_3": 0.84821, + "map_at_5": 0.85446, + "mrr_at_1": 0.7857142857142857, + "mrr_at_10": 0.8566633597883598, + "mrr_at_20": 0.8577582800797087, + "mrr_at_3": 0.8482142857142857, + "mrr_at_5": 0.8544642857142858, + "ndcg_at_1": 0.78571, + "ndcg_at_10": 0.88557, + "ndcg_at_20": 0.88941, + "ndcg_at_3": 0.86883, + "ndcg_at_5": 0.87998, + "precision_at_1": 0.78571, + "precision_at_10": 0.09732, + "precision_at_20": 0.0494, + "precision_at_3": 0.30952, + "precision_at_5": 0.19107, + "recall_at_1": 0.78571, + "recall_at_10": 0.97321, + "recall_at_20": 0.9881, + "recall_at_3": 0.92857, + "recall_at_5": 0.95536 + } + ] + }, + "task_name": "LEMBSummScreenFDRetrievalChunked" +} \ No newline at end of file diff --git a/results-truncation/embed_size_8192/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-truncation/embed_size_8192/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json index f34aedb..7d1c0fb 100644 --- a/results-truncation/embed_size_8192/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ b/results-truncation/embed_size_8192/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -1,6 +1,6 @@ { - "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", - "evaluation_time": 32.949374198913574, + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 36.08832788467407, "kg_co2_emissions": null, "mteb_version": "1.14.20", "scores": { From 9043a3fb5283a9600b314060911c0f565e167f0c Mon Sep 17 00:00:00 2001 From: dannyjameswilliams Date: Thu, 26 Sep 2024 16:40:54 +0100 Subject: [PATCH 12/30] experiments --- parse_experiments.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/parse_experiments.py b/parse_experiments.py index ddc4d91..32e204a 100644 --- a/parse_experiments.py +++ b/parse_experiments.py @@ -71,7 +71,7 @@ ax.set_ylabel("nCDG@10", fontsize=14) ax.legend(loc='best', fontsize=12) - ax.axhline(truncated_boundary_ndcg, label=f"Truncated Boundary (at 8192)", color="k", linestyle='-.', linewidth=2) + # ax.axhline(truncated_boundary_ndcg, label=f"Truncated Boundary (at 8192)", color="k", linestyle='-.', linewidth=2) ax.set_xlabel("Overlap Size", fontsize=14) fig.suptitle("NDCG@10 for Different Embed Sizes and Overlap Sizes", fontsize=18) fig.tight_layout(rect=[0, 0, 1, 0.96]) @@ -95,7 +95,22 @@ ax.legend(loc='best', fontsize=12) - ax.axhline(truncated_boundary_map, label=f"Truncated Boundary (at 8192)", color="k", linestyle='-.', linewidth=2) + # ax.axhline(truncated_boundary_map, label=f"Truncated Boundary (at 8192)", color="k", linestyle='-.', linewidth=2) ax.set_xlabel("Overlap Size", fontsize=14) fig.suptitle("mAP@10 for Different Embed Sizes and Overlap Sizes", fontsize=18) - fig.tight_layout(rect=[0, 0, 1, 0.96]) \ No newline at end of file + fig.tight_layout(rect=[0, 0, 1, 0.96]) + + + + from datasets import load_dataset + name = "narrativeqa" + dataset = load_dataset(path="dwzhu/LongEmbed", name=name, split="corpus") + print(dataset.info.download_checksums) + + name = "summ_screen_fd" + dataset = load_dataset(path="dwzhu/LongEmbed", name=name, split="corpus") + print(dataset.info.download_checksums) + + name = "qmsum" + dataset = load_dataset(path="dwzhu/LongEmbed", name=name, split="corpus") + print(dataset.info.download_checksums) \ No newline at end of file From 0a4a27eb93a7583065f73986a58ac17f90dc4e6c Mon Sep 17 00:00:00 2001 From: dannyjameswilliams Date: Thu, 26 Sep 2024 18:02:36 +0100 Subject: [PATCH 13/30] added macro chunk experiment file --- run_chunked_eval_with_macro_chunks.py | 170 ++++++++++++++++++++++++++ soft_token_boundary.py | 11 +- 2 files changed, 177 insertions(+), 4 deletions(-) create mode 100644 run_chunked_eval_with_macro_chunks.py diff --git a/run_chunked_eval_with_macro_chunks.py b/run_chunked_eval_with_macro_chunks.py new file mode 100644 index 0000000..661e46f --- /dev/null +++ b/run_chunked_eval_with_macro_chunks.py @@ -0,0 +1,170 @@ +import click +import torch.cuda +from mteb import MTEB +from transformers import AutoModel, AutoTokenizer + +from chunked_pooling.chunked_eval_tasks import * +from chunked_pooling.wrappers import load_model + +DEFAULT_CHUNKING_STRATEGY = 'fixed' +DEFAULT_N_SENTENCES = 5 +BATCH_SIZE = 1 +DEFAULT_OVERLAP_SIZE = 256 +DEFAULT_SOFT_BOUNDARY_EMBED_SIZE = 8192 +DEFAULT_HARD_BOUNDARY_EMBED_SIZE = 0 + + +@click.command() +@click.option( + '--model-name', + default='jinaai/jina-embeddings-v2-small-en', + help='The name of the model to use.', +) +@click.option( + '--strategy', + default=DEFAULT_CHUNKING_STRATEGY, + help='The chunking strategy to be applied.', +) +@click.option( + '--task-name', default='LEMBWikimQARetrievalChunked', help='The evaluation task to perform.' +) +@click.option( + '--eval-split', default='test', help='The name of the evaluation split in the task.' +) +@click.option( + '--chunking-model', + default=None, + required=False, + help='The name of the model used for semantic chunking.', +) +@click.option( + '--truncate-max-length', + default=None, + type=int, + help='Maximum number of tokens; By default, no truncation is done.', +) +@click.option( + '--n-sentences', + default=DEFAULT_N_SENTENCES, + type=int, + help='Number of sentences per chunk for sentence strategy.', +) +@click.option( + '--soft-boundary-embed-size', + default=DEFAULT_SOFT_BOUNDARY_EMBED_SIZE, + type=int, + help='Token length of the embeddings that come before/after soft boundaries (i.e. overlapping embeddings). Above zero, soft boundaries are used.', +) +@click.option( + '--hard-boundary-embed-size', + default=DEFAULT_HARD_BOUNDARY_EMBED_SIZE, + type=int, + help='Token length of the embeddings that come before/after hard boundaries. Above zero, hard boundaries are used.', +) +@click.option( + '--overlap-size', + default=DEFAULT_OVERLAP_SIZE, + type=int, + help='Number of tokens per chunk for fixed strategy.', +) + +def main( + model_name, + strategy, + task_name, + eval_split, + chunking_model, + truncate_max_length, + n_sentences, + soft_boundary_embed_size, + hard_boundary_embed_size, + overlap_size, +): + try: + task_cls = globals()[task_name] + except: + raise ValueError(f'Unknown task name: {task_name}') + + model, has_instructions = load_model(model_name) + + tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True) + + if torch.cuda.is_available(): + model = model.cuda() + + model.eval() + + chunk_sizes = [8, 16, 32, 64, 128, 256, 512, 1024] + for chunk_size in chunk_sizes: + + print(f'\n\n\n\nEvaluating chunk size: {chunk_size} ({chunk_sizes.index(chunk_size) + 1}/{len(chunk_sizes)}) \n\n\n\n') + + chunking_args = { + 'chunk_size': chunk_size, + 'n_sentences': n_sentences, + 'chunking_strategy': strategy, + 'model_has_instructions': has_instructions, + 'embedding_model_name': chunking_model if chunking_model else model_name, + } + + # == Late Chunking == + tasks = [ + task_cls( + chunked_pooling_enabled=True, + tokenizer=tokenizer, + prune_size=None, + truncate_max_length=truncate_max_length, + soft_boundary_embed_size=soft_boundary_embed_size, + soft_boundary_overlap_size=overlap_size, + hard_boundary_embed_size=hard_boundary_embed_size, + **chunking_args, + ) + ] + + evaluation = MTEB( + tasks=tasks, + chunked_pooling_enabled=True, + tokenizer=tokenizer, + prune_size=None, + **chunking_args, + ) + evaluation.run( + model, + output_folder=f'results-chunked-pooling/chunk_size_{chunk_size}', + eval_splits=[eval_split], + overwrite_results=True, + batch_size=BATCH_SIZE, + encode_kwargs={'batch_size': BATCH_SIZE}, + ) + + # == Naive Chunking == + # naive chunking does not need soft boundaries because chunk size is guaranteed to be <8192 tokens + tasks = [ + task_cls( + chunked_pooling_enabled=False, + tokenizer=tokenizer, + prune_size=None, + truncate_max_length=truncate_max_length, + **chunking_args, + ) + ] + + evaluation = MTEB( + tasks=tasks, + chunked_pooling_enabled=False, + tokenizer=tokenizer, + prune_size=None, + **chunking_args, + ) + evaluation.run( + model, + output_folder=f'results-normal-pooling/chunk_size_{chunk_size}', + eval_splits=[eval_split], + overwrite_results=True, + batch_size=BATCH_SIZE, + encode_kwargs={'batch_size': BATCH_SIZE}, + ) + + +if __name__ == '__main__': + main() diff --git a/soft_token_boundary.py b/soft_token_boundary.py index 88859e3..4a3747a 100644 --- a/soft_token_boundary.py +++ b/soft_token_boundary.py @@ -33,6 +33,9 @@ DEFAULT_CHUNK_SIZE = 256 DEFAULT_N_SENTENCES = 5 BATCH_SIZE = 1 +DEFAULT_SOFT_BOUNDARY_EMBED_SIZE = 8192 +DEFAULT_HARD_BOUNDARY_EMBED_SIZE = 0 + @click.command() @@ -77,14 +80,14 @@ help='Number of sentences per chunk for sentence strategy.', ) @click.option( - '--soft_boundary_embed_size', - default=8192, + '--soft-boundary-embed-size', + default=DEFAULT_SOFT_BOUNDARY_EMBED_SIZE, type=int, help='Token length of the embeddings that come before/after soft boundaries (i.e. overlapping embeddings). Above zero, soft boundaries are used.', ) @click.option( - '--hard_boundary_embed_size', - default=8192, + '--hard-boundary-embed-size', + default=DEFAULT_HARD_BOUNDARY_EMBED_SIZE, type=int, help='Token length of the embeddings that come before/after hard boundaries. Above zero, hard boundaries are used.', ) From 043eec6622439cef66f7c8b47bba38ab0faf02b6 Mon Sep 17 00:00:00 2001 From: dannyjameswilliams Date: Thu, 26 Sep 2024 18:03:10 +0100 Subject: [PATCH 14/30] for merge --- soft_token_boundary.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/soft_token_boundary.py b/soft_token_boundary.py index 88859e3..2b54449 100644 --- a/soft_token_boundary.py +++ b/soft_token_boundary.py @@ -26,7 +26,13 @@ from mteb import MTEB from transformers import AutoModel, AutoTokenizer -from chunked_pooling.chunked_eval_tasks import LEMBWikimQARetrievalChunked, LEMBNeedleRetrievalChunked +from chunked_pooling.chunked_eval_tasks import ( + LEMBWikimQARetrievalChunked, + LEMBNeedleRetrievalChunked, + LEMBNarrativeQARetrievalChunked, + LEMBQMSumRetrievalChunked, + LEMBSummScreenFDRetrievalChunked +) from chunked_pooling.wrappers import load_model DEFAULT_CHUNKING_STRATEGY = 'fixed' @@ -77,13 +83,13 @@ help='Number of sentences per chunk for sentence strategy.', ) @click.option( - '--soft_boundary_embed_size', + '--soft-boundary-embed-size', default=8192, type=int, help='Token length of the embeddings that come before/after soft boundaries (i.e. overlapping embeddings). Above zero, soft boundaries are used.', ) @click.option( - '--hard_boundary_embed_size', + '--hard-boundary-embed-size', default=8192, type=int, help='Token length of the embeddings that come before/after hard boundaries. Above zero, hard boundaries are used.', @@ -178,6 +184,7 @@ def main( prune_size=None, **chunking_args, ) + evaluation.run( model, output_folder=f'results-hard-boundary/embed_size_{hard_boundary_embed_size}', @@ -216,5 +223,7 @@ def main( encode_kwargs={'batch_size': BATCH_SIZE}, ) + x=1 + if __name__ == '__main__': main() From 8e0c80d3323cfa98076afe1ad5f8f8b35dcb198d Mon Sep 17 00:00:00 2001 From: dannyjameswilliams Date: Thu, 26 Sep 2024 19:54:57 +0100 Subject: [PATCH 15/30] chunk size results --- .../LEMBSummScreenFDRetrievalChunked.json | 43 +++++++++++++++++++ .../LEMBWikimQARetrievalChunked.json | 43 +++++++++++++++++++ .../no_revision_available/model_meta.json | 1 + .../LEMBSummScreenFDRetrievalChunked.json | 43 +++++++++++++++++++ .../LEMBWikimQARetrievalChunked.json | 43 +++++++++++++++++++ .../no_revision_available/model_meta.json | 1 + .../LEMBSummScreenFDRetrievalChunked.json | 43 +++++++++++++++++++ .../LEMBWikimQARetrievalChunked.json | 38 ++++++++++++++++ .../no_revision_available/model_meta.json | 1 + .../LEMBSummScreenFDRetrievalChunked.json | 38 ++++++++++++++++ .../LEMBWikimQARetrievalChunked.json | 38 ++++++++++++++++ .../no_revision_available/model_meta.json | 1 + .../LEMBSummScreenFDRetrievalChunked.json | 38 ++++++++++++++++ .../LEMBWikimQARetrievalChunked.json | 43 +++++++++++++++++++ .../no_revision_available/model_meta.json | 1 + .../LEMBSummScreenFDRetrievalChunked.json | 43 +++++++++++++++++++ .../LEMBWikimQARetrievalChunked.json | 43 +++++++++++++++++++ .../no_revision_available/model_meta.json | 1 + .../LEMBSummScreenFDRetrievalChunked.json | 43 +++++++++++++++++++ .../LEMBWikimQARetrievalChunked.json | 43 +++++++++++++++++++ .../no_revision_available/model_meta.json | 1 + .../LEMBQMSumRetrievalChunked.json | 43 +++++++++++++++++++ .../LEMBSummScreenFDRetrievalChunked.json | 43 +++++++++++++++++++ .../LEMBWikimQARetrievalChunked.json | 43 +++++++++++++++++++ .../no_revision_available/model_meta.json | 1 + .../LEMBSummScreenFDRetrievalChunked.json | 43 +++++++++++++++++++ .../LEMBWikimQARetrievalChunked.json | 43 +++++++++++++++++++ .../no_revision_available/model_meta.json | 1 + .../LEMBSummScreenFDRetrievalChunked.json | 43 +++++++++++++++++++ .../LEMBWikimQARetrievalChunked.json | 43 +++++++++++++++++++ .../no_revision_available/model_meta.json | 1 + .../LEMBSummScreenFDRetrievalChunked.json | 43 +++++++++++++++++++ .../LEMBWikimQARetrievalChunked.json | 38 ++++++++++++++++ .../no_revision_available/model_meta.json | 1 + .../LEMBSummScreenFDRetrievalChunked.json | 38 ++++++++++++++++ .../LEMBWikimQARetrievalChunked.json | 38 ++++++++++++++++ .../no_revision_available/model_meta.json | 1 + .../LEMBSummScreenFDRetrievalChunked.json | 38 ++++++++++++++++ .../LEMBWikimQARetrievalChunked.json | 43 +++++++++++++++++++ .../no_revision_available/model_meta.json | 1 + .../LEMBSummScreenFDRetrievalChunked.json | 43 +++++++++++++++++++ .../LEMBWikimQARetrievalChunked.json | 43 +++++++++++++++++++ .../no_revision_available/model_meta.json | 1 + .../LEMBSummScreenFDRetrievalChunked.json | 43 +++++++++++++++++++ .../LEMBWikimQARetrievalChunked.json | 43 +++++++++++++++++++ .../no_revision_available/model_meta.json | 1 + .../LEMBSummScreenFDRetrievalChunked.json | 43 +++++++++++++++++++ .../LEMBWikimQARetrievalChunked.json | 43 +++++++++++++++++++ .../no_revision_available/model_meta.json | 1 + .../LEMBWikimQARetrievalChunked.json | 2 +- .../LEMBWikimQARetrievalChunked.json | 2 +- 51 files changed, 1397 insertions(+), 2 deletions(-) create mode 100644 results-chunked-pooling/chunk_size_1024/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json create mode 100644 results-chunked-pooling/chunk_size_1024/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json create mode 100644 results-chunked-pooling/chunk_size_1024/no_model_name_available/no_revision_available/model_meta.json create mode 100644 results-chunked-pooling/chunk_size_128/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json create mode 100644 results-chunked-pooling/chunk_size_128/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json create mode 100644 results-chunked-pooling/chunk_size_128/no_model_name_available/no_revision_available/model_meta.json create mode 100644 results-chunked-pooling/chunk_size_16/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json create mode 100644 results-chunked-pooling/chunk_size_16/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json create mode 100644 results-chunked-pooling/chunk_size_16/no_model_name_available/no_revision_available/model_meta.json create mode 100644 results-chunked-pooling/chunk_size_256/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json create mode 100644 results-chunked-pooling/chunk_size_256/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json create mode 100644 results-chunked-pooling/chunk_size_256/no_model_name_available/no_revision_available/model_meta.json create mode 100644 results-chunked-pooling/chunk_size_32/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json create mode 100644 results-chunked-pooling/chunk_size_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json create mode 100644 results-chunked-pooling/chunk_size_32/no_model_name_available/no_revision_available/model_meta.json create mode 100644 results-chunked-pooling/chunk_size_512/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json create mode 100644 results-chunked-pooling/chunk_size_512/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json create mode 100644 results-chunked-pooling/chunk_size_512/no_model_name_available/no_revision_available/model_meta.json create mode 100644 results-chunked-pooling/chunk_size_64/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json create mode 100644 results-chunked-pooling/chunk_size_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json create mode 100644 results-chunked-pooling/chunk_size_64/no_model_name_available/no_revision_available/model_meta.json create mode 100644 results-chunked-pooling/chunk_size_8/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json create mode 100644 results-chunked-pooling/chunk_size_8/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json create mode 100644 results-chunked-pooling/chunk_size_8/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json create mode 100644 results-chunked-pooling/chunk_size_8/no_model_name_available/no_revision_available/model_meta.json create mode 100644 results-normal-pooling/chunk_size_1024/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json create mode 100644 results-normal-pooling/chunk_size_1024/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json create mode 100644 results-normal-pooling/chunk_size_1024/no_model_name_available/no_revision_available/model_meta.json create mode 100644 results-normal-pooling/chunk_size_128/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json create mode 100644 results-normal-pooling/chunk_size_128/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json create mode 100644 results-normal-pooling/chunk_size_128/no_model_name_available/no_revision_available/model_meta.json create mode 100644 results-normal-pooling/chunk_size_16/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json create mode 100644 results-normal-pooling/chunk_size_16/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json create mode 100644 results-normal-pooling/chunk_size_16/no_model_name_available/no_revision_available/model_meta.json create mode 100644 results-normal-pooling/chunk_size_256/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json create mode 100644 results-normal-pooling/chunk_size_256/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json create mode 100644 results-normal-pooling/chunk_size_256/no_model_name_available/no_revision_available/model_meta.json create mode 100644 results-normal-pooling/chunk_size_32/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json create mode 100644 results-normal-pooling/chunk_size_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json create mode 100644 results-normal-pooling/chunk_size_32/no_model_name_available/no_revision_available/model_meta.json create mode 100644 results-normal-pooling/chunk_size_512/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json create mode 100644 results-normal-pooling/chunk_size_512/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json create mode 100644 results-normal-pooling/chunk_size_512/no_model_name_available/no_revision_available/model_meta.json create mode 100644 results-normal-pooling/chunk_size_64/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json create mode 100644 results-normal-pooling/chunk_size_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json create mode 100644 results-normal-pooling/chunk_size_64/no_model_name_available/no_revision_available/model_meta.json create mode 100644 results-normal-pooling/chunk_size_8/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json create mode 100644 results-normal-pooling/chunk_size_8/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json create mode 100644 results-normal-pooling/chunk_size_8/no_model_name_available/no_revision_available/model_meta.json diff --git a/results-chunked-pooling/chunk_size_1024/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-chunked-pooling/chunk_size_1024/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json new file mode 100644 index 0000000..a049782 --- /dev/null +++ b/results-chunked-pooling/chunk_size_1024/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json @@ -0,0 +1,43 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 42.20820331573486, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.89797, + "map_at_1": 0.80952, + "map_at_10": 0.87148, + "map_at_20": 0.87238, + "map_at_3": 0.8631, + "map_at_5": 0.86771, + "mrr_at_1": 0.8095238095238095, + "mrr_at_10": 0.8714781746031746, + "mrr_at_20": 0.8723811778499279, + "mrr_at_3": 0.8630952380952381, + "mrr_at_5": 0.8677083333333334, + "ndcg_at_1": 0.80952, + "ndcg_at_10": 0.89797, + "ndcg_at_20": 0.90107, + "ndcg_at_3": 0.88067, + "ndcg_at_5": 0.88912, + "precision_at_1": 0.80952, + "precision_at_10": 0.09792, + "precision_at_20": 0.04955, + "precision_at_3": 0.31052, + "precision_at_5": 0.19048, + "recall_at_1": 0.80952, + "recall_at_10": 0.97917, + "recall_at_20": 0.99107, + "recall_at_3": 0.93155, + "recall_at_5": 0.95238 + } + ] + }, + "task_name": "LEMBSummScreenFDRetrievalChunked" +} \ No newline at end of file diff --git a/results-chunked-pooling/chunk_size_1024/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-chunked-pooling/chunk_size_1024/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json new file mode 100644 index 0000000..90095ac --- /dev/null +++ b/results-chunked-pooling/chunk_size_1024/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -0,0 +1,43 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 37.14106869697571, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.89865, + "map_at_1": 0.84, + "map_at_10": 0.88085, + "map_at_20": 0.8821, + "map_at_3": 0.87389, + "map_at_5": 0.87856, + "mrr_at_1": 0.84, + "mrr_at_10": 0.8808465608465609, + "mrr_at_20": 0.8820996378349318, + "mrr_at_3": 0.8738888888888889, + "mrr_at_5": 0.8785555555555555, + "ndcg_at_1": 0.84, + "ndcg_at_10": 0.89865, + "ndcg_at_20": 0.90299, + "ndcg_at_3": 0.88488, + "ndcg_at_5": 0.8932, + "precision_at_1": 0.84, + "precision_at_10": 0.09533, + "precision_at_20": 0.0485, + "precision_at_3": 0.30556, + "precision_at_5": 0.18733, + "recall_at_1": 0.84, + "recall_at_10": 0.95333, + "recall_at_20": 0.97, + "recall_at_3": 0.91667, + "recall_at_5": 0.93667 + } + ] + }, + "task_name": "LEMBWikimQARetrievalChunked" +} \ No newline at end of file diff --git a/results-chunked-pooling/chunk_size_1024/no_model_name_available/no_revision_available/model_meta.json b/results-chunked-pooling/chunk_size_1024/no_model_name_available/no_revision_available/model_meta.json new file mode 100644 index 0000000..40ff52f --- /dev/null +++ b/results-chunked-pooling/chunk_size_1024/no_model_name_available/no_revision_available/model_meta.json @@ -0,0 +1 @@ +{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-chunked-pooling/chunk_size_128/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-chunked-pooling/chunk_size_128/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json new file mode 100644 index 0000000..8614548 --- /dev/null +++ b/results-chunked-pooling/chunk_size_128/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json @@ -0,0 +1,43 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 46.49887251853943, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.89057, + "map_at_1": 0.79167, + "map_at_10": 0.86152, + "map_at_20": 0.8624, + "map_at_3": 0.85069, + "map_at_5": 0.85873, + "mrr_at_1": 0.7916666666666666, + "mrr_at_10": 0.8615197467876039, + "mrr_at_20": 0.8624014915309559, + "mrr_at_3": 0.8506944444444444, + "mrr_at_5": 0.8587301587301588, + "ndcg_at_1": 0.79167, + "ndcg_at_10": 0.89057, + "ndcg_at_20": 0.89365, + "ndcg_at_3": 0.86922, + "ndcg_at_5": 0.88382, + "precision_at_1": 0.79167, + "precision_at_10": 0.09792, + "precision_at_20": 0.04955, + "precision_at_3": 0.30754, + "precision_at_5": 0.19167, + "recall_at_1": 0.79167, + "recall_at_10": 0.97917, + "recall_at_20": 0.99107, + "recall_at_3": 0.92262, + "recall_at_5": 0.95833 + } + ] + }, + "task_name": "LEMBSummScreenFDRetrievalChunked" +} \ No newline at end of file diff --git a/results-chunked-pooling/chunk_size_128/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-chunked-pooling/chunk_size_128/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json new file mode 100644 index 0000000..ce12f18 --- /dev/null +++ b/results-chunked-pooling/chunk_size_128/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -0,0 +1,43 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 41.79281163215637, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.96324, + "map_at_1": 0.93667, + "map_at_10": 0.95468, + "map_at_20": 0.95486, + "map_at_3": 0.95222, + "map_at_5": 0.95222, + "mrr_at_1": 0.9366666666666666, + "mrr_at_10": 0.9546759259259259, + "mrr_at_20": 0.954861111111111, + "mrr_at_3": 0.9522222222222223, + "mrr_at_5": 0.9522222222222223, + "ndcg_at_1": 0.93667, + "ndcg_at_10": 0.96324, + "ndcg_at_20": 0.96402, + "ndcg_at_3": 0.95762, + "ndcg_at_5": 0.95762, + "precision_at_1": 0.93667, + "precision_at_10": 0.099, + "precision_at_20": 0.04967, + "precision_at_3": 0.32444, + "precision_at_5": 0.19467, + "recall_at_1": 0.93667, + "recall_at_10": 0.99, + "recall_at_20": 0.99333, + "recall_at_3": 0.97333, + "recall_at_5": 0.97333 + } + ] + }, + "task_name": "LEMBWikimQARetrievalChunked" +} \ No newline at end of file diff --git a/results-chunked-pooling/chunk_size_128/no_model_name_available/no_revision_available/model_meta.json b/results-chunked-pooling/chunk_size_128/no_model_name_available/no_revision_available/model_meta.json new file mode 100644 index 0000000..40ff52f --- /dev/null +++ b/results-chunked-pooling/chunk_size_128/no_model_name_available/no_revision_available/model_meta.json @@ -0,0 +1 @@ +{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-chunked-pooling/chunk_size_16/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-chunked-pooling/chunk_size_16/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json new file mode 100644 index 0000000..66fa4c2 --- /dev/null +++ b/results-chunked-pooling/chunk_size_16/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json @@ -0,0 +1,43 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 100.60042762756348, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.91536, + "map_at_1": 0.82738, + "map_at_10": 0.89029, + "map_at_20": 0.89049, + "map_at_3": 0.88046, + "map_at_5": 0.88805, + "mrr_at_1": 0.8273809523809523, + "mrr_at_10": 0.8902895880574452, + "mrr_at_20": 0.8904880007558579, + "mrr_at_3": 0.8804563492063492, + "mrr_at_5": 0.8880456349206349, + "ndcg_at_1": 0.82738, + "ndcg_at_10": 0.91536, + "ndcg_at_20": 0.91611, + "ndcg_at_3": 0.89672, + "ndcg_at_5": 0.91029, + "precision_at_1": 0.82738, + "precision_at_10": 0.09911, + "precision_at_20": 0.0497, + "precision_at_3": 0.31448, + "precision_at_5": 0.19524, + "recall_at_1": 0.82738, + "recall_at_10": 0.99107, + "recall_at_20": 0.99405, + "recall_at_3": 0.94345, + "recall_at_5": 0.97619 + } + ] + }, + "task_name": "LEMBSummScreenFDRetrievalChunked" +} \ No newline at end of file diff --git a/results-chunked-pooling/chunk_size_16/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-chunked-pooling/chunk_size_16/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json new file mode 100644 index 0000000..c26076b --- /dev/null +++ b/results-chunked-pooling/chunk_size_16/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -0,0 +1,38 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 61.05208373069763, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.96675, + "map_at_1": 0.94333, + "map_at_10": 0.96026, + "map_at_3": 0.95889, + "map_at_5": 0.95956, + "mrr_at_1": 0.9433333333333334, + "mrr_at_10": 0.9602592592592593, + "mrr_at_3": 0.9588888888888889, + "mrr_at_5": 0.9595555555555556, + "ndcg_at_1": 0.94333, + "ndcg_at_10": 0.96675, + "ndcg_at_3": 0.96349, + "ndcg_at_5": 0.96478, + "precision_at_1": 0.94333, + "precision_at_10": 0.09867, + "precision_at_3": 0.32556, + "precision_at_5": 0.196, + "recall_at_1": 0.94333, + "recall_at_10": 0.98667, + "recall_at_3": 0.97667, + "recall_at_5": 0.98 + } + ] + }, + "task_name": "LEMBWikimQARetrievalChunked" +} \ No newline at end of file diff --git a/results-chunked-pooling/chunk_size_16/no_model_name_available/no_revision_available/model_meta.json b/results-chunked-pooling/chunk_size_16/no_model_name_available/no_revision_available/model_meta.json new file mode 100644 index 0000000..40ff52f --- /dev/null +++ b/results-chunked-pooling/chunk_size_16/no_model_name_available/no_revision_available/model_meta.json @@ -0,0 +1 @@ +{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-chunked-pooling/chunk_size_256/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-chunked-pooling/chunk_size_256/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json new file mode 100644 index 0000000..6c629a6 --- /dev/null +++ b/results-chunked-pooling/chunk_size_256/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json @@ -0,0 +1,38 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 42.10010838508606, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.8896, + "map_at_1": 0.79464, + "map_at_10": 0.86285, + "map_at_3": 0.85466, + "map_at_5": 0.86032, + "mrr_at_1": 0.7946428571428571, + "mrr_at_10": 0.8628472222222223, + "mrr_at_3": 0.8546626984126985, + "mrr_at_5": 0.8603174603174603, + "ndcg_at_1": 0.79464, + "ndcg_at_10": 0.8896, + "ndcg_at_3": 0.87369, + "ndcg_at_5": 0.88368, + "precision_at_1": 0.79464, + "precision_at_10": 0.09702, + "precision_at_3": 0.30952, + "precision_at_5": 0.19048, + "recall_at_1": 0.79464, + "recall_at_10": 0.97024, + "recall_at_3": 0.92857, + "recall_at_5": 0.95238 + } + ] + }, + "task_name": "LEMBSummScreenFDRetrievalChunked" +} \ No newline at end of file diff --git a/results-chunked-pooling/chunk_size_256/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-chunked-pooling/chunk_size_256/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json new file mode 100644 index 0000000..e301d83 --- /dev/null +++ b/results-chunked-pooling/chunk_size_256/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -0,0 +1,38 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 38.12126922607422, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.94608, + "map_at_1": 0.90667, + "map_at_10": 0.93403, + "map_at_3": 0.93, + "map_at_5": 0.93233, + "mrr_at_1": 0.9066666666666666, + "mrr_at_10": 0.9340317460317461, + "mrr_at_3": 0.93, + "mrr_at_5": 0.9323333333333333, + "ndcg_at_1": 0.90667, + "ndcg_at_10": 0.94608, + "ndcg_at_3": 0.9377, + "ndcg_at_5": 0.94186, + "precision_at_1": 0.90667, + "precision_at_10": 0.09833, + "precision_at_3": 0.32, + "precision_at_5": 0.194, + "recall_at_1": 0.90667, + "recall_at_10": 0.98333, + "recall_at_3": 0.96, + "recall_at_5": 0.97 + } + ] + }, + "task_name": "LEMBWikimQARetrievalChunked" +} \ No newline at end of file diff --git a/results-chunked-pooling/chunk_size_256/no_model_name_available/no_revision_available/model_meta.json b/results-chunked-pooling/chunk_size_256/no_model_name_available/no_revision_available/model_meta.json new file mode 100644 index 0000000..40ff52f --- /dev/null +++ b/results-chunked-pooling/chunk_size_256/no_model_name_available/no_revision_available/model_meta.json @@ -0,0 +1 @@ +{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-chunked-pooling/chunk_size_32/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-chunked-pooling/chunk_size_32/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json new file mode 100644 index 0000000..261d110 --- /dev/null +++ b/results-chunked-pooling/chunk_size_32/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json @@ -0,0 +1,38 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 56.322009563446045, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.89977, + "map_at_1": 0.80357, + "map_at_10": 0.87167, + "map_at_3": 0.8626, + "map_at_5": 0.8687, + "mrr_at_1": 0.8035714285714286, + "mrr_at_10": 0.8716671390778533, + "mrr_at_3": 0.8625992063492063, + "mrr_at_5": 0.8687003968253968, + "ndcg_at_1": 0.80357, + "ndcg_at_10": 0.89977, + "ndcg_at_3": 0.88184, + "ndcg_at_5": 0.89285, + "precision_at_1": 0.80357, + "precision_at_10": 0.09851, + "precision_at_3": 0.3125, + "precision_at_5": 0.19286, + "recall_at_1": 0.80357, + "recall_at_10": 0.98512, + "recall_at_3": 0.9375, + "recall_at_5": 0.96429 + } + ] + }, + "task_name": "LEMBSummScreenFDRetrievalChunked" +} \ No newline at end of file diff --git a/results-chunked-pooling/chunk_size_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-chunked-pooling/chunk_size_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json new file mode 100644 index 0000000..b7a6aff --- /dev/null +++ b/results-chunked-pooling/chunk_size_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -0,0 +1,43 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 49.683130741119385, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.9641, + "map_at_1": 0.93333, + "map_at_10": 0.95644, + "map_at_20": 0.95715, + "map_at_3": 0.95611, + "map_at_5": 0.95611, + "mrr_at_1": 0.9333333333333333, + "mrr_at_10": 0.9564444444444444, + "mrr_at_20": 0.9571522366522366, + "mrr_at_3": 0.956111111111111, + "mrr_at_5": 0.956111111111111, + "ndcg_at_1": 0.93333, + "ndcg_at_10": 0.9641, + "ndcg_at_20": 0.96664, + "ndcg_at_3": 0.96313, + "ndcg_at_5": 0.96313, + "precision_at_1": 0.93333, + "precision_at_10": 0.09867, + "precision_at_20": 0.04983, + "precision_at_3": 0.32778, + "precision_at_5": 0.19667, + "recall_at_1": 0.93333, + "recall_at_10": 0.98667, + "recall_at_20": 0.99667, + "recall_at_3": 0.98333, + "recall_at_5": 0.98333 + } + ] + }, + "task_name": "LEMBWikimQARetrievalChunked" +} \ No newline at end of file diff --git a/results-chunked-pooling/chunk_size_32/no_model_name_available/no_revision_available/model_meta.json b/results-chunked-pooling/chunk_size_32/no_model_name_available/no_revision_available/model_meta.json new file mode 100644 index 0000000..40ff52f --- /dev/null +++ b/results-chunked-pooling/chunk_size_32/no_model_name_available/no_revision_available/model_meta.json @@ -0,0 +1 @@ +{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-chunked-pooling/chunk_size_512/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-chunked-pooling/chunk_size_512/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json new file mode 100644 index 0000000..8c96237 --- /dev/null +++ b/results-chunked-pooling/chunk_size_512/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json @@ -0,0 +1,43 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 41.24476718902588, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.86921, + "map_at_1": 0.75595, + "map_at_10": 0.83613, + "map_at_20": 0.83699, + "map_at_3": 0.82391, + "map_at_5": 0.8315, + "mrr_at_1": 0.7559523809523809, + "mrr_at_10": 0.8361335506424792, + "mrr_at_20": 0.8369887222565795, + "mrr_at_3": 0.8239087301587301, + "mrr_at_5": 0.8314980158730159, + "ndcg_at_1": 0.75595, + "ndcg_at_10": 0.86921, + "ndcg_at_20": 0.87226, + "ndcg_at_3": 0.84478, + "ndcg_at_5": 0.85835, + "precision_at_1": 0.75595, + "precision_at_10": 0.09702, + "precision_at_20": 0.04911, + "precision_at_3": 0.30159, + "precision_at_5": 0.1875, + "recall_at_1": 0.75595, + "recall_at_10": 0.97024, + "recall_at_20": 0.98214, + "recall_at_3": 0.90476, + "recall_at_5": 0.9375 + } + ] + }, + "task_name": "LEMBSummScreenFDRetrievalChunked" +} \ No newline at end of file diff --git a/results-chunked-pooling/chunk_size_512/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-chunked-pooling/chunk_size_512/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json new file mode 100644 index 0000000..188188a --- /dev/null +++ b/results-chunked-pooling/chunk_size_512/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -0,0 +1,43 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 37.502394676208496, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.93658, + "map_at_1": 0.89667, + "map_at_10": 0.92556, + "map_at_20": 0.92677, + "map_at_3": 0.92111, + "map_at_5": 0.92411, + "mrr_at_1": 0.8966666666666666, + "mrr_at_10": 0.9255595238095238, + "mrr_at_20": 0.9267734025234025, + "mrr_at_3": 0.9211111111111111, + "mrr_at_5": 0.9241111111111112, + "ndcg_at_1": 0.89667, + "ndcg_at_10": 0.93658, + "ndcg_at_20": 0.94087, + "ndcg_at_3": 0.92778, + "ndcg_at_5": 0.93323, + "precision_at_1": 0.89667, + "precision_at_10": 0.097, + "precision_at_20": 0.04933, + "precision_at_3": 0.31556, + "precision_at_5": 0.192, + "recall_at_1": 0.89667, + "recall_at_10": 0.97, + "recall_at_20": 0.98667, + "recall_at_3": 0.94667, + "recall_at_5": 0.96 + } + ] + }, + "task_name": "LEMBWikimQARetrievalChunked" +} \ No newline at end of file diff --git a/results-chunked-pooling/chunk_size_512/no_model_name_available/no_revision_available/model_meta.json b/results-chunked-pooling/chunk_size_512/no_model_name_available/no_revision_available/model_meta.json new file mode 100644 index 0000000..40ff52f --- /dev/null +++ b/results-chunked-pooling/chunk_size_512/no_model_name_available/no_revision_available/model_meta.json @@ -0,0 +1 @@ +{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-chunked-pooling/chunk_size_64/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-chunked-pooling/chunk_size_64/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json new file mode 100644 index 0000000..f1eab61 --- /dev/null +++ b/results-chunked-pooling/chunk_size_64/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json @@ -0,0 +1,43 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 49.614307165145874, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.8939, + "map_at_1": 0.79167, + "map_at_10": 0.86543, + "map_at_20": 0.86627, + "map_at_3": 0.85764, + "map_at_5": 0.86314, + "mrr_at_1": 0.7916666666666666, + "mrr_at_10": 0.8654301303854876, + "mrr_at_20": 0.8662683324924866, + "mrr_at_3": 0.857638888888889, + "mrr_at_5": 0.8631448412698413, + "ndcg_at_1": 0.79167, + "ndcg_at_10": 0.8939, + "ndcg_at_20": 0.89693, + "ndcg_at_3": 0.87893, + "ndcg_at_5": 0.88879, + "precision_at_1": 0.79167, + "precision_at_10": 0.09792, + "precision_at_20": 0.04955, + "precision_at_3": 0.31349, + "precision_at_5": 0.19286, + "recall_at_1": 0.79167, + "recall_at_10": 0.97917, + "recall_at_20": 0.99107, + "recall_at_3": 0.94048, + "recall_at_5": 0.96429 + } + ] + }, + "task_name": "LEMBSummScreenFDRetrievalChunked" +} \ No newline at end of file diff --git a/results-chunked-pooling/chunk_size_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-chunked-pooling/chunk_size_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json new file mode 100644 index 0000000..39ae3d1 --- /dev/null +++ b/results-chunked-pooling/chunk_size_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -0,0 +1,43 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 44.609771728515625, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.96349, + "map_at_1": 0.93333, + "map_at_10": 0.95572, + "map_at_20": 0.95623, + "map_at_3": 0.95389, + "map_at_5": 0.95539, + "mrr_at_1": 0.9333333333333333, + "mrr_at_10": 0.9557222222222221, + "mrr_at_20": 0.9562335858585859, + "mrr_at_3": 0.9538888888888889, + "mrr_at_5": 0.955388888888889, + "ndcg_at_1": 0.93333, + "ndcg_at_10": 0.96349, + "ndcg_at_20": 0.96523, + "ndcg_at_3": 0.9598, + "ndcg_at_5": 0.96253, + "precision_at_1": 0.93333, + "precision_at_10": 0.09867, + "precision_at_20": 0.04967, + "precision_at_3": 0.32556, + "precision_at_5": 0.19667, + "recall_at_1": 0.93333, + "recall_at_10": 0.98667, + "recall_at_20": 0.99333, + "recall_at_3": 0.97667, + "recall_at_5": 0.98333 + } + ] + }, + "task_name": "LEMBWikimQARetrievalChunked" +} \ No newline at end of file diff --git a/results-chunked-pooling/chunk_size_64/no_model_name_available/no_revision_available/model_meta.json b/results-chunked-pooling/chunk_size_64/no_model_name_available/no_revision_available/model_meta.json new file mode 100644 index 0000000..40ff52f --- /dev/null +++ b/results-chunked-pooling/chunk_size_64/no_model_name_available/no_revision_available/model_meta.json @@ -0,0 +1 @@ +{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-chunked-pooling/chunk_size_8/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json b/results-chunked-pooling/chunk_size_8/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json new file mode 100644 index 0000000..02c80b5 --- /dev/null +++ b/results-chunked-pooling/chunk_size_8/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json @@ -0,0 +1,43 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 441.941686630249, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.62432, + "map_at_1": 0.48134, + "map_at_10": 0.57465, + "map_at_20": 0.57982, + "map_at_3": 0.54923, + "map_at_5": 0.56285, + "mrr_at_1": 0.481335952848723, + "mrr_at_10": 0.5746509911539381, + "mrr_at_20": 0.5798247824096785, + "mrr_at_3": 0.549225060030561, + "mrr_at_5": 0.5628465400567562, + "ndcg_at_1": 0.48134, + "ndcg_at_10": 0.62432, + "ndcg_at_20": 0.64351, + "ndcg_at_3": 0.57116, + "ndcg_at_5": 0.59585, + "precision_at_1": 0.48134, + "precision_at_10": 0.07826, + "precision_at_20": 0.04296, + "precision_at_3": 0.21153, + "precision_at_5": 0.13897, + "recall_at_1": 0.48134, + "recall_at_10": 0.78258, + "recall_at_20": 0.8592, + "recall_at_3": 0.63458, + "recall_at_5": 0.69483 + } + ] + }, + "task_name": "LEMBQMSumRetrievalChunked" +} \ No newline at end of file diff --git a/results-chunked-pooling/chunk_size_8/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-chunked-pooling/chunk_size_8/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json new file mode 100644 index 0000000..29940f6 --- /dev/null +++ b/results-chunked-pooling/chunk_size_8/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json @@ -0,0 +1,43 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 139.2442500591278, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.91679, + "map_at_1": 0.83333, + "map_at_10": 0.89142, + "map_at_20": 0.89142, + "map_at_3": 0.87847, + "map_at_5": 0.88963, + "mrr_at_1": 0.8333333333333334, + "mrr_at_10": 0.8914186507936508, + "mrr_at_20": 0.8914186507936508, + "mrr_at_3": 0.8784722222222223, + "mrr_at_5": 0.8896329365079365, + "ndcg_at_1": 0.83333, + "ndcg_at_10": 0.91679, + "ndcg_at_20": 0.91679, + "ndcg_at_3": 0.89289, + "ndcg_at_5": 0.91275, + "precision_at_1": 0.83333, + "precision_at_10": 0.0994, + "precision_at_20": 0.0497, + "precision_at_3": 0.31151, + "precision_at_5": 0.19643, + "recall_at_1": 0.83333, + "recall_at_10": 0.99405, + "recall_at_20": 0.99405, + "recall_at_3": 0.93452, + "recall_at_5": 0.98214 + } + ] + }, + "task_name": "LEMBSummScreenFDRetrievalChunked" +} \ No newline at end of file diff --git a/results-chunked-pooling/chunk_size_8/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-chunked-pooling/chunk_size_8/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json new file mode 100644 index 0000000..157d1f3 --- /dev/null +++ b/results-chunked-pooling/chunk_size_8/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -0,0 +1,43 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 114.65108251571655, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.9728, + "map_at_1": 0.95, + "map_at_10": 0.96614, + "map_at_20": 0.96636, + "map_at_3": 0.96389, + "map_at_5": 0.96539, + "mrr_at_1": 0.95, + "mrr_at_10": 0.9661388888888888, + "mrr_at_20": 0.9663611111111112, + "mrr_at_3": 0.9638888888888889, + "mrr_at_5": 0.965388888888889, + "ndcg_at_1": 0.95, + "ndcg_at_10": 0.9728, + "ndcg_at_20": 0.97363, + "ndcg_at_3": 0.96806, + "ndcg_at_5": 0.97078, + "precision_at_1": 0.95, + "precision_at_10": 0.09933, + "precision_at_20": 0.04983, + "precision_at_3": 0.32667, + "precision_at_5": 0.19733, + "recall_at_1": 0.95, + "recall_at_10": 0.99333, + "recall_at_20": 0.99667, + "recall_at_3": 0.98, + "recall_at_5": 0.98667 + } + ] + }, + "task_name": "LEMBWikimQARetrievalChunked" +} \ No newline at end of file diff --git a/results-chunked-pooling/chunk_size_8/no_model_name_available/no_revision_available/model_meta.json b/results-chunked-pooling/chunk_size_8/no_model_name_available/no_revision_available/model_meta.json new file mode 100644 index 0000000..40ff52f --- /dev/null +++ b/results-chunked-pooling/chunk_size_8/no_model_name_available/no_revision_available/model_meta.json @@ -0,0 +1 @@ +{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-normal-pooling/chunk_size_1024/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-normal-pooling/chunk_size_1024/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json new file mode 100644 index 0000000..dac3893 --- /dev/null +++ b/results-normal-pooling/chunk_size_1024/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json @@ -0,0 +1,43 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 22.880125284194946, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.90436, + "map_at_1": 0.8244, + "map_at_10": 0.88092, + "map_at_20": 0.88152, + "map_at_3": 0.8745, + "map_at_5": 0.87793, + "mrr_at_1": 0.8244047619047619, + "mrr_at_10": 0.8809216742252456, + "mrr_at_20": 0.8815229248264962, + "mrr_at_3": 0.8745039682539681, + "mrr_at_5": 0.8779265873015873, + "ndcg_at_1": 0.8244, + "ndcg_at_10": 0.90436, + "ndcg_at_20": 0.90659, + "ndcg_at_3": 0.89069, + "ndcg_at_5": 0.89684, + "precision_at_1": 0.8244, + "precision_at_10": 0.09762, + "precision_at_20": 0.04926, + "precision_at_3": 0.3125, + "precision_at_5": 0.19048, + "recall_at_1": 0.8244, + "recall_at_10": 0.97619, + "recall_at_20": 0.98512, + "recall_at_3": 0.9375, + "recall_at_5": 0.95238 + } + ] + }, + "task_name": "LEMBSummScreenFDRetrievalChunked" +} \ No newline at end of file diff --git a/results-normal-pooling/chunk_size_1024/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-normal-pooling/chunk_size_1024/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json new file mode 100644 index 0000000..d39cb18 --- /dev/null +++ b/results-normal-pooling/chunk_size_1024/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -0,0 +1,43 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 21.713423252105713, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.89332, + "map_at_1": 0.81667, + "map_at_10": 0.87223, + "map_at_20": 0.87337, + "map_at_3": 0.86667, + "map_at_5": 0.86967, + "mrr_at_1": 0.8166666666666667, + "mrr_at_10": 0.8722261904761904, + "mrr_at_20": 0.8733667133520074, + "mrr_at_3": 0.8666666666666667, + "mrr_at_5": 0.8696666666666666, + "ndcg_at_1": 0.81667, + "ndcg_at_10": 0.89332, + "ndcg_at_20": 0.89751, + "ndcg_at_3": 0.88214, + "ndcg_at_5": 0.88759, + "precision_at_1": 0.81667, + "precision_at_10": 0.09567, + "precision_at_20": 0.04867, + "precision_at_3": 0.30889, + "precision_at_5": 0.188, + "recall_at_1": 0.81667, + "recall_at_10": 0.95667, + "recall_at_20": 0.97333, + "recall_at_3": 0.92667, + "recall_at_5": 0.94 + } + ] + }, + "task_name": "LEMBWikimQARetrievalChunked" +} \ No newline at end of file diff --git a/results-normal-pooling/chunk_size_1024/no_model_name_available/no_revision_available/model_meta.json b/results-normal-pooling/chunk_size_1024/no_model_name_available/no_revision_available/model_meta.json new file mode 100644 index 0000000..40ff52f --- /dev/null +++ b/results-normal-pooling/chunk_size_1024/no_model_name_available/no_revision_available/model_meta.json @@ -0,0 +1 @@ +{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-normal-pooling/chunk_size_128/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-normal-pooling/chunk_size_128/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json new file mode 100644 index 0000000..2c82561 --- /dev/null +++ b/results-normal-pooling/chunk_size_128/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json @@ -0,0 +1,43 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 45.15112638473511, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.8809, + "map_at_1": 0.75595, + "map_at_10": 0.84616, + "map_at_20": 0.84616, + "map_at_3": 0.83532, + "map_at_5": 0.84395, + "mrr_at_1": 0.7559523809523809, + "mrr_at_10": 0.8461592970521542, + "mrr_at_20": 0.8461592970521542, + "mrr_at_3": 0.8353174603174603, + "mrr_at_5": 0.8439484126984127, + "ndcg_at_1": 0.75595, + "ndcg_at_10": 0.8809, + "ndcg_at_20": 0.8809, + "ndcg_at_3": 0.86012, + "ndcg_at_5": 0.87587, + "precision_at_1": 0.75595, + "precision_at_10": 0.09851, + "precision_at_20": 0.04926, + "precision_at_3": 0.31052, + "precision_at_5": 0.19405, + "recall_at_1": 0.75595, + "recall_at_10": 0.98512, + "recall_at_20": 0.98512, + "recall_at_3": 0.93155, + "recall_at_5": 0.97024 + } + ] + }, + "task_name": "LEMBSummScreenFDRetrievalChunked" +} \ No newline at end of file diff --git a/results-normal-pooling/chunk_size_128/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-normal-pooling/chunk_size_128/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json new file mode 100644 index 0000000..beb491c --- /dev/null +++ b/results-normal-pooling/chunk_size_128/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -0,0 +1,43 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 41.67530345916748, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.96056, + "map_at_1": 0.93333, + "map_at_10": 0.95298, + "map_at_20": 0.9536, + "map_at_3": 0.95111, + "map_at_5": 0.95194, + "mrr_at_1": 0.9333333333333333, + "mrr_at_10": 0.9529761904761905, + "mrr_at_20": 0.9535960735171262, + "mrr_at_3": 0.951111111111111, + "mrr_at_5": 0.9519444444444444, + "ndcg_at_1": 0.93333, + "ndcg_at_10": 0.96056, + "ndcg_at_20": 0.963, + "ndcg_at_3": 0.95682, + "ndcg_at_5": 0.95826, + "precision_at_1": 0.93333, + "precision_at_10": 0.09833, + "precision_at_20": 0.04967, + "precision_at_3": 0.32444, + "precision_at_5": 0.19533, + "recall_at_1": 0.93333, + "recall_at_10": 0.98333, + "recall_at_20": 0.99333, + "recall_at_3": 0.97333, + "recall_at_5": 0.97667 + } + ] + }, + "task_name": "LEMBWikimQARetrievalChunked" +} \ No newline at end of file diff --git a/results-normal-pooling/chunk_size_128/no_model_name_available/no_revision_available/model_meta.json b/results-normal-pooling/chunk_size_128/no_model_name_available/no_revision_available/model_meta.json new file mode 100644 index 0000000..40ff52f --- /dev/null +++ b/results-normal-pooling/chunk_size_128/no_model_name_available/no_revision_available/model_meta.json @@ -0,0 +1 @@ +{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-normal-pooling/chunk_size_16/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-normal-pooling/chunk_size_16/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json new file mode 100644 index 0000000..1fd9a28 --- /dev/null +++ b/results-normal-pooling/chunk_size_16/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json @@ -0,0 +1,43 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 253.56742310523987, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.77601, + "map_at_1": 0.58929, + "map_at_10": 0.71954, + "map_at_20": 0.72214, + "map_at_3": 0.69544, + "map_at_5": 0.71255, + "mrr_at_1": 0.5892857142857143, + "mrr_at_10": 0.7195401077097505, + "mrr_at_20": 0.7221398856592078, + "mrr_at_3": 0.6954365079365079, + "mrr_at_5": 0.7125496031746031, + "ndcg_at_1": 0.58929, + "ndcg_at_10": 0.77601, + "ndcg_at_20": 0.78519, + "ndcg_at_3": 0.72867, + "ndcg_at_5": 0.75941, + "precision_at_1": 0.58929, + "precision_at_10": 0.09494, + "precision_at_20": 0.04926, + "precision_at_3": 0.2748, + "precision_at_5": 0.17976, + "recall_at_1": 0.58929, + "recall_at_10": 0.9494, + "recall_at_20": 0.98512, + "recall_at_3": 0.8244, + "recall_at_5": 0.89881 + } + ] + }, + "task_name": "LEMBSummScreenFDRetrievalChunked" +} \ No newline at end of file diff --git a/results-normal-pooling/chunk_size_16/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-normal-pooling/chunk_size_16/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json new file mode 100644 index 0000000..3ae2554 --- /dev/null +++ b/results-normal-pooling/chunk_size_16/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -0,0 +1,38 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 203.59319949150085, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.89535, + "map_at_1": 0.83, + "map_at_10": 0.87642, + "map_at_3": 0.87, + "map_at_5": 0.87367, + "mrr_at_1": 0.83, + "mrr_at_10": 0.8769761904761905, + "mrr_at_3": 0.8705555555555556, + "mrr_at_5": 0.8742222222222221, + "ndcg_at_1": 0.83, + "ndcg_at_10": 0.89535, + "ndcg_at_3": 0.88206, + "ndcg_at_5": 0.8888, + "precision_at_1": 0.83, + "precision_at_10": 0.09533, + "precision_at_3": 0.30556, + "precision_at_5": 0.18667, + "recall_at_1": 0.83, + "recall_at_10": 0.95333, + "recall_at_3": 0.91667, + "recall_at_5": 0.93333 + } + ] + }, + "task_name": "LEMBWikimQARetrievalChunked" +} \ No newline at end of file diff --git a/results-normal-pooling/chunk_size_16/no_model_name_available/no_revision_available/model_meta.json b/results-normal-pooling/chunk_size_16/no_model_name_available/no_revision_available/model_meta.json new file mode 100644 index 0000000..40ff52f --- /dev/null +++ b/results-normal-pooling/chunk_size_16/no_model_name_available/no_revision_available/model_meta.json @@ -0,0 +1 @@ +{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-normal-pooling/chunk_size_256/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-normal-pooling/chunk_size_256/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json new file mode 100644 index 0000000..a2e52e0 --- /dev/null +++ b/results-normal-pooling/chunk_size_256/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json @@ -0,0 +1,38 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 30.23233437538147, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.89279, + "map_at_1": 0.78869, + "map_at_10": 0.86062, + "map_at_3": 0.84573, + "map_at_5": 0.85734, + "mrr_at_1": 0.7886904761904762, + "mrr_at_10": 0.8606150793650794, + "mrr_at_3": 0.8457341269841271, + "mrr_at_5": 0.8573412698412698, + "ndcg_at_1": 0.78869, + "ndcg_at_10": 0.89279, + "ndcg_at_3": 0.86469, + "ndcg_at_5": 0.88556, + "precision_at_1": 0.78869, + "precision_at_10": 0.09911, + "precision_at_3": 0.30655, + "precision_at_5": 0.19405, + "recall_at_1": 0.78869, + "recall_at_10": 0.99107, + "recall_at_3": 0.91964, + "recall_at_5": 0.97024 + } + ] + }, + "task_name": "LEMBSummScreenFDRetrievalChunked" +} \ No newline at end of file diff --git a/results-normal-pooling/chunk_size_256/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-normal-pooling/chunk_size_256/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json new file mode 100644 index 0000000..e2fa347 --- /dev/null +++ b/results-normal-pooling/chunk_size_256/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -0,0 +1,38 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 28.12085223197937, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.94218, + "map_at_1": 0.90667, + "map_at_10": 0.92928, + "map_at_3": 0.92389, + "map_at_5": 0.92606, + "mrr_at_1": 0.9066666666666666, + "mrr_at_10": 0.9292817460317461, + "mrr_at_3": 0.923888888888889, + "mrr_at_5": 0.9260555555555555, + "ndcg_at_1": 0.90667, + "ndcg_at_10": 0.94218, + "ndcg_at_3": 0.93052, + "ndcg_at_5": 0.93453, + "precision_at_1": 0.90667, + "precision_at_10": 0.09833, + "precision_at_3": 0.31667, + "precision_at_5": 0.192, + "recall_at_1": 0.90667, + "recall_at_10": 0.98333, + "recall_at_3": 0.95, + "recall_at_5": 0.96 + } + ] + }, + "task_name": "LEMBWikimQARetrievalChunked" +} \ No newline at end of file diff --git a/results-normal-pooling/chunk_size_256/no_model_name_available/no_revision_available/model_meta.json b/results-normal-pooling/chunk_size_256/no_model_name_available/no_revision_available/model_meta.json new file mode 100644 index 0000000..40ff52f --- /dev/null +++ b/results-normal-pooling/chunk_size_256/no_model_name_available/no_revision_available/model_meta.json @@ -0,0 +1 @@ +{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-normal-pooling/chunk_size_32/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-normal-pooling/chunk_size_32/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json new file mode 100644 index 0000000..f38062f --- /dev/null +++ b/results-normal-pooling/chunk_size_32/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json @@ -0,0 +1,38 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 121.14276123046875, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.81621, + "map_at_1": 0.64583, + "map_at_10": 0.76397, + "map_at_3": 0.74454, + "map_at_5": 0.75764, + "mrr_at_1": 0.6458333333333334, + "mrr_at_10": 0.7639691987906274, + "mrr_at_3": 0.7445436507936508, + "mrr_at_5": 0.7576388888888889, + "ndcg_at_1": 0.64583, + "ndcg_at_10": 0.81621, + "ndcg_at_3": 0.77795, + "ndcg_at_5": 0.80139, + "precision_at_1": 0.64583, + "precision_at_10": 0.09762, + "precision_at_3": 0.29167, + "precision_at_5": 0.18631, + "recall_at_1": 0.64583, + "recall_at_10": 0.97619, + "recall_at_3": 0.875, + "recall_at_5": 0.93155 + } + ] + }, + "task_name": "LEMBSummScreenFDRetrievalChunked" +} \ No newline at end of file diff --git a/results-normal-pooling/chunk_size_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-normal-pooling/chunk_size_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json new file mode 100644 index 0000000..3819ca8 --- /dev/null +++ b/results-normal-pooling/chunk_size_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -0,0 +1,43 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 110.62627148628235, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.93753, + "map_at_1": 0.89333, + "map_at_10": 0.92385, + "map_at_20": 0.92385, + "map_at_3": 0.91778, + "map_at_5": 0.92161, + "mrr_at_1": 0.8933333333333333, + "mrr_at_10": 0.9238465608465608, + "mrr_at_20": 0.9238465608465608, + "mrr_at_3": 0.9177777777777777, + "mrr_at_5": 0.9216111111111112, + "ndcg_at_1": 0.89333, + "ndcg_at_10": 0.93753, + "ndcg_at_20": 0.93753, + "ndcg_at_3": 0.92524, + "ndcg_at_5": 0.93212, + "precision_at_1": 0.89333, + "precision_at_10": 0.098, + "precision_at_20": 0.049, + "precision_at_3": 0.31556, + "precision_at_5": 0.19267, + "recall_at_1": 0.89333, + "recall_at_10": 0.98, + "recall_at_20": 0.98, + "recall_at_3": 0.94667, + "recall_at_5": 0.96333 + } + ] + }, + "task_name": "LEMBWikimQARetrievalChunked" +} \ No newline at end of file diff --git a/results-normal-pooling/chunk_size_32/no_model_name_available/no_revision_available/model_meta.json b/results-normal-pooling/chunk_size_32/no_model_name_available/no_revision_available/model_meta.json new file mode 100644 index 0000000..40ff52f --- /dev/null +++ b/results-normal-pooling/chunk_size_32/no_model_name_available/no_revision_available/model_meta.json @@ -0,0 +1 @@ +{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-normal-pooling/chunk_size_512/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-normal-pooling/chunk_size_512/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json new file mode 100644 index 0000000..5cb312d --- /dev/null +++ b/results-normal-pooling/chunk_size_512/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json @@ -0,0 +1,43 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 23.719088792800903, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.89714, + "map_at_1": 0.79762, + "map_at_10": 0.86832, + "map_at_20": 0.86871, + "map_at_3": 0.85863, + "map_at_5": 0.86354, + "mrr_at_1": 0.7976190476190477, + "mrr_at_10": 0.8683153817082389, + "mrr_at_20": 0.8687139786470143, + "mrr_at_3": 0.8586309523809523, + "mrr_at_5": 0.8635416666666667, + "ndcg_at_1": 0.79762, + "ndcg_at_10": 0.89714, + "ndcg_at_20": 0.89863, + "ndcg_at_3": 0.87744, + "ndcg_at_5": 0.88615, + "precision_at_1": 0.79762, + "precision_at_10": 0.09851, + "precision_at_20": 0.04955, + "precision_at_3": 0.31052, + "precision_at_5": 0.19048, + "recall_at_1": 0.79762, + "recall_at_10": 0.98512, + "recall_at_20": 0.99107, + "recall_at_3": 0.93155, + "recall_at_5": 0.95238 + } + ] + }, + "task_name": "LEMBSummScreenFDRetrievalChunked" +} \ No newline at end of file diff --git a/results-normal-pooling/chunk_size_512/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-normal-pooling/chunk_size_512/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json new file mode 100644 index 0000000..e743fa2 --- /dev/null +++ b/results-normal-pooling/chunk_size_512/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -0,0 +1,43 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 22.589195013046265, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.91776, + "map_at_1": 0.86, + "map_at_10": 0.89989, + "map_at_20": 0.90087, + "map_at_3": 0.89389, + "map_at_5": 0.89689, + "mrr_at_1": 0.86, + "mrr_at_10": 0.899887566137566, + "mrr_at_20": 0.9008698535869589, + "mrr_at_3": 0.893888888888889, + "mrr_at_5": 0.8968888888888888, + "ndcg_at_1": 0.86, + "ndcg_at_10": 0.91776, + "ndcg_at_20": 0.92175, + "ndcg_at_3": 0.90488, + "ndcg_at_5": 0.91033, + "precision_at_1": 0.86, + "precision_at_10": 0.09733, + "precision_at_20": 0.0495, + "precision_at_3": 0.31222, + "precision_at_5": 0.19, + "recall_at_1": 0.86, + "recall_at_10": 0.97333, + "recall_at_20": 0.99, + "recall_at_3": 0.93667, + "recall_at_5": 0.95 + } + ] + }, + "task_name": "LEMBWikimQARetrievalChunked" +} \ No newline at end of file diff --git a/results-normal-pooling/chunk_size_512/no_model_name_available/no_revision_available/model_meta.json b/results-normal-pooling/chunk_size_512/no_model_name_available/no_revision_available/model_meta.json new file mode 100644 index 0000000..40ff52f --- /dev/null +++ b/results-normal-pooling/chunk_size_512/no_model_name_available/no_revision_available/model_meta.json @@ -0,0 +1 @@ +{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-normal-pooling/chunk_size_64/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-normal-pooling/chunk_size_64/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json new file mode 100644 index 0000000..1a185d8 --- /dev/null +++ b/results-normal-pooling/chunk_size_64/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json @@ -0,0 +1,43 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 70.64590835571289, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.85213, + "map_at_1": 0.71726, + "map_at_10": 0.81326, + "map_at_20": 0.81467, + "map_at_3": 0.79613, + "map_at_5": 0.80967, + "mrr_at_1": 0.7172619047619048, + "mrr_at_10": 0.8132594009826152, + "mrr_at_20": 0.8146659309605737, + "mrr_at_3": 0.7961309523809523, + "mrr_at_5": 0.8096726190476191, + "ndcg_at_1": 0.71726, + "ndcg_at_10": 0.85213, + "ndcg_at_20": 0.85734, + "ndcg_at_3": 0.81962, + "ndcg_at_5": 0.84345, + "precision_at_1": 0.71726, + "precision_at_10": 0.09702, + "precision_at_20": 0.04955, + "precision_at_3": 0.29563, + "precision_at_5": 0.18869, + "recall_at_1": 0.71726, + "recall_at_10": 0.97024, + "recall_at_20": 0.99107, + "recall_at_3": 0.8869, + "recall_at_5": 0.94345 + } + ] + }, + "task_name": "LEMBSummScreenFDRetrievalChunked" +} \ No newline at end of file diff --git a/results-normal-pooling/chunk_size_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-normal-pooling/chunk_size_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json new file mode 100644 index 0000000..0f1bdb6 --- /dev/null +++ b/results-normal-pooling/chunk_size_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -0,0 +1,43 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 65.4407320022583, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.95835, + "map_at_1": 0.92333, + "map_at_10": 0.94886, + "map_at_20": 0.94914, + "map_at_3": 0.94611, + "map_at_5": 0.94844, + "mrr_at_1": 0.9233333333333333, + "mrr_at_10": 0.9488611111111112, + "mrr_at_20": 0.9491388888888889, + "mrr_at_3": 0.946111111111111, + "mrr_at_5": 0.9484444444444445, + "ndcg_at_1": 0.92333, + "ndcg_at_10": 0.95835, + "ndcg_at_20": 0.95925, + "ndcg_at_3": 0.95313, + "ndcg_at_5": 0.95729, + "precision_at_1": 0.92333, + "precision_at_10": 0.09867, + "precision_at_20": 0.0495, + "precision_at_3": 0.32444, + "precision_at_5": 0.19667, + "recall_at_1": 0.92333, + "recall_at_10": 0.98667, + "recall_at_20": 0.99, + "recall_at_3": 0.97333, + "recall_at_5": 0.98333 + } + ] + }, + "task_name": "LEMBWikimQARetrievalChunked" +} \ No newline at end of file diff --git a/results-normal-pooling/chunk_size_64/no_model_name_available/no_revision_available/model_meta.json b/results-normal-pooling/chunk_size_64/no_model_name_available/no_revision_available/model_meta.json new file mode 100644 index 0000000..40ff52f --- /dev/null +++ b/results-normal-pooling/chunk_size_64/no_model_name_available/no_revision_available/model_meta.json @@ -0,0 +1 @@ +{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-normal-pooling/chunk_size_8/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-normal-pooling/chunk_size_8/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json new file mode 100644 index 0000000..11f468c --- /dev/null +++ b/results-normal-pooling/chunk_size_8/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json @@ -0,0 +1,43 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 484.68729400634766, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.70607, + "map_at_1": 0.50298, + "map_at_10": 0.63553, + "map_at_20": 0.6394, + "map_at_3": 0.59772, + "map_at_5": 0.62197, + "mrr_at_1": 0.5029761904761905, + "mrr_at_10": 0.6355324074074075, + "mrr_at_20": 0.6393996706102799, + "mrr_at_3": 0.597718253968254, + "mrr_at_5": 0.6219742063492063, + "ndcg_at_1": 0.50298, + "ndcg_at_10": 0.70607, + "ndcg_at_20": 0.7203, + "ndcg_at_3": 0.63056, + "ndcg_at_5": 0.67386, + "precision_at_1": 0.50298, + "precision_at_10": 0.09286, + "precision_at_20": 0.04926, + "precision_at_3": 0.24206, + "precision_at_5": 0.16607, + "recall_at_1": 0.50298, + "recall_at_10": 0.92857, + "recall_at_20": 0.98512, + "recall_at_3": 0.72619, + "recall_at_5": 0.83036 + } + ] + }, + "task_name": "LEMBSummScreenFDRetrievalChunked" +} \ No newline at end of file diff --git a/results-normal-pooling/chunk_size_8/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-normal-pooling/chunk_size_8/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json new file mode 100644 index 0000000..4255b91 --- /dev/null +++ b/results-normal-pooling/chunk_size_8/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -0,0 +1,43 @@ +{ + "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", + "evaluation_time": 435.66471695899963, + "kg_co2_emissions": null, + "mteb_version": "1.14.20", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.92234, + "map_at_1": 0.86, + "map_at_10": 0.90339, + "map_at_20": 0.90372, + "map_at_3": 0.89667, + "map_at_5": 0.902, + "mrr_at_1": 0.86, + "mrr_at_10": 0.9033888888888888, + "mrr_at_20": 0.9037222222222222, + "mrr_at_3": 0.8966666666666666, + "mrr_at_5": 0.902, + "ndcg_at_1": 0.86, + "ndcg_at_10": 0.92234, + "ndcg_at_20": 0.92386, + "ndcg_at_3": 0.90944, + "ndcg_at_5": 0.91905, + "precision_at_1": 0.86, + "precision_at_10": 0.098, + "precision_at_20": 0.04933, + "precision_at_3": 0.31556, + "precision_at_5": 0.194, + "recall_at_1": 0.86, + "recall_at_10": 0.98, + "recall_at_20": 0.98667, + "recall_at_3": 0.94667, + "recall_at_5": 0.97 + } + ] + }, + "task_name": "LEMBWikimQARetrievalChunked" +} \ No newline at end of file diff --git a/results-normal-pooling/chunk_size_8/no_model_name_available/no_revision_available/model_meta.json b/results-normal-pooling/chunk_size_8/no_model_name_available/no_revision_available/model_meta.json new file mode 100644 index 0000000..40ff52f --- /dev/null +++ b/results-normal-pooling/chunk_size_8/no_model_name_available/no_revision_available/model_meta.json @@ -0,0 +1 @@ +{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_8192/overlap_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-soft-boundary/embed_size_8192/overlap_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json index c66dd56..760c521 100644 --- a/results-soft-boundary/embed_size_8192/overlap_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ b/results-soft-boundary/embed_size_8192/overlap_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -1,6 +1,6 @@ { "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 37.43460035324097, + "evaluation_time": 37.77802062034607, "kg_co2_emissions": null, "mteb_version": "1.14.20", "scores": { diff --git a/results-soft-boundary/embed_size_8192/overlap_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-soft-boundary/embed_size_8192/overlap_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json index 6f86397..da6d73d 100644 --- a/results-soft-boundary/embed_size_8192/overlap_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ b/results-soft-boundary/embed_size_8192/overlap_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json @@ -1,6 +1,6 @@ { "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 37.48687982559204, + "evaluation_time": 38.01535248756409, "kg_co2_emissions": null, "mteb_version": "1.14.20", "scores": { From 753b603e0d4304e3e654439110e1d6223ff7b06c Mon Sep 17 00:00:00 2001 From: dannyjameswilliams Date: Thu, 26 Sep 2024 20:18:18 +0100 Subject: [PATCH 16/30] added benchmark files for macro chunks --- chunked_pooling/mteb_chunked_eval.py | 4 +- run_chunked_eval_with_macro_chunks.py | 60 +++++++++++++-------------- 2 files changed, 32 insertions(+), 32 deletions(-) diff --git a/chunked_pooling/mteb_chunked_eval.py b/chunked_pooling/mteb_chunked_eval.py index 4616124..df1b28c 100644 --- a/chunked_pooling/mteb_chunked_eval.py +++ b/chunked_pooling/mteb_chunked_eval.py @@ -245,7 +245,7 @@ def _evaluate_monolingual( text_inputs, return_tensors='pt', padding=True, - truncation=True, + truncation=self.truncate_max_length is not None, max_length=self.truncate_max_length, ) if model.device.type == 'cuda': @@ -263,7 +263,7 @@ def _evaluate_monolingual( output_embs = chunked_pooling( [model_outputs], annotations, max_length=None ) - else: + else: # truncation model_outputs = model(**model_inputs) output_embs = chunked_pooling( model_outputs, annotations, max_length=self.truncate_max_length diff --git a/run_chunked_eval_with_macro_chunks.py b/run_chunked_eval_with_macro_chunks.py index 661e46f..f990f6e 100644 --- a/run_chunked_eval_with_macro_chunks.py +++ b/run_chunked_eval_with_macro_chunks.py @@ -10,8 +10,8 @@ DEFAULT_N_SENTENCES = 5 BATCH_SIZE = 1 DEFAULT_OVERLAP_SIZE = 256 -DEFAULT_SOFT_BOUNDARY_EMBED_SIZE = 8192 -DEFAULT_HARD_BOUNDARY_EMBED_SIZE = 0 +DEFAULT_SOFT_BOUNDARY_EMBED_SIZE = 8192 # set to 0 to disable soft boundaries +DEFAULT_HARD_BOUNDARY_EMBED_SIZE = 0 # set to 0 to disable hard boundaries @click.command() @@ -108,34 +108,34 @@ def main( } # == Late Chunking == - tasks = [ - task_cls( - chunked_pooling_enabled=True, - tokenizer=tokenizer, - prune_size=None, - truncate_max_length=truncate_max_length, - soft_boundary_embed_size=soft_boundary_embed_size, - soft_boundary_overlap_size=overlap_size, - hard_boundary_embed_size=hard_boundary_embed_size, - **chunking_args, - ) - ] - - evaluation = MTEB( - tasks=tasks, - chunked_pooling_enabled=True, - tokenizer=tokenizer, - prune_size=None, - **chunking_args, - ) - evaluation.run( - model, - output_folder=f'results-chunked-pooling/chunk_size_{chunk_size}', - eval_splits=[eval_split], - overwrite_results=True, - batch_size=BATCH_SIZE, - encode_kwargs={'batch_size': BATCH_SIZE}, - ) + # tasks = [ + # task_cls( + # chunked_pooling_enabled=True, + # tokenizer=tokenizer, + # prune_size=None, + # truncate_max_length=truncate_max_length, + # soft_boundary_embed_size=soft_boundary_embed_size, + # soft_boundary_overlap_size=overlap_size, + # hard_boundary_embed_size=hard_boundary_embed_size, + # **chunking_args, + # ) + # ] + + # evaluation = MTEB( + # tasks=tasks, + # chunked_pooling_enabled=True, + # tokenizer=tokenizer, + # prune_size=None, + # **chunking_args, + # ) + # evaluation.run( + # model, + # output_folder=f'results-chunked-pooling/chunk_size_{chunk_size}', + # eval_splits=[eval_split], + # overwrite_results=True, + # batch_size=BATCH_SIZE, + # encode_kwargs={'batch_size': BATCH_SIZE}, + # ) # == Naive Chunking == # naive chunking does not need soft boundaries because chunk size is guaranteed to be <8192 tokens From 6b52c8cf229d5d350b1c372a7b19ad8d836d468c Mon Sep 17 00:00:00 2001 From: dannyjameswilliams Date: Thu, 26 Sep 2024 20:19:28 +0100 Subject: [PATCH 17/30] removed raw results --- .../LEMBSummScreenFDRetrievalChunked.json | 43 ------------------- .../LEMBWikimQARetrievalChunked.json | 43 ------------------- .../no_revision_available/model_meta.json | 1 - .../LEMBSummScreenFDRetrievalChunked.json | 43 ------------------- .../LEMBWikimQARetrievalChunked.json | 43 ------------------- .../no_revision_available/model_meta.json | 1 - .../LEMBSummScreenFDRetrievalChunked.json | 43 ------------------- .../LEMBWikimQARetrievalChunked.json | 38 ---------------- .../no_revision_available/model_meta.json | 1 - .../LEMBSummScreenFDRetrievalChunked.json | 38 ---------------- .../LEMBWikimQARetrievalChunked.json | 38 ---------------- .../no_revision_available/model_meta.json | 1 - .../LEMBSummScreenFDRetrievalChunked.json | 38 ---------------- .../LEMBWikimQARetrievalChunked.json | 43 ------------------- .../no_revision_available/model_meta.json | 1 - .../LEMBSummScreenFDRetrievalChunked.json | 43 ------------------- .../LEMBWikimQARetrievalChunked.json | 43 ------------------- .../no_revision_available/model_meta.json | 1 - .../LEMBSummScreenFDRetrievalChunked.json | 43 ------------------- .../LEMBWikimQARetrievalChunked.json | 43 ------------------- .../no_revision_available/model_meta.json | 1 - .../LEMBQMSumRetrievalChunked.json | 43 ------------------- .../LEMBSummScreenFDRetrievalChunked.json | 43 ------------------- .../LEMBWikimQARetrievalChunked.json | 43 ------------------- .../no_revision_available/model_meta.json | 1 - .../LEMBQMSumRetrievalChunked.json | 43 ------------------- .../LEMBSummScreenFDRetrievalChunked.json | 38 ---------------- .../LEMBWikimQARetrievalChunked.json | 38 ---------------- .../no_revision_available/model_meta.json | 1 - .../LEMBQMSumRetrievalChunked.json | 43 ------------------- .../LEMBSummScreenFDRetrievalChunked.json | 38 ---------------- .../LEMBWikimQARetrievalChunked.json | 38 ---------------- .../no_revision_available/model_meta.json | 1 - .../LEMBQMSumRetrievalChunked.json | 43 ------------------- .../LEMBSummScreenFDRetrievalChunked.json | 38 ---------------- .../LEMBWikimQARetrievalChunked.json | 38 ---------------- .../no_revision_available/model_meta.json | 1 - .../LEMBQMSumRetrievalChunked.json | 43 ------------------- .../LEMBSummScreenFDRetrievalChunked.json | 38 ---------------- .../LEMBWikimQARetrievalChunked.json | 38 ---------------- .../no_revision_available/model_meta.json | 1 - .../LEMBSummScreenFDRetrievalChunked.json | 43 ------------------- .../LEMBWikimQARetrievalChunked.json | 43 ------------------- .../no_revision_available/model_meta.json | 1 - .../LEMBSummScreenFDRetrievalChunked.json | 43 ------------------- .../LEMBWikimQARetrievalChunked.json | 43 ------------------- .../no_revision_available/model_meta.json | 1 - .../LEMBSummScreenFDRetrievalChunked.json | 43 ------------------- .../LEMBWikimQARetrievalChunked.json | 38 ---------------- .../no_revision_available/model_meta.json | 1 - .../LEMBSummScreenFDRetrievalChunked.json | 38 ---------------- .../LEMBWikimQARetrievalChunked.json | 38 ---------------- .../no_revision_available/model_meta.json | 1 - .../LEMBSummScreenFDRetrievalChunked.json | 38 ---------------- .../LEMBWikimQARetrievalChunked.json | 43 ------------------- .../no_revision_available/model_meta.json | 1 - .../LEMBSummScreenFDRetrievalChunked.json | 43 ------------------- .../LEMBWikimQARetrievalChunked.json | 43 ------------------- .../no_revision_available/model_meta.json | 1 - .../LEMBSummScreenFDRetrievalChunked.json | 43 ------------------- .../LEMBWikimQARetrievalChunked.json | 43 ------------------- .../no_revision_available/model_meta.json | 1 - .../LEMBSummScreenFDRetrievalChunked.json | 43 ------------------- .../LEMBWikimQARetrievalChunked.json | 43 ------------------- .../no_revision_available/model_meta.json | 1 - .../LEMBQMSumRetrievalChunked.json | 43 ------------------- .../LEMBSummScreenFDRetrievalChunked.json | 38 ---------------- .../LEMBWikimQARetrievalChunked.json | 38 ---------------- .../no_revision_available/model_meta.json | 1 - .../LEMBQMSumRetrievalChunked.json | 43 ------------------- .../LEMBSummScreenFDRetrievalChunked.json | 38 ---------------- .../LEMBWikimQARetrievalChunked.json | 38 ---------------- .../no_revision_available/model_meta.json | 1 - .../LEMBQMSumRetrievalChunked.json | 43 ------------------- .../LEMBSummScreenFDRetrievalChunked.json | 38 ---------------- .../LEMBWikimQARetrievalChunked.json | 38 ---------------- .../no_revision_available/model_meta.json | 1 - .../LEMBQMSumRetrievalChunked.json | 43 ------------------- .../LEMBSummScreenFDRetrievalChunked.json | 38 ---------------- .../LEMBWikimQARetrievalChunked.json | 38 ---------------- .../no_revision_available/model_meta.json | 1 - .../LEMBQMSumRetrievalChunked.json | 43 ------------------- .../LEMBSummScreenFDRetrievalChunked.json | 38 ---------------- .../LEMBWikimQARetrievalChunked.json | 38 ---------------- .../no_revision_available/model_meta.json | 1 - .../LEMBQMSumRetrievalChunked.json | 43 ------------------- .../LEMBSummScreenFDRetrievalChunked.json | 38 ---------------- .../LEMBWikimQARetrievalChunked.json | 38 ---------------- .../no_revision_available/model_meta.json | 1 - .../LEMBQMSumRetrievalChunked.json | 43 ------------------- .../LEMBSummScreenFDRetrievalChunked.json | 38 ---------------- .../LEMBWikimQARetrievalChunked.json | 38 ---------------- .../no_revision_available/model_meta.json | 1 - .../LEMBQMSumRetrievalChunked.json | 43 ------------------- .../LEMBSummScreenFDRetrievalChunked.json | 38 ---------------- .../LEMBWikimQARetrievalChunked.json | 38 ---------------- .../no_revision_available/model_meta.json | 1 - .../LEMBQMSumRetrievalChunked.json | 43 ------------------- .../LEMBSummScreenFDRetrievalChunked.json | 38 ---------------- .../LEMBWikimQARetrievalChunked.json | 38 ---------------- .../no_revision_available/model_meta.json | 1 - .../LEMBQMSumRetrievalChunked.json | 43 ------------------- .../LEMBSummScreenFDRetrievalChunked.json | 38 ---------------- .../LEMBWikimQARetrievalChunked.json | 38 ---------------- .../no_revision_available/model_meta.json | 1 - .../LEMBQMSumRetrievalChunked.json | 43 ------------------- .../LEMBSummScreenFDRetrievalChunked.json | 38 ---------------- .../LEMBWikimQARetrievalChunked.json | 38 ---------------- .../no_revision_available/model_meta.json | 1 - .../LEMBQMSumRetrievalChunked.json | 43 ------------------- .../LEMBSummScreenFDRetrievalChunked.json | 38 ---------------- .../LEMBWikimQARetrievalChunked.json | 38 ---------------- .../no_revision_available/model_meta.json | 1 - .../LEMBQMSumRetrievalChunked.json | 43 ------------------- .../LEMBSummScreenFDRetrievalChunked.json | 38 ---------------- .../LEMBWikimQARetrievalChunked.json | 38 ---------------- .../no_revision_available/model_meta.json | 1 - .../LEMBQMSumRetrievalChunked.json | 43 ------------------- .../LEMBSummScreenFDRetrievalChunked.json | 38 ---------------- .../LEMBWikimQARetrievalChunked.json | 38 ---------------- .../no_revision_available/model_meta.json | 1 - .../LEMBQMSumRetrievalChunked.json | 43 ------------------- .../LEMBSummScreenFDRetrievalChunked.json | 38 ---------------- .../LEMBWikimQARetrievalChunked.json | 38 ---------------- .../no_revision_available/model_meta.json | 1 - .../LEMBQMSumRetrievalChunked.json | 43 ------------------- .../no_revision_available/model_meta.json | 1 - .../LEMBQMSumRetrievalChunked.json | 43 ------------------- .../no_revision_available/model_meta.json | 1 - .../LEMBQMSumRetrievalChunked.json | 43 ------------------- .../no_revision_available/model_meta.json | 1 - .../no_revision_available/model_meta.json | 1 - .../LEMBQMSumRetrievalChunked.json | 43 ------------------- .../no_revision_available/model_meta.json | 1 - .../LEMBQMSumRetrievalChunked.json | 43 ------------------- .../LEMBSummScreenFDRetrievalChunked.json | 38 ---------------- .../LEMBWikimQARetrievalChunked.json | 38 ---------------- .../no_revision_available/model_meta.json | 1 - .../LEMBQMSumRetrievalChunked.json | 43 ------------------- .../LEMBSummScreenFDRetrievalChunked.json | 38 ---------------- .../LEMBWikimQARetrievalChunked.json | 38 ---------------- .../no_revision_available/model_meta.json | 1 - .../LEMBQMSumRetrievalChunked.json | 43 ------------------- .../LEMBSummScreenFDRetrievalChunked.json | 38 ---------------- .../LEMBWikimQARetrievalChunked.json | 38 ---------------- .../no_revision_available/model_meta.json | 1 - .../LEMBQMSumRetrievalChunked.json | 43 ------------------- .../LEMBSummScreenFDRetrievalChunked.json | 38 ---------------- .../LEMBWikimQARetrievalChunked.json | 38 ---------------- .../no_revision_available/model_meta.json | 1 - .../LEMBQMSumRetrievalChunked.json | 43 ------------------- .../LEMBSummScreenFDRetrievalChunked.json | 38 ---------------- .../LEMBWikimQARetrievalChunked.json | 38 ---------------- .../no_revision_available/model_meta.json | 1 - .../LEMBQMSumRetrievalChunked.json | 43 ------------------- .../LEMBSummScreenFDRetrievalChunked.json | 43 ------------------- .../LEMBWikimQARetrievalChunked.json | 43 ------------------- .../no_revision_available/model_meta.json | 1 - 158 files changed, 4582 deletions(-) delete mode 100644 results-chunked-pooling/chunk_size_1024/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json delete mode 100644 results-chunked-pooling/chunk_size_1024/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json delete mode 100644 results-chunked-pooling/chunk_size_1024/no_model_name_available/no_revision_available/model_meta.json delete mode 100644 results-chunked-pooling/chunk_size_128/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json delete mode 100644 results-chunked-pooling/chunk_size_128/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json delete mode 100644 results-chunked-pooling/chunk_size_128/no_model_name_available/no_revision_available/model_meta.json delete mode 100644 results-chunked-pooling/chunk_size_16/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json delete mode 100644 results-chunked-pooling/chunk_size_16/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json delete mode 100644 results-chunked-pooling/chunk_size_16/no_model_name_available/no_revision_available/model_meta.json delete mode 100644 results-chunked-pooling/chunk_size_256/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json delete mode 100644 results-chunked-pooling/chunk_size_256/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json delete mode 100644 results-chunked-pooling/chunk_size_256/no_model_name_available/no_revision_available/model_meta.json delete mode 100644 results-chunked-pooling/chunk_size_32/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json delete mode 100644 results-chunked-pooling/chunk_size_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json delete mode 100644 results-chunked-pooling/chunk_size_32/no_model_name_available/no_revision_available/model_meta.json delete mode 100644 results-chunked-pooling/chunk_size_512/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json delete mode 100644 results-chunked-pooling/chunk_size_512/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json delete mode 100644 results-chunked-pooling/chunk_size_512/no_model_name_available/no_revision_available/model_meta.json delete mode 100644 results-chunked-pooling/chunk_size_64/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json delete mode 100644 results-chunked-pooling/chunk_size_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json delete mode 100644 results-chunked-pooling/chunk_size_64/no_model_name_available/no_revision_available/model_meta.json delete mode 100644 results-chunked-pooling/chunk_size_8/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json delete mode 100644 results-chunked-pooling/chunk_size_8/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json delete mode 100644 results-chunked-pooling/chunk_size_8/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json delete mode 100644 results-chunked-pooling/chunk_size_8/no_model_name_available/no_revision_available/model_meta.json delete mode 100644 results-hard-boundary/embed_size_1024/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json delete mode 100644 results-hard-boundary/embed_size_1024/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json delete mode 100644 results-hard-boundary/embed_size_1024/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json delete mode 100644 results-hard-boundary/embed_size_1024/no_model_name_available/no_revision_available/model_meta.json delete mode 100644 results-hard-boundary/embed_size_2048/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json delete mode 100644 results-hard-boundary/embed_size_2048/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json delete mode 100644 results-hard-boundary/embed_size_2048/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json delete mode 100644 results-hard-boundary/embed_size_2048/no_model_name_available/no_revision_available/model_meta.json delete mode 100644 results-hard-boundary/embed_size_4096/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json delete mode 100644 results-hard-boundary/embed_size_4096/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json delete mode 100644 results-hard-boundary/embed_size_4096/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json delete mode 100644 results-hard-boundary/embed_size_4096/no_model_name_available/no_revision_available/model_meta.json delete mode 100644 results-hard-boundary/embed_size_8192/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json delete mode 100644 results-hard-boundary/embed_size_8192/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json delete mode 100644 results-hard-boundary/embed_size_8192/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json delete mode 100644 results-hard-boundary/embed_size_8192/no_model_name_available/no_revision_available/model_meta.json delete mode 100644 results-normal-pooling/chunk_size_1024/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json delete mode 100644 results-normal-pooling/chunk_size_1024/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json delete mode 100644 results-normal-pooling/chunk_size_1024/no_model_name_available/no_revision_available/model_meta.json delete mode 100644 results-normal-pooling/chunk_size_128/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json delete mode 100644 results-normal-pooling/chunk_size_128/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json delete mode 100644 results-normal-pooling/chunk_size_128/no_model_name_available/no_revision_available/model_meta.json delete mode 100644 results-normal-pooling/chunk_size_16/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json delete mode 100644 results-normal-pooling/chunk_size_16/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json delete mode 100644 results-normal-pooling/chunk_size_16/no_model_name_available/no_revision_available/model_meta.json delete mode 100644 results-normal-pooling/chunk_size_256/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json delete mode 100644 results-normal-pooling/chunk_size_256/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json delete mode 100644 results-normal-pooling/chunk_size_256/no_model_name_available/no_revision_available/model_meta.json delete mode 100644 results-normal-pooling/chunk_size_32/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json delete mode 100644 results-normal-pooling/chunk_size_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json delete mode 100644 results-normal-pooling/chunk_size_32/no_model_name_available/no_revision_available/model_meta.json delete mode 100644 results-normal-pooling/chunk_size_512/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json delete mode 100644 results-normal-pooling/chunk_size_512/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json delete mode 100644 results-normal-pooling/chunk_size_512/no_model_name_available/no_revision_available/model_meta.json delete mode 100644 results-normal-pooling/chunk_size_64/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json delete mode 100644 results-normal-pooling/chunk_size_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json delete mode 100644 results-normal-pooling/chunk_size_64/no_model_name_available/no_revision_available/model_meta.json delete mode 100644 results-normal-pooling/chunk_size_8/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json delete mode 100644 results-normal-pooling/chunk_size_8/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json delete mode 100644 results-normal-pooling/chunk_size_8/no_model_name_available/no_revision_available/model_meta.json delete mode 100644 results-soft-boundary/embed_size_1024/overlap_128/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json delete mode 100644 results-soft-boundary/embed_size_1024/overlap_128/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json delete mode 100644 results-soft-boundary/embed_size_1024/overlap_128/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json delete mode 100644 results-soft-boundary/embed_size_1024/overlap_128/no_model_name_available/no_revision_available/model_meta.json delete mode 100644 results-soft-boundary/embed_size_1024/overlap_256/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json delete mode 100644 results-soft-boundary/embed_size_1024/overlap_256/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json delete mode 100644 results-soft-boundary/embed_size_1024/overlap_256/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json delete mode 100644 results-soft-boundary/embed_size_1024/overlap_256/no_model_name_available/no_revision_available/model_meta.json delete mode 100644 results-soft-boundary/embed_size_1024/overlap_32/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json delete mode 100644 results-soft-boundary/embed_size_1024/overlap_32/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json delete mode 100644 results-soft-boundary/embed_size_1024/overlap_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json delete mode 100644 results-soft-boundary/embed_size_1024/overlap_32/no_model_name_available/no_revision_available/model_meta.json delete mode 100644 results-soft-boundary/embed_size_1024/overlap_512/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json delete mode 100644 results-soft-boundary/embed_size_1024/overlap_512/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json delete mode 100644 results-soft-boundary/embed_size_1024/overlap_512/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json delete mode 100644 results-soft-boundary/embed_size_1024/overlap_512/no_model_name_available/no_revision_available/model_meta.json delete mode 100644 results-soft-boundary/embed_size_1024/overlap_64/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json delete mode 100644 results-soft-boundary/embed_size_1024/overlap_64/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json delete mode 100644 results-soft-boundary/embed_size_1024/overlap_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json delete mode 100644 results-soft-boundary/embed_size_1024/overlap_64/no_model_name_available/no_revision_available/model_meta.json delete mode 100644 results-soft-boundary/embed_size_2048/overlap_128/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json delete mode 100644 results-soft-boundary/embed_size_2048/overlap_128/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json delete mode 100644 results-soft-boundary/embed_size_2048/overlap_128/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json delete mode 100644 results-soft-boundary/embed_size_2048/overlap_128/no_model_name_available/no_revision_available/model_meta.json delete mode 100644 results-soft-boundary/embed_size_2048/overlap_256/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json delete mode 100644 results-soft-boundary/embed_size_2048/overlap_256/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json delete mode 100644 results-soft-boundary/embed_size_2048/overlap_256/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json delete mode 100644 results-soft-boundary/embed_size_2048/overlap_256/no_model_name_available/no_revision_available/model_meta.json delete mode 100644 results-soft-boundary/embed_size_2048/overlap_32/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json delete mode 100644 results-soft-boundary/embed_size_2048/overlap_32/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json delete mode 100644 results-soft-boundary/embed_size_2048/overlap_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json delete mode 100644 results-soft-boundary/embed_size_2048/overlap_32/no_model_name_available/no_revision_available/model_meta.json delete mode 100644 results-soft-boundary/embed_size_2048/overlap_512/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json delete mode 100644 results-soft-boundary/embed_size_2048/overlap_512/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json delete mode 100644 results-soft-boundary/embed_size_2048/overlap_512/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json delete mode 100644 results-soft-boundary/embed_size_2048/overlap_512/no_model_name_available/no_revision_available/model_meta.json delete mode 100644 results-soft-boundary/embed_size_2048/overlap_64/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json delete mode 100644 results-soft-boundary/embed_size_2048/overlap_64/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json delete mode 100644 results-soft-boundary/embed_size_2048/overlap_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json delete mode 100644 results-soft-boundary/embed_size_2048/overlap_64/no_model_name_available/no_revision_available/model_meta.json delete mode 100644 results-soft-boundary/embed_size_4096/overlap_128/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json delete mode 100644 results-soft-boundary/embed_size_4096/overlap_128/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json delete mode 100644 results-soft-boundary/embed_size_4096/overlap_128/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json delete mode 100644 results-soft-boundary/embed_size_4096/overlap_128/no_model_name_available/no_revision_available/model_meta.json delete mode 100644 results-soft-boundary/embed_size_4096/overlap_256/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json delete mode 100644 results-soft-boundary/embed_size_4096/overlap_256/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json delete mode 100644 results-soft-boundary/embed_size_4096/overlap_256/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json delete mode 100644 results-soft-boundary/embed_size_4096/overlap_256/no_model_name_available/no_revision_available/model_meta.json delete mode 100644 results-soft-boundary/embed_size_4096/overlap_32/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json delete mode 100644 results-soft-boundary/embed_size_4096/overlap_32/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json delete mode 100644 results-soft-boundary/embed_size_4096/overlap_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json delete mode 100644 results-soft-boundary/embed_size_4096/overlap_32/no_model_name_available/no_revision_available/model_meta.json delete mode 100644 results-soft-boundary/embed_size_4096/overlap_512/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json delete mode 100644 results-soft-boundary/embed_size_4096/overlap_512/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json delete mode 100644 results-soft-boundary/embed_size_4096/overlap_512/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json delete mode 100644 results-soft-boundary/embed_size_4096/overlap_512/no_model_name_available/no_revision_available/model_meta.json delete mode 100644 results-soft-boundary/embed_size_4096/overlap_64/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json delete mode 100644 results-soft-boundary/embed_size_4096/overlap_64/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json delete mode 100644 results-soft-boundary/embed_size_4096/overlap_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json delete mode 100644 results-soft-boundary/embed_size_4096/overlap_64/no_model_name_available/no_revision_available/model_meta.json delete mode 100644 results-soft-boundary/embed_size_512/overlap_128/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json delete mode 100644 results-soft-boundary/embed_size_512/overlap_128/no_model_name_available/no_revision_available/model_meta.json delete mode 100644 results-soft-boundary/embed_size_512/overlap_256/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json delete mode 100644 results-soft-boundary/embed_size_512/overlap_256/no_model_name_available/no_revision_available/model_meta.json delete mode 100644 results-soft-boundary/embed_size_512/overlap_32/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json delete mode 100644 results-soft-boundary/embed_size_512/overlap_32/no_model_name_available/no_revision_available/model_meta.json delete mode 100644 results-soft-boundary/embed_size_512/overlap_512/no_model_name_available/no_revision_available/model_meta.json delete mode 100644 results-soft-boundary/embed_size_512/overlap_64/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json delete mode 100644 results-soft-boundary/embed_size_512/overlap_64/no_model_name_available/no_revision_available/model_meta.json delete mode 100644 results-soft-boundary/embed_size_8192/overlap_128/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json delete mode 100644 results-soft-boundary/embed_size_8192/overlap_128/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json delete mode 100644 results-soft-boundary/embed_size_8192/overlap_128/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json delete mode 100644 results-soft-boundary/embed_size_8192/overlap_128/no_model_name_available/no_revision_available/model_meta.json delete mode 100644 results-soft-boundary/embed_size_8192/overlap_256/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json delete mode 100644 results-soft-boundary/embed_size_8192/overlap_256/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json delete mode 100644 results-soft-boundary/embed_size_8192/overlap_256/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json delete mode 100644 results-soft-boundary/embed_size_8192/overlap_256/no_model_name_available/no_revision_available/model_meta.json delete mode 100644 results-soft-boundary/embed_size_8192/overlap_32/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json delete mode 100644 results-soft-boundary/embed_size_8192/overlap_32/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json delete mode 100644 results-soft-boundary/embed_size_8192/overlap_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json delete mode 100644 results-soft-boundary/embed_size_8192/overlap_32/no_model_name_available/no_revision_available/model_meta.json delete mode 100644 results-soft-boundary/embed_size_8192/overlap_512/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json delete mode 100644 results-soft-boundary/embed_size_8192/overlap_512/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json delete mode 100644 results-soft-boundary/embed_size_8192/overlap_512/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json delete mode 100644 results-soft-boundary/embed_size_8192/overlap_512/no_model_name_available/no_revision_available/model_meta.json delete mode 100644 results-soft-boundary/embed_size_8192/overlap_64/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json delete mode 100644 results-soft-boundary/embed_size_8192/overlap_64/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json delete mode 100644 results-soft-boundary/embed_size_8192/overlap_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json delete mode 100644 results-soft-boundary/embed_size_8192/overlap_64/no_model_name_available/no_revision_available/model_meta.json delete mode 100644 results-truncation/embed_size_8192/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json delete mode 100644 results-truncation/embed_size_8192/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json delete mode 100644 results-truncation/embed_size_8192/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json delete mode 100644 results-truncation/embed_size_8192/no_model_name_available/no_revision_available/model_meta.json diff --git a/results-chunked-pooling/chunk_size_1024/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-chunked-pooling/chunk_size_1024/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json deleted file mode 100644 index a049782..0000000 --- a/results-chunked-pooling/chunk_size_1024/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 42.20820331573486, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.89797, - "map_at_1": 0.80952, - "map_at_10": 0.87148, - "map_at_20": 0.87238, - "map_at_3": 0.8631, - "map_at_5": 0.86771, - "mrr_at_1": 0.8095238095238095, - "mrr_at_10": 0.8714781746031746, - "mrr_at_20": 0.8723811778499279, - "mrr_at_3": 0.8630952380952381, - "mrr_at_5": 0.8677083333333334, - "ndcg_at_1": 0.80952, - "ndcg_at_10": 0.89797, - "ndcg_at_20": 0.90107, - "ndcg_at_3": 0.88067, - "ndcg_at_5": 0.88912, - "precision_at_1": 0.80952, - "precision_at_10": 0.09792, - "precision_at_20": 0.04955, - "precision_at_3": 0.31052, - "precision_at_5": 0.19048, - "recall_at_1": 0.80952, - "recall_at_10": 0.97917, - "recall_at_20": 0.99107, - "recall_at_3": 0.93155, - "recall_at_5": 0.95238 - } - ] - }, - "task_name": "LEMBSummScreenFDRetrievalChunked" -} \ No newline at end of file diff --git a/results-chunked-pooling/chunk_size_1024/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-chunked-pooling/chunk_size_1024/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json deleted file mode 100644 index 90095ac..0000000 --- a/results-chunked-pooling/chunk_size_1024/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 37.14106869697571, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.89865, - "map_at_1": 0.84, - "map_at_10": 0.88085, - "map_at_20": 0.8821, - "map_at_3": 0.87389, - "map_at_5": 0.87856, - "mrr_at_1": 0.84, - "mrr_at_10": 0.8808465608465609, - "mrr_at_20": 0.8820996378349318, - "mrr_at_3": 0.8738888888888889, - "mrr_at_5": 0.8785555555555555, - "ndcg_at_1": 0.84, - "ndcg_at_10": 0.89865, - "ndcg_at_20": 0.90299, - "ndcg_at_3": 0.88488, - "ndcg_at_5": 0.8932, - "precision_at_1": 0.84, - "precision_at_10": 0.09533, - "precision_at_20": 0.0485, - "precision_at_3": 0.30556, - "precision_at_5": 0.18733, - "recall_at_1": 0.84, - "recall_at_10": 0.95333, - "recall_at_20": 0.97, - "recall_at_3": 0.91667, - "recall_at_5": 0.93667 - } - ] - }, - "task_name": "LEMBWikimQARetrievalChunked" -} \ No newline at end of file diff --git a/results-chunked-pooling/chunk_size_1024/no_model_name_available/no_revision_available/model_meta.json b/results-chunked-pooling/chunk_size_1024/no_model_name_available/no_revision_available/model_meta.json deleted file mode 100644 index 40ff52f..0000000 --- a/results-chunked-pooling/chunk_size_1024/no_model_name_available/no_revision_available/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-chunked-pooling/chunk_size_128/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-chunked-pooling/chunk_size_128/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json deleted file mode 100644 index 8614548..0000000 --- a/results-chunked-pooling/chunk_size_128/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 46.49887251853943, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.89057, - "map_at_1": 0.79167, - "map_at_10": 0.86152, - "map_at_20": 0.8624, - "map_at_3": 0.85069, - "map_at_5": 0.85873, - "mrr_at_1": 0.7916666666666666, - "mrr_at_10": 0.8615197467876039, - "mrr_at_20": 0.8624014915309559, - "mrr_at_3": 0.8506944444444444, - "mrr_at_5": 0.8587301587301588, - "ndcg_at_1": 0.79167, - "ndcg_at_10": 0.89057, - "ndcg_at_20": 0.89365, - "ndcg_at_3": 0.86922, - "ndcg_at_5": 0.88382, - "precision_at_1": 0.79167, - "precision_at_10": 0.09792, - "precision_at_20": 0.04955, - "precision_at_3": 0.30754, - "precision_at_5": 0.19167, - "recall_at_1": 0.79167, - "recall_at_10": 0.97917, - "recall_at_20": 0.99107, - "recall_at_3": 0.92262, - "recall_at_5": 0.95833 - } - ] - }, - "task_name": "LEMBSummScreenFDRetrievalChunked" -} \ No newline at end of file diff --git a/results-chunked-pooling/chunk_size_128/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-chunked-pooling/chunk_size_128/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json deleted file mode 100644 index ce12f18..0000000 --- a/results-chunked-pooling/chunk_size_128/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 41.79281163215637, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.96324, - "map_at_1": 0.93667, - "map_at_10": 0.95468, - "map_at_20": 0.95486, - "map_at_3": 0.95222, - "map_at_5": 0.95222, - "mrr_at_1": 0.9366666666666666, - "mrr_at_10": 0.9546759259259259, - "mrr_at_20": 0.954861111111111, - "mrr_at_3": 0.9522222222222223, - "mrr_at_5": 0.9522222222222223, - "ndcg_at_1": 0.93667, - "ndcg_at_10": 0.96324, - "ndcg_at_20": 0.96402, - "ndcg_at_3": 0.95762, - "ndcg_at_5": 0.95762, - "precision_at_1": 0.93667, - "precision_at_10": 0.099, - "precision_at_20": 0.04967, - "precision_at_3": 0.32444, - "precision_at_5": 0.19467, - "recall_at_1": 0.93667, - "recall_at_10": 0.99, - "recall_at_20": 0.99333, - "recall_at_3": 0.97333, - "recall_at_5": 0.97333 - } - ] - }, - "task_name": "LEMBWikimQARetrievalChunked" -} \ No newline at end of file diff --git a/results-chunked-pooling/chunk_size_128/no_model_name_available/no_revision_available/model_meta.json b/results-chunked-pooling/chunk_size_128/no_model_name_available/no_revision_available/model_meta.json deleted file mode 100644 index 40ff52f..0000000 --- a/results-chunked-pooling/chunk_size_128/no_model_name_available/no_revision_available/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-chunked-pooling/chunk_size_16/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-chunked-pooling/chunk_size_16/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json deleted file mode 100644 index 66fa4c2..0000000 --- a/results-chunked-pooling/chunk_size_16/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 100.60042762756348, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.91536, - "map_at_1": 0.82738, - "map_at_10": 0.89029, - "map_at_20": 0.89049, - "map_at_3": 0.88046, - "map_at_5": 0.88805, - "mrr_at_1": 0.8273809523809523, - "mrr_at_10": 0.8902895880574452, - "mrr_at_20": 0.8904880007558579, - "mrr_at_3": 0.8804563492063492, - "mrr_at_5": 0.8880456349206349, - "ndcg_at_1": 0.82738, - "ndcg_at_10": 0.91536, - "ndcg_at_20": 0.91611, - "ndcg_at_3": 0.89672, - "ndcg_at_5": 0.91029, - "precision_at_1": 0.82738, - "precision_at_10": 0.09911, - "precision_at_20": 0.0497, - "precision_at_3": 0.31448, - "precision_at_5": 0.19524, - "recall_at_1": 0.82738, - "recall_at_10": 0.99107, - "recall_at_20": 0.99405, - "recall_at_3": 0.94345, - "recall_at_5": 0.97619 - } - ] - }, - "task_name": "LEMBSummScreenFDRetrievalChunked" -} \ No newline at end of file diff --git a/results-chunked-pooling/chunk_size_16/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-chunked-pooling/chunk_size_16/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json deleted file mode 100644 index c26076b..0000000 --- a/results-chunked-pooling/chunk_size_16/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 61.05208373069763, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.96675, - "map_at_1": 0.94333, - "map_at_10": 0.96026, - "map_at_3": 0.95889, - "map_at_5": 0.95956, - "mrr_at_1": 0.9433333333333334, - "mrr_at_10": 0.9602592592592593, - "mrr_at_3": 0.9588888888888889, - "mrr_at_5": 0.9595555555555556, - "ndcg_at_1": 0.94333, - "ndcg_at_10": 0.96675, - "ndcg_at_3": 0.96349, - "ndcg_at_5": 0.96478, - "precision_at_1": 0.94333, - "precision_at_10": 0.09867, - "precision_at_3": 0.32556, - "precision_at_5": 0.196, - "recall_at_1": 0.94333, - "recall_at_10": 0.98667, - "recall_at_3": 0.97667, - "recall_at_5": 0.98 - } - ] - }, - "task_name": "LEMBWikimQARetrievalChunked" -} \ No newline at end of file diff --git a/results-chunked-pooling/chunk_size_16/no_model_name_available/no_revision_available/model_meta.json b/results-chunked-pooling/chunk_size_16/no_model_name_available/no_revision_available/model_meta.json deleted file mode 100644 index 40ff52f..0000000 --- a/results-chunked-pooling/chunk_size_16/no_model_name_available/no_revision_available/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-chunked-pooling/chunk_size_256/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-chunked-pooling/chunk_size_256/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json deleted file mode 100644 index 6c629a6..0000000 --- a/results-chunked-pooling/chunk_size_256/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 42.10010838508606, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.8896, - "map_at_1": 0.79464, - "map_at_10": 0.86285, - "map_at_3": 0.85466, - "map_at_5": 0.86032, - "mrr_at_1": 0.7946428571428571, - "mrr_at_10": 0.8628472222222223, - "mrr_at_3": 0.8546626984126985, - "mrr_at_5": 0.8603174603174603, - "ndcg_at_1": 0.79464, - "ndcg_at_10": 0.8896, - "ndcg_at_3": 0.87369, - "ndcg_at_5": 0.88368, - "precision_at_1": 0.79464, - "precision_at_10": 0.09702, - "precision_at_3": 0.30952, - "precision_at_5": 0.19048, - "recall_at_1": 0.79464, - "recall_at_10": 0.97024, - "recall_at_3": 0.92857, - "recall_at_5": 0.95238 - } - ] - }, - "task_name": "LEMBSummScreenFDRetrievalChunked" -} \ No newline at end of file diff --git a/results-chunked-pooling/chunk_size_256/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-chunked-pooling/chunk_size_256/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json deleted file mode 100644 index e301d83..0000000 --- a/results-chunked-pooling/chunk_size_256/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 38.12126922607422, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.94608, - "map_at_1": 0.90667, - "map_at_10": 0.93403, - "map_at_3": 0.93, - "map_at_5": 0.93233, - "mrr_at_1": 0.9066666666666666, - "mrr_at_10": 0.9340317460317461, - "mrr_at_3": 0.93, - "mrr_at_5": 0.9323333333333333, - "ndcg_at_1": 0.90667, - "ndcg_at_10": 0.94608, - "ndcg_at_3": 0.9377, - "ndcg_at_5": 0.94186, - "precision_at_1": 0.90667, - "precision_at_10": 0.09833, - "precision_at_3": 0.32, - "precision_at_5": 0.194, - "recall_at_1": 0.90667, - "recall_at_10": 0.98333, - "recall_at_3": 0.96, - "recall_at_5": 0.97 - } - ] - }, - "task_name": "LEMBWikimQARetrievalChunked" -} \ No newline at end of file diff --git a/results-chunked-pooling/chunk_size_256/no_model_name_available/no_revision_available/model_meta.json b/results-chunked-pooling/chunk_size_256/no_model_name_available/no_revision_available/model_meta.json deleted file mode 100644 index 40ff52f..0000000 --- a/results-chunked-pooling/chunk_size_256/no_model_name_available/no_revision_available/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-chunked-pooling/chunk_size_32/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-chunked-pooling/chunk_size_32/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json deleted file mode 100644 index 261d110..0000000 --- a/results-chunked-pooling/chunk_size_32/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 56.322009563446045, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.89977, - "map_at_1": 0.80357, - "map_at_10": 0.87167, - "map_at_3": 0.8626, - "map_at_5": 0.8687, - "mrr_at_1": 0.8035714285714286, - "mrr_at_10": 0.8716671390778533, - "mrr_at_3": 0.8625992063492063, - "mrr_at_5": 0.8687003968253968, - "ndcg_at_1": 0.80357, - "ndcg_at_10": 0.89977, - "ndcg_at_3": 0.88184, - "ndcg_at_5": 0.89285, - "precision_at_1": 0.80357, - "precision_at_10": 0.09851, - "precision_at_3": 0.3125, - "precision_at_5": 0.19286, - "recall_at_1": 0.80357, - "recall_at_10": 0.98512, - "recall_at_3": 0.9375, - "recall_at_5": 0.96429 - } - ] - }, - "task_name": "LEMBSummScreenFDRetrievalChunked" -} \ No newline at end of file diff --git a/results-chunked-pooling/chunk_size_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-chunked-pooling/chunk_size_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json deleted file mode 100644 index b7a6aff..0000000 --- a/results-chunked-pooling/chunk_size_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 49.683130741119385, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.9641, - "map_at_1": 0.93333, - "map_at_10": 0.95644, - "map_at_20": 0.95715, - "map_at_3": 0.95611, - "map_at_5": 0.95611, - "mrr_at_1": 0.9333333333333333, - "mrr_at_10": 0.9564444444444444, - "mrr_at_20": 0.9571522366522366, - "mrr_at_3": 0.956111111111111, - "mrr_at_5": 0.956111111111111, - "ndcg_at_1": 0.93333, - "ndcg_at_10": 0.9641, - "ndcg_at_20": 0.96664, - "ndcg_at_3": 0.96313, - "ndcg_at_5": 0.96313, - "precision_at_1": 0.93333, - "precision_at_10": 0.09867, - "precision_at_20": 0.04983, - "precision_at_3": 0.32778, - "precision_at_5": 0.19667, - "recall_at_1": 0.93333, - "recall_at_10": 0.98667, - "recall_at_20": 0.99667, - "recall_at_3": 0.98333, - "recall_at_5": 0.98333 - } - ] - }, - "task_name": "LEMBWikimQARetrievalChunked" -} \ No newline at end of file diff --git a/results-chunked-pooling/chunk_size_32/no_model_name_available/no_revision_available/model_meta.json b/results-chunked-pooling/chunk_size_32/no_model_name_available/no_revision_available/model_meta.json deleted file mode 100644 index 40ff52f..0000000 --- a/results-chunked-pooling/chunk_size_32/no_model_name_available/no_revision_available/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-chunked-pooling/chunk_size_512/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-chunked-pooling/chunk_size_512/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json deleted file mode 100644 index 8c96237..0000000 --- a/results-chunked-pooling/chunk_size_512/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 41.24476718902588, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.86921, - "map_at_1": 0.75595, - "map_at_10": 0.83613, - "map_at_20": 0.83699, - "map_at_3": 0.82391, - "map_at_5": 0.8315, - "mrr_at_1": 0.7559523809523809, - "mrr_at_10": 0.8361335506424792, - "mrr_at_20": 0.8369887222565795, - "mrr_at_3": 0.8239087301587301, - "mrr_at_5": 0.8314980158730159, - "ndcg_at_1": 0.75595, - "ndcg_at_10": 0.86921, - "ndcg_at_20": 0.87226, - "ndcg_at_3": 0.84478, - "ndcg_at_5": 0.85835, - "precision_at_1": 0.75595, - "precision_at_10": 0.09702, - "precision_at_20": 0.04911, - "precision_at_3": 0.30159, - "precision_at_5": 0.1875, - "recall_at_1": 0.75595, - "recall_at_10": 0.97024, - "recall_at_20": 0.98214, - "recall_at_3": 0.90476, - "recall_at_5": 0.9375 - } - ] - }, - "task_name": "LEMBSummScreenFDRetrievalChunked" -} \ No newline at end of file diff --git a/results-chunked-pooling/chunk_size_512/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-chunked-pooling/chunk_size_512/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json deleted file mode 100644 index 188188a..0000000 --- a/results-chunked-pooling/chunk_size_512/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 37.502394676208496, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.93658, - "map_at_1": 0.89667, - "map_at_10": 0.92556, - "map_at_20": 0.92677, - "map_at_3": 0.92111, - "map_at_5": 0.92411, - "mrr_at_1": 0.8966666666666666, - "mrr_at_10": 0.9255595238095238, - "mrr_at_20": 0.9267734025234025, - "mrr_at_3": 0.9211111111111111, - "mrr_at_5": 0.9241111111111112, - "ndcg_at_1": 0.89667, - "ndcg_at_10": 0.93658, - "ndcg_at_20": 0.94087, - "ndcg_at_3": 0.92778, - "ndcg_at_5": 0.93323, - "precision_at_1": 0.89667, - "precision_at_10": 0.097, - "precision_at_20": 0.04933, - "precision_at_3": 0.31556, - "precision_at_5": 0.192, - "recall_at_1": 0.89667, - "recall_at_10": 0.97, - "recall_at_20": 0.98667, - "recall_at_3": 0.94667, - "recall_at_5": 0.96 - } - ] - }, - "task_name": "LEMBWikimQARetrievalChunked" -} \ No newline at end of file diff --git a/results-chunked-pooling/chunk_size_512/no_model_name_available/no_revision_available/model_meta.json b/results-chunked-pooling/chunk_size_512/no_model_name_available/no_revision_available/model_meta.json deleted file mode 100644 index 40ff52f..0000000 --- a/results-chunked-pooling/chunk_size_512/no_model_name_available/no_revision_available/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-chunked-pooling/chunk_size_64/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-chunked-pooling/chunk_size_64/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json deleted file mode 100644 index f1eab61..0000000 --- a/results-chunked-pooling/chunk_size_64/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 49.614307165145874, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.8939, - "map_at_1": 0.79167, - "map_at_10": 0.86543, - "map_at_20": 0.86627, - "map_at_3": 0.85764, - "map_at_5": 0.86314, - "mrr_at_1": 0.7916666666666666, - "mrr_at_10": 0.8654301303854876, - "mrr_at_20": 0.8662683324924866, - "mrr_at_3": 0.857638888888889, - "mrr_at_5": 0.8631448412698413, - "ndcg_at_1": 0.79167, - "ndcg_at_10": 0.8939, - "ndcg_at_20": 0.89693, - "ndcg_at_3": 0.87893, - "ndcg_at_5": 0.88879, - "precision_at_1": 0.79167, - "precision_at_10": 0.09792, - "precision_at_20": 0.04955, - "precision_at_3": 0.31349, - "precision_at_5": 0.19286, - "recall_at_1": 0.79167, - "recall_at_10": 0.97917, - "recall_at_20": 0.99107, - "recall_at_3": 0.94048, - "recall_at_5": 0.96429 - } - ] - }, - "task_name": "LEMBSummScreenFDRetrievalChunked" -} \ No newline at end of file diff --git a/results-chunked-pooling/chunk_size_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-chunked-pooling/chunk_size_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json deleted file mode 100644 index 39ae3d1..0000000 --- a/results-chunked-pooling/chunk_size_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 44.609771728515625, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.96349, - "map_at_1": 0.93333, - "map_at_10": 0.95572, - "map_at_20": 0.95623, - "map_at_3": 0.95389, - "map_at_5": 0.95539, - "mrr_at_1": 0.9333333333333333, - "mrr_at_10": 0.9557222222222221, - "mrr_at_20": 0.9562335858585859, - "mrr_at_3": 0.9538888888888889, - "mrr_at_5": 0.955388888888889, - "ndcg_at_1": 0.93333, - "ndcg_at_10": 0.96349, - "ndcg_at_20": 0.96523, - "ndcg_at_3": 0.9598, - "ndcg_at_5": 0.96253, - "precision_at_1": 0.93333, - "precision_at_10": 0.09867, - "precision_at_20": 0.04967, - "precision_at_3": 0.32556, - "precision_at_5": 0.19667, - "recall_at_1": 0.93333, - "recall_at_10": 0.98667, - "recall_at_20": 0.99333, - "recall_at_3": 0.97667, - "recall_at_5": 0.98333 - } - ] - }, - "task_name": "LEMBWikimQARetrievalChunked" -} \ No newline at end of file diff --git a/results-chunked-pooling/chunk_size_64/no_model_name_available/no_revision_available/model_meta.json b/results-chunked-pooling/chunk_size_64/no_model_name_available/no_revision_available/model_meta.json deleted file mode 100644 index 40ff52f..0000000 --- a/results-chunked-pooling/chunk_size_64/no_model_name_available/no_revision_available/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-chunked-pooling/chunk_size_8/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json b/results-chunked-pooling/chunk_size_8/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json deleted file mode 100644 index 02c80b5..0000000 --- a/results-chunked-pooling/chunk_size_8/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 441.941686630249, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.62432, - "map_at_1": 0.48134, - "map_at_10": 0.57465, - "map_at_20": 0.57982, - "map_at_3": 0.54923, - "map_at_5": 0.56285, - "mrr_at_1": 0.481335952848723, - "mrr_at_10": 0.5746509911539381, - "mrr_at_20": 0.5798247824096785, - "mrr_at_3": 0.549225060030561, - "mrr_at_5": 0.5628465400567562, - "ndcg_at_1": 0.48134, - "ndcg_at_10": 0.62432, - "ndcg_at_20": 0.64351, - "ndcg_at_3": 0.57116, - "ndcg_at_5": 0.59585, - "precision_at_1": 0.48134, - "precision_at_10": 0.07826, - "precision_at_20": 0.04296, - "precision_at_3": 0.21153, - "precision_at_5": 0.13897, - "recall_at_1": 0.48134, - "recall_at_10": 0.78258, - "recall_at_20": 0.8592, - "recall_at_3": 0.63458, - "recall_at_5": 0.69483 - } - ] - }, - "task_name": "LEMBQMSumRetrievalChunked" -} \ No newline at end of file diff --git a/results-chunked-pooling/chunk_size_8/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-chunked-pooling/chunk_size_8/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json deleted file mode 100644 index 29940f6..0000000 --- a/results-chunked-pooling/chunk_size_8/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 139.2442500591278, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.91679, - "map_at_1": 0.83333, - "map_at_10": 0.89142, - "map_at_20": 0.89142, - "map_at_3": 0.87847, - "map_at_5": 0.88963, - "mrr_at_1": 0.8333333333333334, - "mrr_at_10": 0.8914186507936508, - "mrr_at_20": 0.8914186507936508, - "mrr_at_3": 0.8784722222222223, - "mrr_at_5": 0.8896329365079365, - "ndcg_at_1": 0.83333, - "ndcg_at_10": 0.91679, - "ndcg_at_20": 0.91679, - "ndcg_at_3": 0.89289, - "ndcg_at_5": 0.91275, - "precision_at_1": 0.83333, - "precision_at_10": 0.0994, - "precision_at_20": 0.0497, - "precision_at_3": 0.31151, - "precision_at_5": 0.19643, - "recall_at_1": 0.83333, - "recall_at_10": 0.99405, - "recall_at_20": 0.99405, - "recall_at_3": 0.93452, - "recall_at_5": 0.98214 - } - ] - }, - "task_name": "LEMBSummScreenFDRetrievalChunked" -} \ No newline at end of file diff --git a/results-chunked-pooling/chunk_size_8/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-chunked-pooling/chunk_size_8/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json deleted file mode 100644 index 157d1f3..0000000 --- a/results-chunked-pooling/chunk_size_8/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 114.65108251571655, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.9728, - "map_at_1": 0.95, - "map_at_10": 0.96614, - "map_at_20": 0.96636, - "map_at_3": 0.96389, - "map_at_5": 0.96539, - "mrr_at_1": 0.95, - "mrr_at_10": 0.9661388888888888, - "mrr_at_20": 0.9663611111111112, - "mrr_at_3": 0.9638888888888889, - "mrr_at_5": 0.965388888888889, - "ndcg_at_1": 0.95, - "ndcg_at_10": 0.9728, - "ndcg_at_20": 0.97363, - "ndcg_at_3": 0.96806, - "ndcg_at_5": 0.97078, - "precision_at_1": 0.95, - "precision_at_10": 0.09933, - "precision_at_20": 0.04983, - "precision_at_3": 0.32667, - "precision_at_5": 0.19733, - "recall_at_1": 0.95, - "recall_at_10": 0.99333, - "recall_at_20": 0.99667, - "recall_at_3": 0.98, - "recall_at_5": 0.98667 - } - ] - }, - "task_name": "LEMBWikimQARetrievalChunked" -} \ No newline at end of file diff --git a/results-chunked-pooling/chunk_size_8/no_model_name_available/no_revision_available/model_meta.json b/results-chunked-pooling/chunk_size_8/no_model_name_available/no_revision_available/model_meta.json deleted file mode 100644 index 40ff52f..0000000 --- a/results-chunked-pooling/chunk_size_8/no_model_name_available/no_revision_available/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-hard-boundary/embed_size_1024/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json b/results-hard-boundary/embed_size_1024/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json deleted file mode 100644 index 7ecc33f..0000000 --- a/results-hard-boundary/embed_size_1024/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 34.07488179206848, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.56988, - "map_at_1": 0.42698, - "map_at_10": 0.52032, - "map_at_20": 0.52581, - "map_at_3": 0.49356, - "map_at_5": 0.50839, - "mrr_at_1": 0.426981008513425, - "mrr_at_10": 0.520315277387969, - "mrr_at_20": 0.5258095925001836, - "mrr_at_3": 0.49356035800043657, - "mrr_at_5": 0.5083933638943462, - "ndcg_at_1": 0.42698, - "ndcg_at_10": 0.56988, - "ndcg_at_20": 0.59002, - "ndcg_at_3": 0.51484, - "ndcg_at_5": 0.54142, - "precision_at_1": 0.42698, - "precision_at_10": 0.07276, - "precision_at_20": 0.04037, - "precision_at_3": 0.1921, - "precision_at_5": 0.12809, - "recall_at_1": 0.42698, - "recall_at_10": 0.72757, - "recall_at_20": 0.80747, - "recall_at_3": 0.57629, - "recall_at_5": 0.64047 - } - ] - }, - "task_name": "LEMBQMSumRetrievalChunked" -} \ No newline at end of file diff --git a/results-hard-boundary/embed_size_1024/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-hard-boundary/embed_size_1024/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json deleted file mode 100644 index 3b7fadd..0000000 --- a/results-hard-boundary/embed_size_1024/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 16.640787363052368, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.87932, - "map_at_1": 0.77083, - "map_at_10": 0.84754, - "map_at_3": 0.83631, - "map_at_5": 0.84375, - "mrr_at_1": 0.7708333333333334, - "mrr_at_10": 0.847535194633409, - "mrr_at_3": 0.8363095238095238, - "mrr_at_5": 0.84375, - "ndcg_at_1": 0.77083, - "ndcg_at_10": 0.87932, - "ndcg_at_3": 0.857, - "ndcg_at_5": 0.87045, - "precision_at_1": 0.77083, - "precision_at_10": 0.09762, - "precision_at_3": 0.30556, - "precision_at_5": 0.18988, - "recall_at_1": 0.77083, - "recall_at_10": 0.97619, - "recall_at_3": 0.91667, - "recall_at_5": 0.9494 - } - ] - }, - "task_name": "LEMBSummScreenFDRetrievalChunked" -} \ No newline at end of file diff --git a/results-hard-boundary/embed_size_1024/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-hard-boundary/embed_size_1024/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json deleted file mode 100644 index b8b7910..0000000 --- a/results-hard-boundary/embed_size_1024/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 15.625593185424805, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.94987, - "map_at_1": 0.91333, - "map_at_10": 0.93901, - "map_at_3": 0.93389, - "map_at_5": 0.93772, - "mrr_at_1": 0.9133333333333333, - "mrr_at_10": 0.9390079365079366, - "mrr_at_3": 0.933888888888889, - "mrr_at_5": 0.9377222222222222, - "ndcg_at_1": 0.91333, - "ndcg_at_10": 0.94987, - "ndcg_at_3": 0.9398, - "ndcg_at_5": 0.94669, - "precision_at_1": 0.91333, - "precision_at_10": 0.09833, - "precision_at_3": 0.31889, - "precision_at_5": 0.19467, - "recall_at_1": 0.91333, - "recall_at_10": 0.98333, - "recall_at_3": 0.95667, - "recall_at_5": 0.97333 - } - ] - }, - "task_name": "LEMBWikimQARetrievalChunked" -} \ No newline at end of file diff --git a/results-hard-boundary/embed_size_1024/no_model_name_available/no_revision_available/model_meta.json b/results-hard-boundary/embed_size_1024/no_model_name_available/no_revision_available/model_meta.json deleted file mode 100644 index 40ff52f..0000000 --- a/results-hard-boundary/embed_size_1024/no_model_name_available/no_revision_available/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-hard-boundary/embed_size_2048/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json b/results-hard-boundary/embed_size_2048/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json deleted file mode 100644 index c75facf..0000000 --- a/results-hard-boundary/embed_size_2048/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 38.28924012184143, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.57083, - "map_at_1": 0.42895, - "map_at_10": 0.52134, - "map_at_20": 0.52685, - "map_at_3": 0.49552, - "map_at_5": 0.5101, - "mrr_at_1": 0.4289456450556647, - "mrr_at_10": 0.5213446325921768, - "mrr_at_20": 0.5268485590590946, - "mrr_at_3": 0.49552499454267623, - "mrr_at_5": 0.5100960488976206, - "ndcg_at_1": 0.42895, - "ndcg_at_10": 0.57083, - "ndcg_at_20": 0.59108, - "ndcg_at_3": 0.51743, - "ndcg_at_5": 0.54378, - "precision_at_1": 0.42895, - "precision_at_10": 0.07282, - "precision_at_20": 0.04044, - "precision_at_3": 0.19363, - "precision_at_5": 0.12901, - "recall_at_1": 0.42895, - "recall_at_10": 0.72823, - "recall_at_20": 0.80878, - "recall_at_3": 0.58088, - "recall_at_5": 0.64506 - } - ] - }, - "task_name": "LEMBQMSumRetrievalChunked" -} \ No newline at end of file diff --git a/results-hard-boundary/embed_size_2048/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-hard-boundary/embed_size_2048/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json deleted file mode 100644 index 0653cf7..0000000 --- a/results-hard-boundary/embed_size_2048/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 21.060544967651367, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.89083, - "map_at_1": 0.79762, - "map_at_10": 0.86375, - "map_at_3": 0.85417, - "map_at_5": 0.86027, - "mrr_at_1": 0.7976190476190477, - "mrr_at_10": 0.8637518896447467, - "mrr_at_3": 0.8541666666666666, - "mrr_at_5": 0.8602678571428571, - "ndcg_at_1": 0.79762, - "ndcg_at_10": 0.89083, - "ndcg_at_3": 0.87181, - "ndcg_at_5": 0.88282, - "precision_at_1": 0.79762, - "precision_at_10": 0.09732, - "precision_at_3": 0.30754, - "precision_at_5": 0.18988, - "recall_at_1": 0.79762, - "recall_at_10": 0.97321, - "recall_at_3": 0.92262, - "recall_at_5": 0.9494 - } - ] - }, - "task_name": "LEMBSummScreenFDRetrievalChunked" -} \ No newline at end of file diff --git a/results-hard-boundary/embed_size_2048/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-hard-boundary/embed_size_2048/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json deleted file mode 100644 index 7f50882..0000000 --- a/results-hard-boundary/embed_size_2048/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 19.39264702796936, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.94552, - "map_at_1": 0.91, - "map_at_10": 0.93434, - "map_at_3": 0.93, - "map_at_5": 0.93233, - "mrr_at_1": 0.91, - "mrr_at_10": 0.9343373015873015, - "mrr_at_3": 0.93, - "mrr_at_5": 0.9323333333333333, - "ndcg_at_1": 0.91, - "ndcg_at_10": 0.94552, - "ndcg_at_3": 0.93682, - "ndcg_at_5": 0.94099, - "precision_at_1": 0.91, - "precision_at_10": 0.098, - "precision_at_3": 0.31889, - "precision_at_5": 0.19333, - "recall_at_1": 0.91, - "recall_at_10": 0.98, - "recall_at_3": 0.95667, - "recall_at_5": 0.96667 - } - ] - }, - "task_name": "LEMBWikimQARetrievalChunked" -} \ No newline at end of file diff --git a/results-hard-boundary/embed_size_2048/no_model_name_available/no_revision_available/model_meta.json b/results-hard-boundary/embed_size_2048/no_model_name_available/no_revision_available/model_meta.json deleted file mode 100644 index 40ff52f..0000000 --- a/results-hard-boundary/embed_size_2048/no_model_name_available/no_revision_available/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-hard-boundary/embed_size_4096/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json b/results-hard-boundary/embed_size_4096/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json deleted file mode 100644 index 36e918f..0000000 --- a/results-hard-boundary/embed_size_4096/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 45.90134239196777, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.571, - "map_at_1": 0.42829, - "map_at_10": 0.52151, - "map_at_20": 0.52705, - "map_at_3": 0.49596, - "map_at_5": 0.5103, - "mrr_at_1": 0.42829076620825146, - "mrr_at_10": 0.5215057535784451, - "mrr_at_20": 0.5270534163370235, - "mrr_at_3": 0.4959615804409518, - "mrr_at_5": 0.5103034271993014, - "ndcg_at_1": 0.42829, - "ndcg_at_10": 0.571, - "ndcg_at_20": 0.59141, - "ndcg_at_3": 0.51826, - "ndcg_at_5": 0.54413, - "precision_at_1": 0.42829, - "precision_at_10": 0.07282, - "precision_at_20": 0.04047, - "precision_at_3": 0.19428, - "precision_at_5": 0.12914, - "recall_at_1": 0.42829, - "recall_at_10": 0.72823, - "recall_at_20": 0.80943, - "recall_at_3": 0.58284, - "recall_at_5": 0.64571 - } - ] - }, - "task_name": "LEMBQMSumRetrievalChunked" -} \ No newline at end of file diff --git a/results-hard-boundary/embed_size_4096/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-hard-boundary/embed_size_4096/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json deleted file mode 100644 index 9506068..0000000 --- a/results-hard-boundary/embed_size_4096/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 28.929830312728882, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.888, - "map_at_1": 0.78869, - "map_at_10": 0.85985, - "map_at_3": 0.85069, - "map_at_5": 0.85694, - "mrr_at_1": 0.7886904761904762, - "mrr_at_10": 0.8598462301587302, - "mrr_at_3": 0.8506944444444444, - "mrr_at_5": 0.8569444444444445, - "ndcg_at_1": 0.78869, - "ndcg_at_10": 0.888, - "ndcg_at_3": 0.87, - "ndcg_at_5": 0.88115, - "precision_at_1": 0.78869, - "precision_at_10": 0.09732, - "precision_at_3": 0.30853, - "precision_at_5": 0.19048, - "recall_at_1": 0.78869, - "recall_at_10": 0.97321, - "recall_at_3": 0.9256, - "recall_at_5": 0.95238 - } - ] - }, - "task_name": "LEMBSummScreenFDRetrievalChunked" -} \ No newline at end of file diff --git a/results-hard-boundary/embed_size_4096/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-hard-boundary/embed_size_4096/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json deleted file mode 100644 index aa1f560..0000000 --- a/results-hard-boundary/embed_size_4096/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 26.394963026046753, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.94626, - "map_at_1": 0.90667, - "map_at_10": 0.9342, - "map_at_3": 0.93, - "map_at_5": 0.93283, - "mrr_at_1": 0.9066666666666666, - "mrr_at_10": 0.9341984126984126, - "mrr_at_3": 0.93, - "mrr_at_5": 0.9328333333333334, - "ndcg_at_1": 0.90667, - "ndcg_at_10": 0.94626, - "ndcg_at_3": 0.9377, - "ndcg_at_5": 0.943, - "precision_at_1": 0.90667, - "precision_at_10": 0.09833, - "precision_at_3": 0.32, - "precision_at_5": 0.19467, - "recall_at_1": 0.90667, - "recall_at_10": 0.98333, - "recall_at_3": 0.96, - "recall_at_5": 0.97333 - } - ] - }, - "task_name": "LEMBWikimQARetrievalChunked" -} \ No newline at end of file diff --git a/results-hard-boundary/embed_size_4096/no_model_name_available/no_revision_available/model_meta.json b/results-hard-boundary/embed_size_4096/no_model_name_available/no_revision_available/model_meta.json deleted file mode 100644 index 40ff52f..0000000 --- a/results-hard-boundary/embed_size_4096/no_model_name_available/no_revision_available/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-hard-boundary/embed_size_8192/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json b/results-hard-boundary/embed_size_8192/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json deleted file mode 100644 index 6ab0b4c..0000000 --- a/results-hard-boundary/embed_size_8192/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 59.748815298080444, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.57042, - "map_at_1": 0.42698, - "map_at_10": 0.52075, - "map_at_20": 0.52611, - "map_at_3": 0.49509, - "map_at_5": 0.50989, - "mrr_at_1": 0.426981008513425, - "mrr_at_10": 0.5207518632862445, - "mrr_at_20": 0.526107118718291, - "mrr_at_3": 0.4950884086444008, - "mrr_at_5": 0.5098886705959398, - "ndcg_at_1": 0.42698, - "ndcg_at_10": 0.57042, - "ndcg_at_20": 0.59027, - "ndcg_at_3": 0.51745, - "ndcg_at_5": 0.54413, - "precision_at_1": 0.42698, - "precision_at_10": 0.07282, - "precision_at_20": 0.04037, - "precision_at_3": 0.19406, - "precision_at_5": 0.1294, - "recall_at_1": 0.42698, - "recall_at_10": 0.72823, - "recall_at_20": 0.80747, - "recall_at_3": 0.58219, - "recall_at_5": 0.64702 - } - ] - }, - "task_name": "LEMBQMSumRetrievalChunked" -} \ No newline at end of file diff --git a/results-hard-boundary/embed_size_8192/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-hard-boundary/embed_size_8192/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json deleted file mode 100644 index 69b97ae..0000000 --- a/results-hard-boundary/embed_size_8192/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 41.61167931556702, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.88999, - "map_at_1": 0.79464, - "map_at_10": 0.86334, - "map_at_3": 0.85516, - "map_at_5": 0.86081, - "mrr_at_1": 0.7946428571428571, - "mrr_at_10": 0.863343253968254, - "mrr_at_3": 0.8551587301587301, - "mrr_at_5": 0.8608134920634921, - "ndcg_at_1": 0.79464, - "ndcg_at_10": 0.88999, - "ndcg_at_3": 0.87408, - "ndcg_at_5": 0.88407, - "precision_at_1": 0.79464, - "precision_at_10": 0.09702, - "precision_at_3": 0.30952, - "precision_at_5": 0.19048, - "recall_at_1": 0.79464, - "recall_at_10": 0.97024, - "recall_at_3": 0.92857, - "recall_at_5": 0.95238 - } - ] - }, - "task_name": "LEMBSummScreenFDRetrievalChunked" -} \ No newline at end of file diff --git a/results-hard-boundary/embed_size_8192/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-hard-boundary/embed_size_8192/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json deleted file mode 100644 index fa8bab3..0000000 --- a/results-hard-boundary/embed_size_8192/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 37.43858456611633, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.94608, - "map_at_1": 0.90667, - "map_at_10": 0.93403, - "map_at_3": 0.93, - "map_at_5": 0.93233, - "mrr_at_1": 0.9066666666666666, - "mrr_at_10": 0.9340317460317461, - "mrr_at_3": 0.93, - "mrr_at_5": 0.9323333333333333, - "ndcg_at_1": 0.90667, - "ndcg_at_10": 0.94608, - "ndcg_at_3": 0.9377, - "ndcg_at_5": 0.94186, - "precision_at_1": 0.90667, - "precision_at_10": 0.09833, - "precision_at_3": 0.32, - "precision_at_5": 0.194, - "recall_at_1": 0.90667, - "recall_at_10": 0.98333, - "recall_at_3": 0.96, - "recall_at_5": 0.97 - } - ] - }, - "task_name": "LEMBWikimQARetrievalChunked" -} \ No newline at end of file diff --git a/results-hard-boundary/embed_size_8192/no_model_name_available/no_revision_available/model_meta.json b/results-hard-boundary/embed_size_8192/no_model_name_available/no_revision_available/model_meta.json deleted file mode 100644 index 40ff52f..0000000 --- a/results-hard-boundary/embed_size_8192/no_model_name_available/no_revision_available/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-normal-pooling/chunk_size_1024/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-normal-pooling/chunk_size_1024/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json deleted file mode 100644 index dac3893..0000000 --- a/results-normal-pooling/chunk_size_1024/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 22.880125284194946, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.90436, - "map_at_1": 0.8244, - "map_at_10": 0.88092, - "map_at_20": 0.88152, - "map_at_3": 0.8745, - "map_at_5": 0.87793, - "mrr_at_1": 0.8244047619047619, - "mrr_at_10": 0.8809216742252456, - "mrr_at_20": 0.8815229248264962, - "mrr_at_3": 0.8745039682539681, - "mrr_at_5": 0.8779265873015873, - "ndcg_at_1": 0.8244, - "ndcg_at_10": 0.90436, - "ndcg_at_20": 0.90659, - "ndcg_at_3": 0.89069, - "ndcg_at_5": 0.89684, - "precision_at_1": 0.8244, - "precision_at_10": 0.09762, - "precision_at_20": 0.04926, - "precision_at_3": 0.3125, - "precision_at_5": 0.19048, - "recall_at_1": 0.8244, - "recall_at_10": 0.97619, - "recall_at_20": 0.98512, - "recall_at_3": 0.9375, - "recall_at_5": 0.95238 - } - ] - }, - "task_name": "LEMBSummScreenFDRetrievalChunked" -} \ No newline at end of file diff --git a/results-normal-pooling/chunk_size_1024/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-normal-pooling/chunk_size_1024/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json deleted file mode 100644 index d39cb18..0000000 --- a/results-normal-pooling/chunk_size_1024/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 21.713423252105713, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.89332, - "map_at_1": 0.81667, - "map_at_10": 0.87223, - "map_at_20": 0.87337, - "map_at_3": 0.86667, - "map_at_5": 0.86967, - "mrr_at_1": 0.8166666666666667, - "mrr_at_10": 0.8722261904761904, - "mrr_at_20": 0.8733667133520074, - "mrr_at_3": 0.8666666666666667, - "mrr_at_5": 0.8696666666666666, - "ndcg_at_1": 0.81667, - "ndcg_at_10": 0.89332, - "ndcg_at_20": 0.89751, - "ndcg_at_3": 0.88214, - "ndcg_at_5": 0.88759, - "precision_at_1": 0.81667, - "precision_at_10": 0.09567, - "precision_at_20": 0.04867, - "precision_at_3": 0.30889, - "precision_at_5": 0.188, - "recall_at_1": 0.81667, - "recall_at_10": 0.95667, - "recall_at_20": 0.97333, - "recall_at_3": 0.92667, - "recall_at_5": 0.94 - } - ] - }, - "task_name": "LEMBWikimQARetrievalChunked" -} \ No newline at end of file diff --git a/results-normal-pooling/chunk_size_1024/no_model_name_available/no_revision_available/model_meta.json b/results-normal-pooling/chunk_size_1024/no_model_name_available/no_revision_available/model_meta.json deleted file mode 100644 index 40ff52f..0000000 --- a/results-normal-pooling/chunk_size_1024/no_model_name_available/no_revision_available/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-normal-pooling/chunk_size_128/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-normal-pooling/chunk_size_128/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json deleted file mode 100644 index 2c82561..0000000 --- a/results-normal-pooling/chunk_size_128/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 45.15112638473511, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.8809, - "map_at_1": 0.75595, - "map_at_10": 0.84616, - "map_at_20": 0.84616, - "map_at_3": 0.83532, - "map_at_5": 0.84395, - "mrr_at_1": 0.7559523809523809, - "mrr_at_10": 0.8461592970521542, - "mrr_at_20": 0.8461592970521542, - "mrr_at_3": 0.8353174603174603, - "mrr_at_5": 0.8439484126984127, - "ndcg_at_1": 0.75595, - "ndcg_at_10": 0.8809, - "ndcg_at_20": 0.8809, - "ndcg_at_3": 0.86012, - "ndcg_at_5": 0.87587, - "precision_at_1": 0.75595, - "precision_at_10": 0.09851, - "precision_at_20": 0.04926, - "precision_at_3": 0.31052, - "precision_at_5": 0.19405, - "recall_at_1": 0.75595, - "recall_at_10": 0.98512, - "recall_at_20": 0.98512, - "recall_at_3": 0.93155, - "recall_at_5": 0.97024 - } - ] - }, - "task_name": "LEMBSummScreenFDRetrievalChunked" -} \ No newline at end of file diff --git a/results-normal-pooling/chunk_size_128/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-normal-pooling/chunk_size_128/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json deleted file mode 100644 index beb491c..0000000 --- a/results-normal-pooling/chunk_size_128/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 41.67530345916748, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.96056, - "map_at_1": 0.93333, - "map_at_10": 0.95298, - "map_at_20": 0.9536, - "map_at_3": 0.95111, - "map_at_5": 0.95194, - "mrr_at_1": 0.9333333333333333, - "mrr_at_10": 0.9529761904761905, - "mrr_at_20": 0.9535960735171262, - "mrr_at_3": 0.951111111111111, - "mrr_at_5": 0.9519444444444444, - "ndcg_at_1": 0.93333, - "ndcg_at_10": 0.96056, - "ndcg_at_20": 0.963, - "ndcg_at_3": 0.95682, - "ndcg_at_5": 0.95826, - "precision_at_1": 0.93333, - "precision_at_10": 0.09833, - "precision_at_20": 0.04967, - "precision_at_3": 0.32444, - "precision_at_5": 0.19533, - "recall_at_1": 0.93333, - "recall_at_10": 0.98333, - "recall_at_20": 0.99333, - "recall_at_3": 0.97333, - "recall_at_5": 0.97667 - } - ] - }, - "task_name": "LEMBWikimQARetrievalChunked" -} \ No newline at end of file diff --git a/results-normal-pooling/chunk_size_128/no_model_name_available/no_revision_available/model_meta.json b/results-normal-pooling/chunk_size_128/no_model_name_available/no_revision_available/model_meta.json deleted file mode 100644 index 40ff52f..0000000 --- a/results-normal-pooling/chunk_size_128/no_model_name_available/no_revision_available/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-normal-pooling/chunk_size_16/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-normal-pooling/chunk_size_16/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json deleted file mode 100644 index 1fd9a28..0000000 --- a/results-normal-pooling/chunk_size_16/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 253.56742310523987, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.77601, - "map_at_1": 0.58929, - "map_at_10": 0.71954, - "map_at_20": 0.72214, - "map_at_3": 0.69544, - "map_at_5": 0.71255, - "mrr_at_1": 0.5892857142857143, - "mrr_at_10": 0.7195401077097505, - "mrr_at_20": 0.7221398856592078, - "mrr_at_3": 0.6954365079365079, - "mrr_at_5": 0.7125496031746031, - "ndcg_at_1": 0.58929, - "ndcg_at_10": 0.77601, - "ndcg_at_20": 0.78519, - "ndcg_at_3": 0.72867, - "ndcg_at_5": 0.75941, - "precision_at_1": 0.58929, - "precision_at_10": 0.09494, - "precision_at_20": 0.04926, - "precision_at_3": 0.2748, - "precision_at_5": 0.17976, - "recall_at_1": 0.58929, - "recall_at_10": 0.9494, - "recall_at_20": 0.98512, - "recall_at_3": 0.8244, - "recall_at_5": 0.89881 - } - ] - }, - "task_name": "LEMBSummScreenFDRetrievalChunked" -} \ No newline at end of file diff --git a/results-normal-pooling/chunk_size_16/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-normal-pooling/chunk_size_16/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json deleted file mode 100644 index 3ae2554..0000000 --- a/results-normal-pooling/chunk_size_16/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 203.59319949150085, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.89535, - "map_at_1": 0.83, - "map_at_10": 0.87642, - "map_at_3": 0.87, - "map_at_5": 0.87367, - "mrr_at_1": 0.83, - "mrr_at_10": 0.8769761904761905, - "mrr_at_3": 0.8705555555555556, - "mrr_at_5": 0.8742222222222221, - "ndcg_at_1": 0.83, - "ndcg_at_10": 0.89535, - "ndcg_at_3": 0.88206, - "ndcg_at_5": 0.8888, - "precision_at_1": 0.83, - "precision_at_10": 0.09533, - "precision_at_3": 0.30556, - "precision_at_5": 0.18667, - "recall_at_1": 0.83, - "recall_at_10": 0.95333, - "recall_at_3": 0.91667, - "recall_at_5": 0.93333 - } - ] - }, - "task_name": "LEMBWikimQARetrievalChunked" -} \ No newline at end of file diff --git a/results-normal-pooling/chunk_size_16/no_model_name_available/no_revision_available/model_meta.json b/results-normal-pooling/chunk_size_16/no_model_name_available/no_revision_available/model_meta.json deleted file mode 100644 index 40ff52f..0000000 --- a/results-normal-pooling/chunk_size_16/no_model_name_available/no_revision_available/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-normal-pooling/chunk_size_256/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-normal-pooling/chunk_size_256/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json deleted file mode 100644 index a2e52e0..0000000 --- a/results-normal-pooling/chunk_size_256/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 30.23233437538147, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.89279, - "map_at_1": 0.78869, - "map_at_10": 0.86062, - "map_at_3": 0.84573, - "map_at_5": 0.85734, - "mrr_at_1": 0.7886904761904762, - "mrr_at_10": 0.8606150793650794, - "mrr_at_3": 0.8457341269841271, - "mrr_at_5": 0.8573412698412698, - "ndcg_at_1": 0.78869, - "ndcg_at_10": 0.89279, - "ndcg_at_3": 0.86469, - "ndcg_at_5": 0.88556, - "precision_at_1": 0.78869, - "precision_at_10": 0.09911, - "precision_at_3": 0.30655, - "precision_at_5": 0.19405, - "recall_at_1": 0.78869, - "recall_at_10": 0.99107, - "recall_at_3": 0.91964, - "recall_at_5": 0.97024 - } - ] - }, - "task_name": "LEMBSummScreenFDRetrievalChunked" -} \ No newline at end of file diff --git a/results-normal-pooling/chunk_size_256/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-normal-pooling/chunk_size_256/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json deleted file mode 100644 index e2fa347..0000000 --- a/results-normal-pooling/chunk_size_256/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 28.12085223197937, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.94218, - "map_at_1": 0.90667, - "map_at_10": 0.92928, - "map_at_3": 0.92389, - "map_at_5": 0.92606, - "mrr_at_1": 0.9066666666666666, - "mrr_at_10": 0.9292817460317461, - "mrr_at_3": 0.923888888888889, - "mrr_at_5": 0.9260555555555555, - "ndcg_at_1": 0.90667, - "ndcg_at_10": 0.94218, - "ndcg_at_3": 0.93052, - "ndcg_at_5": 0.93453, - "precision_at_1": 0.90667, - "precision_at_10": 0.09833, - "precision_at_3": 0.31667, - "precision_at_5": 0.192, - "recall_at_1": 0.90667, - "recall_at_10": 0.98333, - "recall_at_3": 0.95, - "recall_at_5": 0.96 - } - ] - }, - "task_name": "LEMBWikimQARetrievalChunked" -} \ No newline at end of file diff --git a/results-normal-pooling/chunk_size_256/no_model_name_available/no_revision_available/model_meta.json b/results-normal-pooling/chunk_size_256/no_model_name_available/no_revision_available/model_meta.json deleted file mode 100644 index 40ff52f..0000000 --- a/results-normal-pooling/chunk_size_256/no_model_name_available/no_revision_available/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-normal-pooling/chunk_size_32/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-normal-pooling/chunk_size_32/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json deleted file mode 100644 index f38062f..0000000 --- a/results-normal-pooling/chunk_size_32/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 121.14276123046875, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.81621, - "map_at_1": 0.64583, - "map_at_10": 0.76397, - "map_at_3": 0.74454, - "map_at_5": 0.75764, - "mrr_at_1": 0.6458333333333334, - "mrr_at_10": 0.7639691987906274, - "mrr_at_3": 0.7445436507936508, - "mrr_at_5": 0.7576388888888889, - "ndcg_at_1": 0.64583, - "ndcg_at_10": 0.81621, - "ndcg_at_3": 0.77795, - "ndcg_at_5": 0.80139, - "precision_at_1": 0.64583, - "precision_at_10": 0.09762, - "precision_at_3": 0.29167, - "precision_at_5": 0.18631, - "recall_at_1": 0.64583, - "recall_at_10": 0.97619, - "recall_at_3": 0.875, - "recall_at_5": 0.93155 - } - ] - }, - "task_name": "LEMBSummScreenFDRetrievalChunked" -} \ No newline at end of file diff --git a/results-normal-pooling/chunk_size_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-normal-pooling/chunk_size_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json deleted file mode 100644 index 3819ca8..0000000 --- a/results-normal-pooling/chunk_size_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 110.62627148628235, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.93753, - "map_at_1": 0.89333, - "map_at_10": 0.92385, - "map_at_20": 0.92385, - "map_at_3": 0.91778, - "map_at_5": 0.92161, - "mrr_at_1": 0.8933333333333333, - "mrr_at_10": 0.9238465608465608, - "mrr_at_20": 0.9238465608465608, - "mrr_at_3": 0.9177777777777777, - "mrr_at_5": 0.9216111111111112, - "ndcg_at_1": 0.89333, - "ndcg_at_10": 0.93753, - "ndcg_at_20": 0.93753, - "ndcg_at_3": 0.92524, - "ndcg_at_5": 0.93212, - "precision_at_1": 0.89333, - "precision_at_10": 0.098, - "precision_at_20": 0.049, - "precision_at_3": 0.31556, - "precision_at_5": 0.19267, - "recall_at_1": 0.89333, - "recall_at_10": 0.98, - "recall_at_20": 0.98, - "recall_at_3": 0.94667, - "recall_at_5": 0.96333 - } - ] - }, - "task_name": "LEMBWikimQARetrievalChunked" -} \ No newline at end of file diff --git a/results-normal-pooling/chunk_size_32/no_model_name_available/no_revision_available/model_meta.json b/results-normal-pooling/chunk_size_32/no_model_name_available/no_revision_available/model_meta.json deleted file mode 100644 index 40ff52f..0000000 --- a/results-normal-pooling/chunk_size_32/no_model_name_available/no_revision_available/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-normal-pooling/chunk_size_512/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-normal-pooling/chunk_size_512/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json deleted file mode 100644 index 5cb312d..0000000 --- a/results-normal-pooling/chunk_size_512/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 23.719088792800903, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.89714, - "map_at_1": 0.79762, - "map_at_10": 0.86832, - "map_at_20": 0.86871, - "map_at_3": 0.85863, - "map_at_5": 0.86354, - "mrr_at_1": 0.7976190476190477, - "mrr_at_10": 0.8683153817082389, - "mrr_at_20": 0.8687139786470143, - "mrr_at_3": 0.8586309523809523, - "mrr_at_5": 0.8635416666666667, - "ndcg_at_1": 0.79762, - "ndcg_at_10": 0.89714, - "ndcg_at_20": 0.89863, - "ndcg_at_3": 0.87744, - "ndcg_at_5": 0.88615, - "precision_at_1": 0.79762, - "precision_at_10": 0.09851, - "precision_at_20": 0.04955, - "precision_at_3": 0.31052, - "precision_at_5": 0.19048, - "recall_at_1": 0.79762, - "recall_at_10": 0.98512, - "recall_at_20": 0.99107, - "recall_at_3": 0.93155, - "recall_at_5": 0.95238 - } - ] - }, - "task_name": "LEMBSummScreenFDRetrievalChunked" -} \ No newline at end of file diff --git a/results-normal-pooling/chunk_size_512/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-normal-pooling/chunk_size_512/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json deleted file mode 100644 index e743fa2..0000000 --- a/results-normal-pooling/chunk_size_512/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 22.589195013046265, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.91776, - "map_at_1": 0.86, - "map_at_10": 0.89989, - "map_at_20": 0.90087, - "map_at_3": 0.89389, - "map_at_5": 0.89689, - "mrr_at_1": 0.86, - "mrr_at_10": 0.899887566137566, - "mrr_at_20": 0.9008698535869589, - "mrr_at_3": 0.893888888888889, - "mrr_at_5": 0.8968888888888888, - "ndcg_at_1": 0.86, - "ndcg_at_10": 0.91776, - "ndcg_at_20": 0.92175, - "ndcg_at_3": 0.90488, - "ndcg_at_5": 0.91033, - "precision_at_1": 0.86, - "precision_at_10": 0.09733, - "precision_at_20": 0.0495, - "precision_at_3": 0.31222, - "precision_at_5": 0.19, - "recall_at_1": 0.86, - "recall_at_10": 0.97333, - "recall_at_20": 0.99, - "recall_at_3": 0.93667, - "recall_at_5": 0.95 - } - ] - }, - "task_name": "LEMBWikimQARetrievalChunked" -} \ No newline at end of file diff --git a/results-normal-pooling/chunk_size_512/no_model_name_available/no_revision_available/model_meta.json b/results-normal-pooling/chunk_size_512/no_model_name_available/no_revision_available/model_meta.json deleted file mode 100644 index 40ff52f..0000000 --- a/results-normal-pooling/chunk_size_512/no_model_name_available/no_revision_available/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-normal-pooling/chunk_size_64/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-normal-pooling/chunk_size_64/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json deleted file mode 100644 index 1a185d8..0000000 --- a/results-normal-pooling/chunk_size_64/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 70.64590835571289, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.85213, - "map_at_1": 0.71726, - "map_at_10": 0.81326, - "map_at_20": 0.81467, - "map_at_3": 0.79613, - "map_at_5": 0.80967, - "mrr_at_1": 0.7172619047619048, - "mrr_at_10": 0.8132594009826152, - "mrr_at_20": 0.8146659309605737, - "mrr_at_3": 0.7961309523809523, - "mrr_at_5": 0.8096726190476191, - "ndcg_at_1": 0.71726, - "ndcg_at_10": 0.85213, - "ndcg_at_20": 0.85734, - "ndcg_at_3": 0.81962, - "ndcg_at_5": 0.84345, - "precision_at_1": 0.71726, - "precision_at_10": 0.09702, - "precision_at_20": 0.04955, - "precision_at_3": 0.29563, - "precision_at_5": 0.18869, - "recall_at_1": 0.71726, - "recall_at_10": 0.97024, - "recall_at_20": 0.99107, - "recall_at_3": 0.8869, - "recall_at_5": 0.94345 - } - ] - }, - "task_name": "LEMBSummScreenFDRetrievalChunked" -} \ No newline at end of file diff --git a/results-normal-pooling/chunk_size_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-normal-pooling/chunk_size_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json deleted file mode 100644 index 0f1bdb6..0000000 --- a/results-normal-pooling/chunk_size_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 65.4407320022583, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.95835, - "map_at_1": 0.92333, - "map_at_10": 0.94886, - "map_at_20": 0.94914, - "map_at_3": 0.94611, - "map_at_5": 0.94844, - "mrr_at_1": 0.9233333333333333, - "mrr_at_10": 0.9488611111111112, - "mrr_at_20": 0.9491388888888889, - "mrr_at_3": 0.946111111111111, - "mrr_at_5": 0.9484444444444445, - "ndcg_at_1": 0.92333, - "ndcg_at_10": 0.95835, - "ndcg_at_20": 0.95925, - "ndcg_at_3": 0.95313, - "ndcg_at_5": 0.95729, - "precision_at_1": 0.92333, - "precision_at_10": 0.09867, - "precision_at_20": 0.0495, - "precision_at_3": 0.32444, - "precision_at_5": 0.19667, - "recall_at_1": 0.92333, - "recall_at_10": 0.98667, - "recall_at_20": 0.99, - "recall_at_3": 0.97333, - "recall_at_5": 0.98333 - } - ] - }, - "task_name": "LEMBWikimQARetrievalChunked" -} \ No newline at end of file diff --git a/results-normal-pooling/chunk_size_64/no_model_name_available/no_revision_available/model_meta.json b/results-normal-pooling/chunk_size_64/no_model_name_available/no_revision_available/model_meta.json deleted file mode 100644 index 40ff52f..0000000 --- a/results-normal-pooling/chunk_size_64/no_model_name_available/no_revision_available/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-normal-pooling/chunk_size_8/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-normal-pooling/chunk_size_8/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json deleted file mode 100644 index 11f468c..0000000 --- a/results-normal-pooling/chunk_size_8/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 484.68729400634766, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.70607, - "map_at_1": 0.50298, - "map_at_10": 0.63553, - "map_at_20": 0.6394, - "map_at_3": 0.59772, - "map_at_5": 0.62197, - "mrr_at_1": 0.5029761904761905, - "mrr_at_10": 0.6355324074074075, - "mrr_at_20": 0.6393996706102799, - "mrr_at_3": 0.597718253968254, - "mrr_at_5": 0.6219742063492063, - "ndcg_at_1": 0.50298, - "ndcg_at_10": 0.70607, - "ndcg_at_20": 0.7203, - "ndcg_at_3": 0.63056, - "ndcg_at_5": 0.67386, - "precision_at_1": 0.50298, - "precision_at_10": 0.09286, - "precision_at_20": 0.04926, - "precision_at_3": 0.24206, - "precision_at_5": 0.16607, - "recall_at_1": 0.50298, - "recall_at_10": 0.92857, - "recall_at_20": 0.98512, - "recall_at_3": 0.72619, - "recall_at_5": 0.83036 - } - ] - }, - "task_name": "LEMBSummScreenFDRetrievalChunked" -} \ No newline at end of file diff --git a/results-normal-pooling/chunk_size_8/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-normal-pooling/chunk_size_8/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json deleted file mode 100644 index 4255b91..0000000 --- a/results-normal-pooling/chunk_size_8/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 435.66471695899963, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.92234, - "map_at_1": 0.86, - "map_at_10": 0.90339, - "map_at_20": 0.90372, - "map_at_3": 0.89667, - "map_at_5": 0.902, - "mrr_at_1": 0.86, - "mrr_at_10": 0.9033888888888888, - "mrr_at_20": 0.9037222222222222, - "mrr_at_3": 0.8966666666666666, - "mrr_at_5": 0.902, - "ndcg_at_1": 0.86, - "ndcg_at_10": 0.92234, - "ndcg_at_20": 0.92386, - "ndcg_at_3": 0.90944, - "ndcg_at_5": 0.91905, - "precision_at_1": 0.86, - "precision_at_10": 0.098, - "precision_at_20": 0.04933, - "precision_at_3": 0.31556, - "precision_at_5": 0.194, - "recall_at_1": 0.86, - "recall_at_10": 0.98, - "recall_at_20": 0.98667, - "recall_at_3": 0.94667, - "recall_at_5": 0.97 - } - ] - }, - "task_name": "LEMBWikimQARetrievalChunked" -} \ No newline at end of file diff --git a/results-normal-pooling/chunk_size_8/no_model_name_available/no_revision_available/model_meta.json b/results-normal-pooling/chunk_size_8/no_model_name_available/no_revision_available/model_meta.json deleted file mode 100644 index 40ff52f..0000000 --- a/results-normal-pooling/chunk_size_8/no_model_name_available/no_revision_available/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_1024/overlap_128/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json b/results-soft-boundary/embed_size_1024/overlap_128/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json deleted file mode 100644 index a7363fe..0000000 --- a/results-soft-boundary/embed_size_1024/overlap_128/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 35.60650682449341, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.56849, - "map_at_1": 0.42829, - "map_at_10": 0.51994, - "map_at_20": 0.52575, - "map_at_3": 0.4952, - "map_at_5": 0.50849, - "mrr_at_1": 0.42829076620825146, - "mrr_at_10": 0.5199366430702383, - "mrr_at_20": 0.5257458272212815, - "mrr_at_3": 0.4951975551189696, - "mrr_at_5": 0.5084915957214582, - "ndcg_at_1": 0.42829, - "ndcg_at_10": 0.56849, - "ndcg_at_20": 0.58968, - "ndcg_at_3": 0.51703, - "ndcg_at_5": 0.54088, - "precision_at_1": 0.42829, - "precision_at_10": 0.0723, - "precision_at_20": 0.04034, - "precision_at_3": 0.19341, - "precision_at_5": 0.12757, - "recall_at_1": 0.42829, - "recall_at_10": 0.72299, - "recall_at_20": 0.80681, - "recall_at_3": 0.58022, - "recall_at_5": 0.63785 - } - ] - }, - "task_name": "LEMBQMSumRetrievalChunked" -} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_1024/overlap_128/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-soft-boundary/embed_size_1024/overlap_128/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json deleted file mode 100644 index 9984c8c..0000000 --- a/results-soft-boundary/embed_size_1024/overlap_128/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 17.512223482131958, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.88919, - "map_at_1": 0.78869, - "map_at_10": 0.86056, - "map_at_3": 0.85268, - "map_at_5": 0.85699, - "mrr_at_1": 0.7886904761904762, - "mrr_at_10": 0.860557208994709, - "mrr_at_3": 0.8526785714285714, - "mrr_at_5": 0.8569940476190476, - "ndcg_at_1": 0.78869, - "ndcg_at_10": 0.88919, - "ndcg_at_3": 0.87298, - "ndcg_at_5": 0.88054, - "precision_at_1": 0.78869, - "precision_at_10": 0.09762, - "precision_at_3": 0.31052, - "precision_at_5": 0.18988, - "recall_at_1": 0.78869, - "recall_at_10": 0.97619, - "recall_at_3": 0.93155, - "recall_at_5": 0.9494 - } - ] - }, - "task_name": "LEMBSummScreenFDRetrievalChunked" -} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_1024/overlap_128/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-soft-boundary/embed_size_1024/overlap_128/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json deleted file mode 100644 index a3bfb03..0000000 --- a/results-soft-boundary/embed_size_1024/overlap_128/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 16.536237955093384, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.94486, - "map_at_1": 0.90667, - "map_at_10": 0.93423, - "map_at_3": 0.93333, - "map_at_5": 0.93333, - "mrr_at_1": 0.9066666666666666, - "mrr_at_10": 0.9342261904761906, - "mrr_at_3": 0.9333333333333333, - "mrr_at_5": 0.9333333333333333, - "ndcg_at_1": 0.90667, - "ndcg_at_10": 0.94486, - "ndcg_at_3": 0.9427, - "ndcg_at_5": 0.9427, - "precision_at_1": 0.90667, - "precision_at_10": 0.09767, - "precision_at_3": 0.32333, - "precision_at_5": 0.194, - "recall_at_1": 0.90667, - "recall_at_10": 0.97667, - "recall_at_3": 0.97, - "recall_at_5": 0.97 - } - ] - }, - "task_name": "LEMBWikimQARetrievalChunked" -} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_1024/overlap_128/no_model_name_available/no_revision_available/model_meta.json b/results-soft-boundary/embed_size_1024/overlap_128/no_model_name_available/no_revision_available/model_meta.json deleted file mode 100644 index 40ff52f..0000000 --- a/results-soft-boundary/embed_size_1024/overlap_128/no_model_name_available/no_revision_available/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_1024/overlap_256/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json b/results-soft-boundary/embed_size_1024/overlap_256/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json deleted file mode 100644 index 0a722a2..0000000 --- a/results-soft-boundary/embed_size_1024/overlap_256/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 36.81988477706909, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.57045, - "map_at_1": 0.42764, - "map_at_10": 0.52085, - "map_at_20": 0.5265, - "map_at_3": 0.49574, - "map_at_5": 0.50986, - "mrr_at_1": 0.42763588736083824, - "mrr_at_10": 0.5208480161328884, - "mrr_at_20": 0.5264964381197267, - "mrr_at_3": 0.495743287491814, - "mrr_at_5": 0.509855926653569, - "ndcg_at_1": 0.42764, - "ndcg_at_10": 0.57045, - "ndcg_at_20": 0.591, - "ndcg_at_3": 0.51826, - "ndcg_at_5": 0.54379, - "precision_at_1": 0.42764, - "precision_at_10": 0.07282, - "precision_at_20": 0.04047, - "precision_at_3": 0.1945, - "precision_at_5": 0.12914, - "recall_at_1": 0.42764, - "recall_at_10": 0.72823, - "recall_at_20": 0.80943, - "recall_at_3": 0.5835, - "recall_at_5": 0.64571 - } - ] - }, - "task_name": "LEMBQMSumRetrievalChunked" -} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_1024/overlap_256/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-soft-boundary/embed_size_1024/overlap_256/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json deleted file mode 100644 index bccd972..0000000 --- a/results-soft-boundary/embed_size_1024/overlap_256/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 19.22271418571472, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.88196, - "map_at_1": 0.78274, - "map_at_10": 0.85208, - "map_at_3": 0.84276, - "map_at_5": 0.84826, - "mrr_at_1": 0.7827380952380952, - "mrr_at_10": 0.8520833333333334, - "mrr_at_3": 0.8427579365079365, - "mrr_at_5": 0.8482638888888888, - "ndcg_at_1": 0.78274, - "ndcg_at_10": 0.88196, - "ndcg_at_3": 0.8632, - "ndcg_at_5": 0.87306, - "precision_at_1": 0.78274, - "precision_at_10": 0.09732, - "precision_at_3": 0.30754, - "precision_at_5": 0.18929, - "recall_at_1": 0.78274, - "recall_at_10": 0.97321, - "recall_at_3": 0.92262, - "recall_at_5": 0.94643 - } - ] - }, - "task_name": "LEMBSummScreenFDRetrievalChunked" -} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_1024/overlap_256/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-soft-boundary/embed_size_1024/overlap_256/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json deleted file mode 100644 index 9de7121..0000000 --- a/results-soft-boundary/embed_size_1024/overlap_256/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 18.07439875602722, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.9528, - "map_at_1": 0.92, - "map_at_10": 0.94381, - "map_at_3": 0.94167, - "map_at_5": 0.94233, - "mrr_at_1": 0.92, - "mrr_at_10": 0.9438148148148148, - "mrr_at_3": 0.9416666666666667, - "mrr_at_5": 0.9423333333333332, - "ndcg_at_1": 0.92, - "ndcg_at_10": 0.9528, - "ndcg_at_3": 0.94813, - "ndcg_at_5": 0.94942, - "precision_at_1": 0.92, - "precision_at_10": 0.098, - "precision_at_3": 0.32222, - "precision_at_5": 0.194, - "recall_at_1": 0.92, - "recall_at_10": 0.98, - "recall_at_3": 0.96667, - "recall_at_5": 0.97 - } - ] - }, - "task_name": "LEMBWikimQARetrievalChunked" -} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_1024/overlap_256/no_model_name_available/no_revision_available/model_meta.json b/results-soft-boundary/embed_size_1024/overlap_256/no_model_name_available/no_revision_available/model_meta.json deleted file mode 100644 index 40ff52f..0000000 --- a/results-soft-boundary/embed_size_1024/overlap_256/no_model_name_available/no_revision_available/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_1024/overlap_32/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json b/results-soft-boundary/embed_size_1024/overlap_32/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json deleted file mode 100644 index 5f77527..0000000 --- a/results-soft-boundary/embed_size_1024/overlap_32/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 34.675989389419556, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.57228, - "map_at_1": 0.43484, - "map_at_10": 0.52436, - "map_at_20": 0.5299, - "map_at_3": 0.49793, - "map_at_5": 0.51325, - "mrr_at_1": 0.4348395546823838, - "mrr_at_10": 0.5243643517219155, - "mrr_at_20": 0.5298984772829408, - "mrr_at_3": 0.49792621698319145, - "mrr_at_5": 0.513250382012661, - "ndcg_at_1": 0.43484, - "ndcg_at_10": 0.57228, - "ndcg_at_20": 0.59247, - "ndcg_at_3": 0.51793, - "ndcg_at_5": 0.54549, - "precision_at_1": 0.43484, - "precision_at_10": 0.0725, - "precision_at_20": 0.04024, - "precision_at_3": 0.19188, - "precision_at_5": 0.12849, - "recall_at_1": 0.43484, - "recall_at_10": 0.72495, - "recall_at_20": 0.80485, - "recall_at_3": 0.57564, - "recall_at_5": 0.64244 - } - ] - }, - "task_name": "LEMBQMSumRetrievalChunked" -} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_1024/overlap_32/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-soft-boundary/embed_size_1024/overlap_32/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json deleted file mode 100644 index 656231d..0000000 --- a/results-soft-boundary/embed_size_1024/overlap_32/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 16.89709448814392, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.88061, - "map_at_1": 0.78274, - "map_at_10": 0.85128, - "map_at_3": 0.83879, - "map_at_5": 0.84846, - "mrr_at_1": 0.7827380952380952, - "mrr_at_10": 0.8512849584278156, - "mrr_at_3": 0.8387896825396824, - "mrr_at_5": 0.8484623015873015, - "ndcg_at_1": 0.78274, - "ndcg_at_10": 0.88061, - "ndcg_at_3": 0.85654, - "ndcg_at_5": 0.87383, - "precision_at_1": 0.78274, - "precision_at_10": 0.09702, - "precision_at_3": 0.30258, - "precision_at_5": 0.18988, - "recall_at_1": 0.78274, - "recall_at_10": 0.97024, - "recall_at_3": 0.90774, - "recall_at_5": 0.9494 - } - ] - }, - "task_name": "LEMBSummScreenFDRetrievalChunked" -} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_1024/overlap_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-soft-boundary/embed_size_1024/overlap_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json deleted file mode 100644 index 2a9e0d2..0000000 --- a/results-soft-boundary/embed_size_1024/overlap_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 15.907702445983887, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.9417, - "map_at_1": 0.90333, - "map_at_10": 0.93037, - "map_at_3": 0.92722, - "map_at_5": 0.92872, - "mrr_at_1": 0.9033333333333333, - "mrr_at_10": 0.9303650793650794, - "mrr_at_3": 0.9272222222222223, - "mrr_at_5": 0.9287222222222222, - "ndcg_at_1": 0.90333, - "ndcg_at_10": 0.9417, - "ndcg_at_3": 0.9348, - "ndcg_at_5": 0.93753, - "precision_at_1": 0.90333, - "precision_at_10": 0.09767, - "precision_at_3": 0.31889, - "precision_at_5": 0.19267, - "recall_at_1": 0.90333, - "recall_at_10": 0.97667, - "recall_at_3": 0.95667, - "recall_at_5": 0.96333 - } - ] - }, - "task_name": "LEMBWikimQARetrievalChunked" -} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_1024/overlap_32/no_model_name_available/no_revision_available/model_meta.json b/results-soft-boundary/embed_size_1024/overlap_32/no_model_name_available/no_revision_available/model_meta.json deleted file mode 100644 index 40ff52f..0000000 --- a/results-soft-boundary/embed_size_1024/overlap_32/no_model_name_available/no_revision_available/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_1024/overlap_512/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json b/results-soft-boundary/embed_size_1024/overlap_512/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json deleted file mode 100644 index 6aff1db..0000000 --- a/results-soft-boundary/embed_size_1024/overlap_512/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 42.45911240577698, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.56841, - "map_at_1": 0.43091, - "map_at_10": 0.51941, - "map_at_20": 0.5254, - "map_at_3": 0.4928, - "map_at_5": 0.5076, - "mrr_at_1": 0.4309102815979044, - "mrr_at_10": 0.5194078024136963, - "mrr_at_20": 0.5253965671643295, - "mrr_at_3": 0.49279633267845446, - "mrr_at_5": 0.5075965946299935, - "ndcg_at_1": 0.43091, - "ndcg_at_10": 0.56841, - "ndcg_at_20": 0.58993, - "ndcg_at_3": 0.51337, - "ndcg_at_5": 0.54006, - "precision_at_1": 0.43091, - "precision_at_10": 0.0725, - "precision_at_20": 0.04047, - "precision_at_3": 0.19101, - "precision_at_5": 0.12757, - "recall_at_1": 0.43091, - "recall_at_10": 0.72495, - "recall_at_20": 0.80943, - "recall_at_3": 0.57302, - "recall_at_5": 0.63785 - } - ] - }, - "task_name": "LEMBQMSumRetrievalChunked" -} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_1024/overlap_512/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-soft-boundary/embed_size_1024/overlap_512/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json deleted file mode 100644 index 32c7514..0000000 --- a/results-soft-boundary/embed_size_1024/overlap_512/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 24.504801988601685, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.8796, - "map_at_1": 0.78274, - "map_at_10": 0.85008, - "map_at_3": 0.83879, - "map_at_5": 0.84697, - "mrr_at_1": 0.7827380952380952, - "mrr_at_10": 0.8500767668178382, - "mrr_at_3": 0.8387896825396824, - "mrr_at_5": 0.8469742063492063, - "ndcg_at_1": 0.78274, - "ndcg_at_10": 0.8796, - "ndcg_at_3": 0.85725, - "ndcg_at_5": 0.87198, - "precision_at_1": 0.78274, - "precision_at_10": 0.09702, - "precision_at_3": 0.30357, - "precision_at_5": 0.18929, - "recall_at_1": 0.78274, - "recall_at_10": 0.97024, - "recall_at_3": 0.91071, - "recall_at_5": 0.94643 - } - ] - }, - "task_name": "LEMBSummScreenFDRetrievalChunked" -} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_1024/overlap_512/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-soft-boundary/embed_size_1024/overlap_512/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json deleted file mode 100644 index 77b9950..0000000 --- a/results-soft-boundary/embed_size_1024/overlap_512/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 22.70527148246765, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.95035, - "map_at_1": 0.91333, - "map_at_10": 0.94048, - "map_at_3": 0.93833, - "map_at_5": 0.93833, - "mrr_at_1": 0.9133333333333333, - "mrr_at_10": 0.9404761904761906, - "mrr_at_3": 0.9383333333333334, - "mrr_at_5": 0.9383333333333334, - "ndcg_at_1": 0.91333, - "ndcg_at_10": 0.95035, - "ndcg_at_3": 0.94567, - "ndcg_at_5": 0.94567, - "precision_at_1": 0.91333, - "precision_at_10": 0.098, - "precision_at_3": 0.32222, - "precision_at_5": 0.19333, - "recall_at_1": 0.91333, - "recall_at_10": 0.98, - "recall_at_3": 0.96667, - "recall_at_5": 0.96667 - } - ] - }, - "task_name": "LEMBWikimQARetrievalChunked" -} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_1024/overlap_512/no_model_name_available/no_revision_available/model_meta.json b/results-soft-boundary/embed_size_1024/overlap_512/no_model_name_available/no_revision_available/model_meta.json deleted file mode 100644 index 40ff52f..0000000 --- a/results-soft-boundary/embed_size_1024/overlap_512/no_model_name_available/no_revision_available/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_1024/overlap_64/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json b/results-soft-boundary/embed_size_1024/overlap_64/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json deleted file mode 100644 index ad3d20f..0000000 --- a/results-soft-boundary/embed_size_1024/overlap_64/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 34.64872407913208, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.56738, - "map_at_1": 0.42764, - "map_at_10": 0.51932, - "map_at_20": 0.52511, - "map_at_3": 0.49432, - "map_at_5": 0.5087, - "mrr_at_1": 0.42763588736083824, - "mrr_at_10": 0.5193168470182223, - "mrr_at_20": 0.5251082902831308, - "mrr_at_3": 0.49432438332241874, - "mrr_at_5": 0.508698974023139, - "ndcg_at_1": 0.42764, - "ndcg_at_10": 0.56738, - "ndcg_at_20": 0.58832, - "ndcg_at_3": 0.51605, - "ndcg_at_5": 0.54209, - "precision_at_1": 0.42764, - "precision_at_10": 0.07197, - "precision_at_20": 0.04011, - "precision_at_3": 0.19297, - "precision_at_5": 0.12849, - "recall_at_1": 0.42764, - "recall_at_10": 0.71971, - "recall_at_20": 0.80223, - "recall_at_3": 0.57891, - "recall_at_5": 0.64244 - } - ] - }, - "task_name": "LEMBQMSumRetrievalChunked" -} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_1024/overlap_64/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-soft-boundary/embed_size_1024/overlap_64/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json deleted file mode 100644 index 0955202..0000000 --- a/results-soft-boundary/embed_size_1024/overlap_64/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 17.055257558822632, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.89068, - "map_at_1": 0.8006, - "map_at_10": 0.86288, - "map_at_3": 0.85218, - "map_at_5": 0.85873, - "mrr_at_1": 0.8005952380952381, - "mrr_at_10": 0.8628838340891912, - "mrr_at_3": 0.8521825396825397, - "mrr_at_5": 0.8587301587301588, - "ndcg_at_1": 0.8006, - "ndcg_at_10": 0.89068, - "ndcg_at_3": 0.86947, - "ndcg_at_5": 0.88088, - "precision_at_1": 0.8006, - "precision_at_10": 0.09762, - "precision_at_3": 0.30655, - "precision_at_5": 0.18929, - "recall_at_1": 0.8006, - "recall_at_10": 0.97619, - "recall_at_3": 0.91964, - "recall_at_5": 0.94643 - } - ] - }, - "task_name": "LEMBSummScreenFDRetrievalChunked" -} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_1024/overlap_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-soft-boundary/embed_size_1024/overlap_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json deleted file mode 100644 index 93de53d..0000000 --- a/results-soft-boundary/embed_size_1024/overlap_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 15.988184928894043, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.94176, - "map_at_1": 0.89667, - "map_at_10": 0.92912, - "map_at_3": 0.92556, - "map_at_5": 0.92822, - "mrr_at_1": 0.8966666666666666, - "mrr_at_10": 0.9291150793650793, - "mrr_at_3": 0.9255555555555556, - "mrr_at_5": 0.9282222222222222, - "ndcg_at_1": 0.89667, - "ndcg_at_10": 0.94176, - "ndcg_at_3": 0.93444, - "ndcg_at_5": 0.9396, - "precision_at_1": 0.89667, - "precision_at_10": 0.098, - "precision_at_3": 0.32, - "precision_at_5": 0.19467, - "recall_at_1": 0.89667, - "recall_at_10": 0.98, - "recall_at_3": 0.96, - "recall_at_5": 0.97333 - } - ] - }, - "task_name": "LEMBWikimQARetrievalChunked" -} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_1024/overlap_64/no_model_name_available/no_revision_available/model_meta.json b/results-soft-boundary/embed_size_1024/overlap_64/no_model_name_available/no_revision_available/model_meta.json deleted file mode 100644 index 40ff52f..0000000 --- a/results-soft-boundary/embed_size_1024/overlap_64/no_model_name_available/no_revision_available/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_2048/overlap_128/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json b/results-soft-boundary/embed_size_2048/overlap_128/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json deleted file mode 100644 index c178da9..0000000 --- a/results-soft-boundary/embed_size_2048/overlap_128/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 38.877949237823486, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.57196, - "map_at_1": 0.4296, - "map_at_10": 0.52294, - "map_at_20": 0.5287, - "map_at_3": 0.49705, - "map_at_5": 0.51179, - "mrr_at_1": 0.42960052390307796, - "mrr_at_10": 0.5229368717580898, - "mrr_at_20": 0.5287004998766055, - "mrr_at_3": 0.49705304518664045, - "mrr_at_5": 0.5117878192534381, - "ndcg_at_1": 0.4296, - "ndcg_at_10": 0.57196, - "ndcg_at_20": 0.59319, - "ndcg_at_3": 0.51877, - "ndcg_at_5": 0.54526, - "precision_at_1": 0.4296, - "precision_at_10": 0.07276, - "precision_at_20": 0.0406, - "precision_at_3": 0.19384, - "precision_at_5": 0.12914, - "recall_at_1": 0.4296, - "recall_at_10": 0.72757, - "recall_at_20": 0.81205, - "recall_at_3": 0.58153, - "recall_at_5": 0.64571 - } - ] - }, - "task_name": "LEMBQMSumRetrievalChunked" -} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_2048/overlap_128/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-soft-boundary/embed_size_2048/overlap_128/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json deleted file mode 100644 index 70d6bc7..0000000 --- a/results-soft-boundary/embed_size_2048/overlap_128/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 21.771485567092896, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.88959, - "map_at_1": 0.79762, - "map_at_10": 0.86372, - "map_at_3": 0.85665, - "map_at_5": 0.86156, - "mrr_at_1": 0.7976190476190477, - "mrr_at_10": 0.8637188208616781, - "mrr_at_3": 0.8566468253968254, - "mrr_at_5": 0.8615575396825397, - "ndcg_at_1": 0.79762, - "ndcg_at_10": 0.88959, - "ndcg_at_3": 0.87588, - "ndcg_at_5": 0.8846, - "precision_at_1": 0.79762, - "precision_at_10": 0.09673, - "precision_at_3": 0.31052, - "precision_at_5": 0.19048, - "recall_at_1": 0.79762, - "recall_at_10": 0.96726, - "recall_at_3": 0.93155, - "recall_at_5": 0.95238 - } - ] - }, - "task_name": "LEMBSummScreenFDRetrievalChunked" -} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_2048/overlap_128/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-soft-boundary/embed_size_2048/overlap_128/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json deleted file mode 100644 index 7476616..0000000 --- a/results-soft-boundary/embed_size_2048/overlap_128/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 19.88721489906311, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.94518, - "map_at_1": 0.91, - "map_at_10": 0.93492, - "map_at_3": 0.93056, - "map_at_5": 0.93356, - "mrr_at_1": 0.91, - "mrr_at_10": 0.9349206349206349, - "mrr_at_3": 0.9305555555555556, - "mrr_at_5": 0.9335555555555556, - "ndcg_at_1": 0.91, - "ndcg_at_10": 0.94518, - "ndcg_at_3": 0.93647, - "ndcg_at_5": 0.94192, - "precision_at_1": 0.91, - "precision_at_10": 0.09767, - "precision_at_3": 0.31778, - "precision_at_5": 0.19333, - "recall_at_1": 0.91, - "recall_at_10": 0.97667, - "recall_at_3": 0.95333, - "recall_at_5": 0.96667 - } - ] - }, - "task_name": "LEMBWikimQARetrievalChunked" -} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_2048/overlap_128/no_model_name_available/no_revision_available/model_meta.json b/results-soft-boundary/embed_size_2048/overlap_128/no_model_name_available/no_revision_available/model_meta.json deleted file mode 100644 index 40ff52f..0000000 --- a/results-soft-boundary/embed_size_2048/overlap_128/no_model_name_available/no_revision_available/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_2048/overlap_256/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json b/results-soft-boundary/embed_size_2048/overlap_256/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json deleted file mode 100644 index 213743d..0000000 --- a/results-soft-boundary/embed_size_2048/overlap_256/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 39.476019859313965, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.57129, - "map_at_1": 0.42895, - "map_at_10": 0.52168, - "map_at_20": 0.52722, - "map_at_3": 0.49596, - "map_at_5": 0.51083, - "mrr_at_1": 0.4289456450556647, - "mrr_at_10": 0.5216759701042609, - "mrr_at_20": 0.5272231690823965, - "mrr_at_3": 0.4959615804409518, - "mrr_at_5": 0.510827330277232, - "ndcg_at_1": 0.42895, - "ndcg_at_10": 0.57129, - "ndcg_at_20": 0.5916, - "ndcg_at_3": 0.5184, - "ndcg_at_5": 0.54514, - "precision_at_1": 0.42895, - "precision_at_10": 0.07289, - "precision_at_20": 0.04047, - "precision_at_3": 0.1945, - "precision_at_5": 0.12967, - "recall_at_1": 0.42895, - "recall_at_10": 0.72888, - "recall_at_20": 0.80943, - "recall_at_3": 0.5835, - "recall_at_5": 0.64833 - } - ] - }, - "task_name": "LEMBQMSumRetrievalChunked" -} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_2048/overlap_256/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-soft-boundary/embed_size_2048/overlap_256/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json deleted file mode 100644 index 2a989ce..0000000 --- a/results-soft-boundary/embed_size_2048/overlap_256/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 22.649596452713013, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.8917, - "map_at_1": 0.79762, - "map_at_10": 0.86487, - "map_at_3": 0.85665, - "map_at_5": 0.86171, - "mrr_at_1": 0.7976190476190477, - "mrr_at_10": 0.8648679610733181, - "mrr_at_3": 0.8566468253968254, - "mrr_at_5": 0.8617063492063493, - "ndcg_at_1": 0.79762, - "ndcg_at_10": 0.8917, - "ndcg_at_3": 0.87518, - "ndcg_at_5": 0.88402, - "precision_at_1": 0.79762, - "precision_at_10": 0.09732, - "precision_at_3": 0.30952, - "precision_at_5": 0.18988, - "recall_at_1": 0.79762, - "recall_at_10": 0.97321, - "recall_at_3": 0.92857, - "recall_at_5": 0.9494 - } - ] - }, - "task_name": "LEMBSummScreenFDRetrievalChunked" -} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_2048/overlap_256/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-soft-boundary/embed_size_2048/overlap_256/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json deleted file mode 100644 index 286d43a..0000000 --- a/results-soft-boundary/embed_size_2048/overlap_256/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 20.78504467010498, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.94175, - "map_at_1": 0.9, - "map_at_10": 0.92928, - "map_at_3": 0.925, - "map_at_5": 0.9275, - "mrr_at_1": 0.9, - "mrr_at_10": 0.9292817460317461, - "mrr_at_3": 0.925, - "mrr_at_5": 0.9275, - "ndcg_at_1": 0.9, - "ndcg_at_10": 0.94175, - "ndcg_at_3": 0.93313, - "ndcg_at_5": 0.93744, - "precision_at_1": 0.9, - "precision_at_10": 0.098, - "precision_at_3": 0.31889, - "precision_at_5": 0.19333, - "recall_at_1": 0.9, - "recall_at_10": 0.98, - "recall_at_3": 0.95667, - "recall_at_5": 0.96667 - } - ] - }, - "task_name": "LEMBWikimQARetrievalChunked" -} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_2048/overlap_256/no_model_name_available/no_revision_available/model_meta.json b/results-soft-boundary/embed_size_2048/overlap_256/no_model_name_available/no_revision_available/model_meta.json deleted file mode 100644 index 40ff52f..0000000 --- a/results-soft-boundary/embed_size_2048/overlap_256/no_model_name_available/no_revision_available/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_2048/overlap_32/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json b/results-soft-boundary/embed_size_2048/overlap_32/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json deleted file mode 100644 index 9fee106..0000000 --- a/results-soft-boundary/embed_size_2048/overlap_32/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 38.34771919250488, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.57008, - "map_at_1": 0.42371, - "map_at_10": 0.51964, - "map_at_20": 0.52517, - "map_at_3": 0.49345, - "map_at_5": 0.50779, - "mrr_at_1": 0.42370661427635886, - "mrr_at_10": 0.5196406482261099, - "mrr_at_20": 0.5251709363866996, - "mrr_at_3": 0.49345121152586774, - "mrr_at_5": 0.5077930582842174, - "ndcg_at_1": 0.42371, - "ndcg_at_10": 0.57008, - "ndcg_at_20": 0.5908, - "ndcg_at_3": 0.51608, - "ndcg_at_5": 0.54181, - "precision_at_1": 0.42371, - "precision_at_10": 0.07302, - "precision_at_20": 0.04067, - "precision_at_3": 0.19384, - "precision_at_5": 0.12875, - "recall_at_1": 0.42371, - "recall_at_10": 0.73019, - "recall_at_20": 0.81336, - "recall_at_3": 0.58153, - "recall_at_5": 0.64375 - } - ] - }, - "task_name": "LEMBQMSumRetrievalChunked" -} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_2048/overlap_32/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-soft-boundary/embed_size_2048/overlap_32/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json deleted file mode 100644 index ce84efa..0000000 --- a/results-soft-boundary/embed_size_2048/overlap_32/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 21.172362804412842, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.89001, - "map_at_1": 0.8006, - "map_at_10": 0.86365, - "map_at_3": 0.85367, - "map_at_5": 0.86066, - "mrr_at_1": 0.8005952380952381, - "mrr_at_10": 0.8636526832955405, - "mrr_at_3": 0.8536706349206349, - "mrr_at_5": 0.8606646825396825, - "ndcg_at_1": 0.8006, - "ndcg_at_10": 0.89001, - "ndcg_at_3": 0.87064, - "ndcg_at_5": 0.88307, - "precision_at_1": 0.8006, - "precision_at_10": 0.09702, - "precision_at_3": 0.30655, - "precision_at_5": 0.18988, - "recall_at_1": 0.8006, - "recall_at_10": 0.97024, - "recall_at_3": 0.91964, - "recall_at_5": 0.9494 - } - ] - }, - "task_name": "LEMBSummScreenFDRetrievalChunked" -} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_2048/overlap_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-soft-boundary/embed_size_2048/overlap_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json deleted file mode 100644 index a4cbcb3..0000000 --- a/results-soft-boundary/embed_size_2048/overlap_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 19.420426607131958, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.94103, - "map_at_1": 0.9, - "map_at_10": 0.92838, - "map_at_3": 0.92333, - "map_at_5": 0.92717, - "mrr_at_1": 0.9, - "mrr_at_10": 0.9283835978835979, - "mrr_at_3": 0.9233333333333333, - "mrr_at_5": 0.9271666666666666, - "ndcg_at_1": 0.9, - "ndcg_at_10": 0.94103, - "ndcg_at_3": 0.93103, - "ndcg_at_5": 0.93792, - "precision_at_1": 0.9, - "precision_at_10": 0.098, - "precision_at_3": 0.31778, - "precision_at_5": 0.194, - "recall_at_1": 0.9, - "recall_at_10": 0.98, - "recall_at_3": 0.95333, - "recall_at_5": 0.97 - } - ] - }, - "task_name": "LEMBWikimQARetrievalChunked" -} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_2048/overlap_32/no_model_name_available/no_revision_available/model_meta.json b/results-soft-boundary/embed_size_2048/overlap_32/no_model_name_available/no_revision_available/model_meta.json deleted file mode 100644 index 40ff52f..0000000 --- a/results-soft-boundary/embed_size_2048/overlap_32/no_model_name_available/no_revision_available/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_2048/overlap_512/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json b/results-soft-boundary/embed_size_2048/overlap_512/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json deleted file mode 100644 index 9836433..0000000 --- a/results-soft-boundary/embed_size_2048/overlap_512/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 41.91482329368591, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.56813, - "map_at_1": 0.42633, - "map_at_10": 0.51894, - "map_at_20": 0.5249, - "map_at_3": 0.49301, - "map_at_5": 0.50831, - "mrr_at_1": 0.4263261296660118, - "mrr_at_10": 0.5189382127004917, - "mrr_at_20": 0.5249046808790312, - "mrr_at_3": 0.49301462562759224, - "mrr_at_5": 0.508306046714691, - "ndcg_at_1": 0.42633, - "ndcg_at_10": 0.56813, - "ndcg_at_20": 0.59005, - "ndcg_at_3": 0.51505, - "ndcg_at_5": 0.54245, - "precision_at_1": 0.42633, - "precision_at_10": 0.07243, - "precision_at_20": 0.04057, - "precision_at_3": 0.19297, - "precision_at_5": 0.12901, - "recall_at_1": 0.42633, - "recall_at_10": 0.7243, - "recall_at_20": 0.81139, - "recall_at_3": 0.57891, - "recall_at_5": 0.64506 - } - ] - }, - "task_name": "LEMBQMSumRetrievalChunked" -} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_2048/overlap_512/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-soft-boundary/embed_size_2048/overlap_512/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json deleted file mode 100644 index 9d5cdc0..0000000 --- a/results-soft-boundary/embed_size_2048/overlap_512/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 24.755677223205566, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.88511, - "map_at_1": 0.79167, - "map_at_10": 0.858, - "map_at_3": 0.8502, - "map_at_5": 0.85556, - "mrr_at_1": 0.7916666666666666, - "mrr_at_10": 0.8579967403628117, - "mrr_at_3": 0.8501984126984128, - "mrr_at_5": 0.8555555555555555, - "ndcg_at_1": 0.79167, - "ndcg_at_10": 0.88511, - "ndcg_at_3": 0.86954, - "ndcg_at_5": 0.87927, - "precision_at_1": 0.79167, - "precision_at_10": 0.09673, - "precision_at_3": 0.30853, - "precision_at_5": 0.18988, - "recall_at_1": 0.79167, - "recall_at_10": 0.96726, - "recall_at_3": 0.9256, - "recall_at_5": 0.9494 - } - ] - }, - "task_name": "LEMBSummScreenFDRetrievalChunked" -} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_2048/overlap_512/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-soft-boundary/embed_size_2048/overlap_512/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json deleted file mode 100644 index a100aaa..0000000 --- a/results-soft-boundary/embed_size_2048/overlap_512/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 22.70358395576477, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.9416, - "map_at_1": 0.89667, - "map_at_10": 0.92892, - "map_at_3": 0.92611, - "map_at_5": 0.92761, - "mrr_at_1": 0.8966666666666666, - "mrr_at_10": 0.9289166666666667, - "mrr_at_3": 0.926111111111111, - "mrr_at_5": 0.9276111111111112, - "ndcg_at_1": 0.89667, - "ndcg_at_10": 0.9416, - "ndcg_at_3": 0.93567, - "ndcg_at_5": 0.9384, - "precision_at_1": 0.89667, - "precision_at_10": 0.098, - "precision_at_3": 0.32111, - "precision_at_5": 0.194, - "recall_at_1": 0.89667, - "recall_at_10": 0.98, - "recall_at_3": 0.96333, - "recall_at_5": 0.97 - } - ] - }, - "task_name": "LEMBWikimQARetrievalChunked" -} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_2048/overlap_512/no_model_name_available/no_revision_available/model_meta.json b/results-soft-boundary/embed_size_2048/overlap_512/no_model_name_available/no_revision_available/model_meta.json deleted file mode 100644 index 40ff52f..0000000 --- a/results-soft-boundary/embed_size_2048/overlap_512/no_model_name_available/no_revision_available/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_2048/overlap_64/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json b/results-soft-boundary/embed_size_2048/overlap_64/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json deleted file mode 100644 index 5b277b4..0000000 --- a/results-soft-boundary/embed_size_2048/overlap_64/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 38.49483585357666, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.57013, - "map_at_1": 0.42829, - "map_at_10": 0.52047, - "map_at_20": 0.5263, - "map_at_3": 0.49422, - "map_at_5": 0.50898, - "mrr_at_1": 0.42829076620825146, - "mrr_at_10": 0.5204712009230673, - "mrr_at_20": 0.5263036291241446, - "mrr_at_3": 0.4942152368478498, - "mrr_at_5": 0.5089827548570182, - "ndcg_at_1": 0.42829, - "ndcg_at_10": 0.57013, - "ndcg_at_20": 0.59187, - "ndcg_at_3": 0.51595, - "ndcg_at_5": 0.54247, - "precision_at_1": 0.42829, - "precision_at_10": 0.07282, - "precision_at_20": 0.04077, - "precision_at_3": 0.19297, - "precision_at_5": 0.12862, - "recall_at_1": 0.42829, - "recall_at_10": 0.72823, - "recall_at_20": 0.81532, - "recall_at_3": 0.57891, - "recall_at_5": 0.64309 - } - ] - }, - "task_name": "LEMBQMSumRetrievalChunked" -} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_2048/overlap_64/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-soft-boundary/embed_size_2048/overlap_64/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json deleted file mode 100644 index f15ccfa..0000000 --- a/results-soft-boundary/embed_size_2048/overlap_64/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 21.254602670669556, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.89022, - "map_at_1": 0.8006, - "map_at_10": 0.8639, - "map_at_3": 0.85367, - "map_at_5": 0.86096, - "mrr_at_1": 0.8005952380952381, - "mrr_at_10": 0.8638971560846561, - "mrr_at_3": 0.8536706349206349, - "mrr_at_5": 0.8609623015873017, - "ndcg_at_1": 0.8006, - "ndcg_at_10": 0.89022, - "ndcg_at_3": 0.87064, - "ndcg_at_5": 0.88333, - "precision_at_1": 0.8006, - "precision_at_10": 0.09702, - "precision_at_3": 0.30655, - "precision_at_5": 0.18988, - "recall_at_1": 0.8006, - "recall_at_10": 0.97024, - "recall_at_3": 0.91964, - "recall_at_5": 0.9494 - } - ] - }, - "task_name": "LEMBSummScreenFDRetrievalChunked" -} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_2048/overlap_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-soft-boundary/embed_size_2048/overlap_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json deleted file mode 100644 index 9cdca16..0000000 --- a/results-soft-boundary/embed_size_2048/overlap_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 19.476801872253418, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.94172, - "map_at_1": 0.9, - "map_at_10": 0.92924, - "map_at_3": 0.925, - "map_at_5": 0.9275, - "mrr_at_1": 0.9, - "mrr_at_10": 0.9292394179894181, - "mrr_at_3": 0.925, - "mrr_at_5": 0.9275, - "ndcg_at_1": 0.9, - "ndcg_at_10": 0.94172, - "ndcg_at_3": 0.93313, - "ndcg_at_5": 0.93744, - "precision_at_1": 0.9, - "precision_at_10": 0.098, - "precision_at_3": 0.31889, - "precision_at_5": 0.19333, - "recall_at_1": 0.9, - "recall_at_10": 0.98, - "recall_at_3": 0.95667, - "recall_at_5": 0.96667 - } - ] - }, - "task_name": "LEMBWikimQARetrievalChunked" -} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_2048/overlap_64/no_model_name_available/no_revision_available/model_meta.json b/results-soft-boundary/embed_size_2048/overlap_64/no_model_name_available/no_revision_available/model_meta.json deleted file mode 100644 index 40ff52f..0000000 --- a/results-soft-boundary/embed_size_2048/overlap_64/no_model_name_available/no_revision_available/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_4096/overlap_128/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json b/results-soft-boundary/embed_size_4096/overlap_128/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json deleted file mode 100644 index 0fe9929..0000000 --- a/results-soft-boundary/embed_size_4096/overlap_128/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 46.40877437591553, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.57158, - "map_at_1": 0.42895, - "map_at_10": 0.52188, - "map_at_20": 0.52728, - "map_at_3": 0.49629, - "map_at_5": 0.51119, - "mrr_at_1": 0.4289456450556647, - "mrr_at_10": 0.5218815292980322, - "mrr_at_20": 0.5272814518486844, - "mrr_at_3": 0.4962890198646584, - "mrr_at_5": 0.5111875136433094, - "ndcg_at_1": 0.42895, - "ndcg_at_10": 0.57158, - "ndcg_at_20": 0.59159, - "ndcg_at_3": 0.51865, - "ndcg_at_5": 0.54571, - "precision_at_1": 0.42895, - "precision_at_10": 0.07295, - "precision_at_20": 0.04047, - "precision_at_3": 0.1945, - "precision_at_5": 0.12993, - "recall_at_1": 0.42895, - "recall_at_10": 0.72954, - "recall_at_20": 0.80943, - "recall_at_3": 0.5835, - "recall_at_5": 0.64964 - } - ] - }, - "task_name": "LEMBQMSumRetrievalChunked" -} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_4096/overlap_128/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-soft-boundary/embed_size_4096/overlap_128/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json deleted file mode 100644 index 6a74bfb..0000000 --- a/results-soft-boundary/embed_size_4096/overlap_128/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 29.33760666847229, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.89081, - "map_at_1": 0.79762, - "map_at_10": 0.86367, - "map_at_3": 0.85516, - "map_at_5": 0.86081, - "mrr_at_1": 0.7976190476190477, - "mrr_at_10": 0.8636739417989419, - "mrr_at_3": 0.8551587301587301, - "mrr_at_5": 0.8608134920634921, - "ndcg_at_1": 0.79762, - "ndcg_at_10": 0.89081, - "ndcg_at_3": 0.87401, - "ndcg_at_5": 0.884, - "precision_at_1": 0.79762, - "precision_at_10": 0.09732, - "precision_at_3": 0.30952, - "precision_at_5": 0.19048, - "recall_at_1": 0.79762, - "recall_at_10": 0.97321, - "recall_at_3": 0.92857, - "recall_at_5": 0.95238 - } - ] - }, - "task_name": "LEMBSummScreenFDRetrievalChunked" -} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_4096/overlap_128/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-soft-boundary/embed_size_4096/overlap_128/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json deleted file mode 100644 index 88e4ab0..0000000 --- a/results-soft-boundary/embed_size_4096/overlap_128/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 26.896676778793335, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.9426, - "map_at_1": 0.90333, - "map_at_10": 0.93134, - "map_at_3": 0.92833, - "map_at_5": 0.92983, - "mrr_at_1": 0.9033333333333333, - "mrr_at_10": 0.9313412698412699, - "mrr_at_3": 0.9283333333333333, - "mrr_at_5": 0.9298333333333333, - "ndcg_at_1": 0.90333, - "ndcg_at_10": 0.9426, - "ndcg_at_3": 0.93647, - "ndcg_at_5": 0.93919, - "precision_at_1": 0.90333, - "precision_at_10": 0.09767, - "precision_at_3": 0.32, - "precision_at_5": 0.19333, - "recall_at_1": 0.90333, - "recall_at_10": 0.97667, - "recall_at_3": 0.96, - "recall_at_5": 0.96667 - } - ] - }, - "task_name": "LEMBWikimQARetrievalChunked" -} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_4096/overlap_128/no_model_name_available/no_revision_available/model_meta.json b/results-soft-boundary/embed_size_4096/overlap_128/no_model_name_available/no_revision_available/model_meta.json deleted file mode 100644 index 40ff52f..0000000 --- a/results-soft-boundary/embed_size_4096/overlap_128/no_model_name_available/no_revision_available/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_4096/overlap_256/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json b/results-soft-boundary/embed_size_4096/overlap_256/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json deleted file mode 100644 index 94f5a97..0000000 --- a/results-soft-boundary/embed_size_4096/overlap_256/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 46.92707061767578, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.57072, - "map_at_1": 0.42436, - "map_at_10": 0.52029, - "map_at_20": 0.52563, - "map_at_3": 0.49454, - "map_at_5": 0.50911, - "mrr_at_1": 0.4243614931237721, - "mrr_at_10": 0.5202918888577042, - "mrr_at_20": 0.5256342949085654, - "mrr_at_3": 0.4945426762715564, - "mrr_at_5": 0.5091137306265008, - "ndcg_at_1": 0.42436, - "ndcg_at_10": 0.57072, - "ndcg_at_20": 0.59055, - "ndcg_at_3": 0.51739, - "ndcg_at_5": 0.54374, - "precision_at_1": 0.42436, - "precision_at_10": 0.07308, - "precision_at_20": 0.0405, - "precision_at_3": 0.1945, - "precision_at_5": 0.12954, - "recall_at_1": 0.42436, - "recall_at_10": 0.73084, - "recall_at_20": 0.81009, - "recall_at_3": 0.5835, - "recall_at_5": 0.64768 - } - ] - }, - "task_name": "LEMBQMSumRetrievalChunked" -} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_4096/overlap_256/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-soft-boundary/embed_size_4096/overlap_256/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json deleted file mode 100644 index 28e31ed..0000000 --- a/results-soft-boundary/embed_size_4096/overlap_256/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 29.846808195114136, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.89165, - "map_at_1": 0.79762, - "map_at_10": 0.86473, - "map_at_3": 0.85615, - "map_at_5": 0.8624, - "mrr_at_1": 0.7976190476190477, - "mrr_at_10": 0.8647321428571428, - "mrr_at_3": 0.8561507936507937, - "mrr_at_5": 0.8624007936507936, - "ndcg_at_1": 0.79762, - "ndcg_at_10": 0.89165, - "ndcg_at_3": 0.87479, - "ndcg_at_5": 0.88593, - "precision_at_1": 0.79762, - "precision_at_10": 0.09732, - "precision_at_3": 0.30952, - "precision_at_5": 0.19107, - "recall_at_1": 0.79762, - "recall_at_10": 0.97321, - "recall_at_3": 0.92857, - "recall_at_5": 0.95536 - } - ] - }, - "task_name": "LEMBSummScreenFDRetrievalChunked" -} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_4096/overlap_256/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-soft-boundary/embed_size_4096/overlap_256/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json deleted file mode 100644 index a37303b..0000000 --- a/results-soft-boundary/embed_size_4096/overlap_256/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 27.351439714431763, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.94597, - "map_at_1": 0.91, - "map_at_10": 0.93492, - "map_at_3": 0.93056, - "map_at_5": 0.93356, - "mrr_at_1": 0.91, - "mrr_at_10": 0.9349206349206349, - "mrr_at_3": 0.9305555555555556, - "mrr_at_5": 0.9335555555555556, - "ndcg_at_1": 0.91, - "ndcg_at_10": 0.94597, - "ndcg_at_3": 0.93726, - "ndcg_at_5": 0.94271, - "precision_at_1": 0.91, - "precision_at_10": 0.098, - "precision_at_3": 0.31889, - "precision_at_5": 0.194, - "recall_at_1": 0.91, - "recall_at_10": 0.98, - "recall_at_3": 0.95667, - "recall_at_5": 0.97 - } - ] - }, - "task_name": "LEMBWikimQARetrievalChunked" -} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_4096/overlap_256/no_model_name_available/no_revision_available/model_meta.json b/results-soft-boundary/embed_size_4096/overlap_256/no_model_name_available/no_revision_available/model_meta.json deleted file mode 100644 index 40ff52f..0000000 --- a/results-soft-boundary/embed_size_4096/overlap_256/no_model_name_available/no_revision_available/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_4096/overlap_32/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json b/results-soft-boundary/embed_size_4096/overlap_32/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json deleted file mode 100644 index 8f0b56e..0000000 --- a/results-soft-boundary/embed_size_4096/overlap_32/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 45.77136516571045, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.57235, - "map_at_1": 0.43091, - "map_at_10": 0.52309, - "map_at_20": 0.52872, - "map_at_3": 0.49793, - "map_at_5": 0.51197, - "mrr_at_1": 0.4309102815979044, - "mrr_at_10": 0.5230935749108637, - "mrr_at_20": 0.5287177084175539, - "mrr_at_3": 0.49792621698319145, - "mrr_at_5": 0.5119733682602052, - "ndcg_at_1": 0.43091, - "ndcg_at_10": 0.57235, - "ndcg_at_20": 0.59307, - "ndcg_at_3": 0.52021, - "ndcg_at_5": 0.54568, - "precision_at_1": 0.43091, - "precision_at_10": 0.07289, - "precision_at_20": 0.04057, - "precision_at_3": 0.19494, - "precision_at_5": 0.1294, - "recall_at_1": 0.43091, - "recall_at_10": 0.72888, - "recall_at_20": 0.81139, - "recall_at_3": 0.58481, - "recall_at_5": 0.64702 - } - ] - }, - "task_name": "LEMBQMSumRetrievalChunked" -} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_4096/overlap_32/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-soft-boundary/embed_size_4096/overlap_32/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json deleted file mode 100644 index 573b8e4..0000000 --- a/results-soft-boundary/embed_size_4096/overlap_32/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 28.918544054031372, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.8896, - "map_at_1": 0.79464, - "map_at_10": 0.86285, - "map_at_3": 0.85466, - "map_at_5": 0.86032, - "mrr_at_1": 0.7946428571428571, - "mrr_at_10": 0.8628472222222223, - "mrr_at_3": 0.8546626984126985, - "mrr_at_5": 0.8603174603174603, - "ndcg_at_1": 0.79464, - "ndcg_at_10": 0.8896, - "ndcg_at_3": 0.87369, - "ndcg_at_5": 0.88368, - "precision_at_1": 0.79464, - "precision_at_10": 0.09702, - "precision_at_3": 0.30952, - "precision_at_5": 0.19048, - "recall_at_1": 0.79464, - "recall_at_10": 0.97024, - "recall_at_3": 0.92857, - "recall_at_5": 0.95238 - } - ] - }, - "task_name": "LEMBSummScreenFDRetrievalChunked" -} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_4096/overlap_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-soft-boundary/embed_size_4096/overlap_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json deleted file mode 100644 index ccec7ca..0000000 --- a/results-soft-boundary/embed_size_4096/overlap_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 26.577144145965576, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.94472, - "map_at_1": 0.90333, - "map_at_10": 0.9322, - "map_at_3": 0.92833, - "map_at_5": 0.9305, - "mrr_at_1": 0.9033333333333333, - "mrr_at_10": 0.932202380952381, - "mrr_at_3": 0.9283333333333333, - "mrr_at_5": 0.9304999999999999, - "ndcg_at_1": 0.90333, - "ndcg_at_10": 0.94472, - "ndcg_at_3": 0.93647, - "ndcg_at_5": 0.94048, - "precision_at_1": 0.90333, - "precision_at_10": 0.09833, - "precision_at_3": 0.32, - "precision_at_5": 0.194, - "recall_at_1": 0.90333, - "recall_at_10": 0.98333, - "recall_at_3": 0.96, - "recall_at_5": 0.97 - } - ] - }, - "task_name": "LEMBWikimQARetrievalChunked" -} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_4096/overlap_32/no_model_name_available/no_revision_available/model_meta.json b/results-soft-boundary/embed_size_4096/overlap_32/no_model_name_available/no_revision_available/model_meta.json deleted file mode 100644 index 40ff52f..0000000 --- a/results-soft-boundary/embed_size_4096/overlap_32/no_model_name_available/no_revision_available/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_4096/overlap_512/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json b/results-soft-boundary/embed_size_4096/overlap_512/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json deleted file mode 100644 index 7c51255..0000000 --- a/results-soft-boundary/embed_size_4096/overlap_512/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 48.13499999046326, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.5719, - "map_at_1": 0.42633, - "map_at_10": 0.52184, - "map_at_20": 0.52697, - "map_at_3": 0.49673, - "map_at_5": 0.51051, - "mrr_at_1": 0.4263261296660118, - "mrr_at_10": 0.5218412490514651, - "mrr_at_20": 0.5269714581870543, - "mrr_at_3": 0.49672560576293384, - "mrr_at_5": 0.5105108055009823, - "ndcg_at_1": 0.42633, - "ndcg_at_10": 0.5719, - "ndcg_at_20": 0.59093, - "ndcg_at_3": 0.51968, - "ndcg_at_5": 0.54465, - "precision_at_1": 0.42633, - "precision_at_10": 0.07308, - "precision_at_20": 0.04034, - "precision_at_3": 0.19537, - "precision_at_5": 0.1294, - "recall_at_1": 0.42633, - "recall_at_10": 0.73084, - "recall_at_20": 0.80681, - "recall_at_3": 0.58612, - "recall_at_5": 0.64702 - } - ] - }, - "task_name": "LEMBQMSumRetrievalChunked" -} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_4096/overlap_512/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-soft-boundary/embed_size_4096/overlap_512/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json deleted file mode 100644 index 6c915d7..0000000 --- a/results-soft-boundary/embed_size_4096/overlap_512/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 31.02289605140686, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.89229, - "map_at_1": 0.8006, - "map_at_10": 0.8657, - "map_at_3": 0.85665, - "map_at_5": 0.86245, - "mrr_at_1": 0.8005952380952381, - "mrr_at_10": 0.8657029478458049, - "mrr_at_3": 0.8566468253968254, - "mrr_at_5": 0.8624503968253969, - "ndcg_at_1": 0.8006, - "ndcg_at_10": 0.89229, - "ndcg_at_3": 0.8744, - "ndcg_at_5": 0.88452, - "precision_at_1": 0.8006, - "precision_at_10": 0.09732, - "precision_at_3": 0.30853, - "precision_at_5": 0.18988, - "recall_at_1": 0.8006, - "recall_at_10": 0.97321, - "recall_at_3": 0.9256, - "recall_at_5": 0.9494 - } - ] - }, - "task_name": "LEMBSummScreenFDRetrievalChunked" -} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_4096/overlap_512/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-soft-boundary/embed_size_4096/overlap_512/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json deleted file mode 100644 index c7d214f..0000000 --- a/results-soft-boundary/embed_size_4096/overlap_512/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 28.48093295097351, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.94569, - "map_at_1": 0.90667, - "map_at_10": 0.93356, - "map_at_3": 0.92889, - "map_at_5": 0.93122, - "mrr_at_1": 0.9066666666666666, - "mrr_at_10": 0.9335595238095238, - "mrr_at_3": 0.928888888888889, - "mrr_at_5": 0.9312222222222223, - "ndcg_at_1": 0.90667, - "ndcg_at_10": 0.94569, - "ndcg_at_3": 0.93603, - "ndcg_at_5": 0.94019, - "precision_at_1": 0.90667, - "precision_at_10": 0.09833, - "precision_at_3": 0.31889, - "precision_at_5": 0.19333, - "recall_at_1": 0.90667, - "recall_at_10": 0.98333, - "recall_at_3": 0.95667, - "recall_at_5": 0.96667 - } - ] - }, - "task_name": "LEMBWikimQARetrievalChunked" -} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_4096/overlap_512/no_model_name_available/no_revision_available/model_meta.json b/results-soft-boundary/embed_size_4096/overlap_512/no_model_name_available/no_revision_available/model_meta.json deleted file mode 100644 index 40ff52f..0000000 --- a/results-soft-boundary/embed_size_4096/overlap_512/no_model_name_available/no_revision_available/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_4096/overlap_64/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json b/results-soft-boundary/embed_size_4096/overlap_64/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json deleted file mode 100644 index ddca10d..0000000 --- a/results-soft-boundary/embed_size_4096/overlap_64/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 46.11063885688782, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.57256, - "map_at_1": 0.4296, - "map_at_10": 0.52266, - "map_at_20": 0.52806, - "map_at_3": 0.49705, - "map_at_5": 0.511, - "mrr_at_1": 0.42960052390307796, - "mrr_at_10": 0.5226562093949128, - "mrr_at_20": 0.5280638645171482, - "mrr_at_3": 0.49705304518664045, - "mrr_at_5": 0.5110019646365422, - "ndcg_at_1": 0.4296, - "ndcg_at_10": 0.57256, - "ndcg_at_20": 0.59269, - "ndcg_at_3": 0.51924, - "ndcg_at_5": 0.54449, - "precision_at_1": 0.4296, - "precision_at_10": 0.07315, - "precision_at_20": 0.0406, - "precision_at_3": 0.1945, - "precision_at_5": 0.12901, - "recall_at_1": 0.4296, - "recall_at_10": 0.7315, - "recall_at_20": 0.81205, - "recall_at_3": 0.5835, - "recall_at_5": 0.64506 - } - ] - }, - "task_name": "LEMBQMSumRetrievalChunked" -} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_4096/overlap_64/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-soft-boundary/embed_size_4096/overlap_64/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json deleted file mode 100644 index b379c91..0000000 --- a/results-soft-boundary/embed_size_4096/overlap_64/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 28.984849214553833, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.89135, - "map_at_1": 0.79762, - "map_at_10": 0.86438, - "map_at_3": 0.85516, - "map_at_5": 0.86156, - "mrr_at_1": 0.7976190476190477, - "mrr_at_10": 0.8643849206349207, - "mrr_at_3": 0.8551587301587301, - "mrr_at_5": 0.8615575396825397, - "ndcg_at_1": 0.79762, - "ndcg_at_10": 0.89135, - "ndcg_at_3": 0.8733, - "ndcg_at_5": 0.88457, - "precision_at_1": 0.79762, - "precision_at_10": 0.09732, - "precision_at_3": 0.30853, - "precision_at_5": 0.19048, - "recall_at_1": 0.79762, - "recall_at_10": 0.97321, - "recall_at_3": 0.9256, - "recall_at_5": 0.95238 - } - ] - }, - "task_name": "LEMBSummScreenFDRetrievalChunked" -} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_4096/overlap_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-soft-boundary/embed_size_4096/overlap_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json deleted file mode 100644 index 8ff2087..0000000 --- a/results-soft-boundary/embed_size_4096/overlap_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 26.565731525421143, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.94488, - "map_at_1": 0.90333, - "map_at_10": 0.93239, - "map_at_3": 0.92833, - "map_at_5": 0.93067, - "mrr_at_1": 0.9033333333333333, - "mrr_at_10": 0.9323888888888888, - "mrr_at_3": 0.9283333333333333, - "mrr_at_5": 0.9306666666666666, - "ndcg_at_1": 0.90333, - "ndcg_at_10": 0.94488, - "ndcg_at_3": 0.93647, - "ndcg_at_5": 0.94063, - "precision_at_1": 0.90333, - "precision_at_10": 0.09833, - "precision_at_3": 0.32, - "precision_at_5": 0.194, - "recall_at_1": 0.90333, - "recall_at_10": 0.98333, - "recall_at_3": 0.96, - "recall_at_5": 0.97 - } - ] - }, - "task_name": "LEMBWikimQARetrievalChunked" -} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_4096/overlap_64/no_model_name_available/no_revision_available/model_meta.json b/results-soft-boundary/embed_size_4096/overlap_64/no_model_name_available/no_revision_available/model_meta.json deleted file mode 100644 index 40ff52f..0000000 --- a/results-soft-boundary/embed_size_4096/overlap_64/no_model_name_available/no_revision_available/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_512/overlap_128/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json b/results-soft-boundary/embed_size_512/overlap_128/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json deleted file mode 100644 index 0752ec3..0000000 --- a/results-soft-boundary/embed_size_512/overlap_128/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 33.797311544418335, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.56433, - "map_at_1": 0.42371, - "map_at_10": 0.51527, - "map_at_20": 0.52098, - "map_at_3": 0.48985, - "map_at_5": 0.50403, - "mrr_at_1": 0.42370661427635886, - "mrr_at_10": 0.5152693319196265, - "mrr_at_20": 0.5209754242570668, - "mrr_at_3": 0.48984937786509497, - "mrr_at_5": 0.5040275049115913, - "ndcg_at_1": 0.42371, - "ndcg_at_10": 0.56433, - "ndcg_at_20": 0.58561, - "ndcg_at_3": 0.51107, - "ndcg_at_5": 0.5368, - "precision_at_1": 0.42371, - "precision_at_10": 0.0721, - "precision_at_20": 0.04031, - "precision_at_3": 0.19079, - "precision_at_5": 0.12705, - "recall_at_1": 0.42371, - "recall_at_10": 0.72102, - "recall_at_20": 0.80616, - "recall_at_3": 0.57236, - "recall_at_5": 0.63523 - } - ] - }, - "task_name": "LEMBQMSumRetrievalChunked" -} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_512/overlap_128/no_model_name_available/no_revision_available/model_meta.json b/results-soft-boundary/embed_size_512/overlap_128/no_model_name_available/no_revision_available/model_meta.json deleted file mode 100644 index 40ff52f..0000000 --- a/results-soft-boundary/embed_size_512/overlap_128/no_model_name_available/no_revision_available/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_512/overlap_256/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json b/results-soft-boundary/embed_size_512/overlap_256/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json deleted file mode 100644 index 78ce41e..0000000 --- a/results-soft-boundary/embed_size_512/overlap_256/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 38.087034463882446, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.56557, - "map_at_1": 0.42698, - "map_at_10": 0.51611, - "map_at_20": 0.52222, - "map_at_3": 0.48941, - "map_at_5": 0.50372, - "mrr_at_1": 0.426981008513425, - "mrr_at_10": 0.516108720283573, - "mrr_at_20": 0.5222164663592797, - "mrr_at_3": 0.48941279196681947, - "mrr_at_5": 0.5037218947827985, - "ndcg_at_1": 0.42698, - "ndcg_at_10": 0.56557, - "ndcg_at_20": 0.58811, - "ndcg_at_3": 0.51018, - "ndcg_at_5": 0.53602, - "precision_at_1": 0.42698, - "precision_at_10": 0.07236, - "precision_at_20": 0.04067, - "precision_at_3": 0.19013, - "precision_at_5": 0.12665, - "recall_at_1": 0.42698, - "recall_at_10": 0.72364, - "recall_at_20": 0.81336, - "recall_at_3": 0.5704, - "recall_at_5": 0.63327 - } - ] - }, - "task_name": "LEMBQMSumRetrievalChunked" -} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_512/overlap_256/no_model_name_available/no_revision_available/model_meta.json b/results-soft-boundary/embed_size_512/overlap_256/no_model_name_available/no_revision_available/model_meta.json deleted file mode 100644 index 40ff52f..0000000 --- a/results-soft-boundary/embed_size_512/overlap_256/no_model_name_available/no_revision_available/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_512/overlap_32/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json b/results-soft-boundary/embed_size_512/overlap_32/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json deleted file mode 100644 index 89fb91f..0000000 --- a/results-soft-boundary/embed_size_512/overlap_32/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 32.38495421409607, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.56145, - "map_at_1": 0.41912, - "map_at_10": 0.51173, - "map_at_20": 0.51745, - "map_at_3": 0.4845, - "map_at_5": 0.50084, - "mrr_at_1": 0.41912246234446626, - "mrr_at_10": 0.5117296078003347, - "mrr_at_20": 0.5174488508122417, - "mrr_at_3": 0.48450120061122026, - "mrr_at_5": 0.5008404278541803, - "ndcg_at_1": 0.41912, - "ndcg_at_10": 0.56145, - "ndcg_at_20": 0.58241, - "ndcg_at_3": 0.50573, - "ndcg_at_5": 0.53501, - "precision_at_1": 0.41912, - "precision_at_10": 0.07197, - "precision_at_20": 0.04014, - "precision_at_3": 0.18904, - "precision_at_5": 0.12757, - "recall_at_1": 0.41912, - "recall_at_10": 0.71971, - "recall_at_20": 0.80288, - "recall_at_3": 0.56713, - "recall_at_5": 0.63785 - } - ] - }, - "task_name": "LEMBQMSumRetrievalChunked" -} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_512/overlap_32/no_model_name_available/no_revision_available/model_meta.json b/results-soft-boundary/embed_size_512/overlap_32/no_model_name_available/no_revision_available/model_meta.json deleted file mode 100644 index 40ff52f..0000000 --- a/results-soft-boundary/embed_size_512/overlap_32/no_model_name_available/no_revision_available/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_512/overlap_512/no_model_name_available/no_revision_available/model_meta.json b/results-soft-boundary/embed_size_512/overlap_512/no_model_name_available/no_revision_available/model_meta.json deleted file mode 100644 index 40ff52f..0000000 --- a/results-soft-boundary/embed_size_512/overlap_512/no_model_name_available/no_revision_available/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_512/overlap_64/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json b/results-soft-boundary/embed_size_512/overlap_64/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json deleted file mode 100644 index 2aed848..0000000 --- a/results-soft-boundary/embed_size_512/overlap_64/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 32.886000633239746, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.56817, - "map_at_1": 0.43353, - "map_at_10": 0.52101, - "map_at_20": 0.52707, - "map_at_3": 0.49509, - "map_at_5": 0.51113, - "mrr_at_1": 0.4335297969875573, - "mrr_at_10": 0.5210093969917152, - "mrr_at_20": 0.5270617882207564, - "mrr_at_3": 0.4950884086444008, - "mrr_at_5": 0.5111329404060249, - "ndcg_at_1": 0.43353, - "ndcg_at_10": 0.56817, - "ndcg_at_20": 0.59042, - "ndcg_at_3": 0.51511, - "ndcg_at_5": 0.54399, - "precision_at_1": 0.43353, - "precision_at_10": 0.07184, - "precision_at_20": 0.04034, - "precision_at_3": 0.19101, - "precision_at_5": 0.12862, - "recall_at_1": 0.43353, - "recall_at_10": 0.7184, - "recall_at_20": 0.80681, - "recall_at_3": 0.57302, - "recall_at_5": 0.64309 - } - ] - }, - "task_name": "LEMBQMSumRetrievalChunked" -} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_512/overlap_64/no_model_name_available/no_revision_available/model_meta.json b/results-soft-boundary/embed_size_512/overlap_64/no_model_name_available/no_revision_available/model_meta.json deleted file mode 100644 index 40ff52f..0000000 --- a/results-soft-boundary/embed_size_512/overlap_64/no_model_name_available/no_revision_available/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_8192/overlap_128/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json b/results-soft-boundary/embed_size_8192/overlap_128/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json deleted file mode 100644 index 510ed14..0000000 --- a/results-soft-boundary/embed_size_8192/overlap_128/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 59.90270757675171, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.57159, - "map_at_1": 0.42895, - "map_at_10": 0.5221, - "map_at_20": 0.52746, - "map_at_3": 0.49629, - "map_at_5": 0.51142, - "mrr_at_1": 0.4289456450556647, - "mrr_at_10": 0.5220993025020529, - "mrr_at_20": 0.5274577033575282, - "mrr_at_3": 0.4962890198646584, - "mrr_at_5": 0.5114167212399039, - "ndcg_at_1": 0.42895, - "ndcg_at_10": 0.57159, - "ndcg_at_20": 0.59144, - "ndcg_at_3": 0.51834, - "ndcg_at_5": 0.54573, - "precision_at_1": 0.42895, - "precision_at_10": 0.07289, - "precision_at_20": 0.04041, - "precision_at_3": 0.19406, - "precision_at_5": 0.1298, - "recall_at_1": 0.42895, - "recall_at_10": 0.72888, - "recall_at_20": 0.80812, - "recall_at_3": 0.58219, - "recall_at_5": 0.64898 - } - ] - }, - "task_name": "LEMBQMSumRetrievalChunked" -} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_8192/overlap_128/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-soft-boundary/embed_size_8192/overlap_128/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json deleted file mode 100644 index e07c115..0000000 --- a/results-soft-boundary/embed_size_8192/overlap_128/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 41.60118389129639, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.8896, - "map_at_1": 0.79464, - "map_at_10": 0.86285, - "map_at_3": 0.85466, - "map_at_5": 0.86032, - "mrr_at_1": 0.7946428571428571, - "mrr_at_10": 0.8628472222222223, - "mrr_at_3": 0.8546626984126985, - "mrr_at_5": 0.8603174603174603, - "ndcg_at_1": 0.79464, - "ndcg_at_10": 0.8896, - "ndcg_at_3": 0.87369, - "ndcg_at_5": 0.88368, - "precision_at_1": 0.79464, - "precision_at_10": 0.09702, - "precision_at_3": 0.30952, - "precision_at_5": 0.19048, - "recall_at_1": 0.79464, - "recall_at_10": 0.97024, - "recall_at_3": 0.92857, - "recall_at_5": 0.95238 - } - ] - }, - "task_name": "LEMBSummScreenFDRetrievalChunked" -} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_8192/overlap_128/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-soft-boundary/embed_size_8192/overlap_128/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json deleted file mode 100644 index 62f0c08..0000000 --- a/results-soft-boundary/embed_size_8192/overlap_128/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 37.52750086784363, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.94608, - "map_at_1": 0.90667, - "map_at_10": 0.93403, - "map_at_3": 0.93, - "map_at_5": 0.93233, - "mrr_at_1": 0.9066666666666666, - "mrr_at_10": 0.9340317460317461, - "mrr_at_3": 0.93, - "mrr_at_5": 0.9323333333333333, - "ndcg_at_1": 0.90667, - "ndcg_at_10": 0.94608, - "ndcg_at_3": 0.9377, - "ndcg_at_5": 0.94186, - "precision_at_1": 0.90667, - "precision_at_10": 0.09833, - "precision_at_3": 0.32, - "precision_at_5": 0.194, - "recall_at_1": 0.90667, - "recall_at_10": 0.98333, - "recall_at_3": 0.96, - "recall_at_5": 0.97 - } - ] - }, - "task_name": "LEMBWikimQARetrievalChunked" -} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_8192/overlap_128/no_model_name_available/no_revision_available/model_meta.json b/results-soft-boundary/embed_size_8192/overlap_128/no_model_name_available/no_revision_available/model_meta.json deleted file mode 100644 index 40ff52f..0000000 --- a/results-soft-boundary/embed_size_8192/overlap_128/no_model_name_available/no_revision_available/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_8192/overlap_256/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json b/results-soft-boundary/embed_size_8192/overlap_256/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json deleted file mode 100644 index 70910a4..0000000 --- a/results-soft-boundary/embed_size_8192/overlap_256/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 60.42517876625061, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.5714, - "map_at_1": 0.42829, - "map_at_10": 0.52166, - "map_at_20": 0.52697, - "map_at_3": 0.49552, - "map_at_5": 0.51095, - "mrr_at_1": 0.42829076620825146, - "mrr_at_10": 0.521663496221453, - "mrr_at_20": 0.5269704423103457, - "mrr_at_3": 0.49552499454267623, - "mrr_at_5": 0.5109473913992578, - "ndcg_at_1": 0.42829, - "ndcg_at_10": 0.5714, - "ndcg_at_20": 0.59108, - "ndcg_at_3": 0.5176, - "ndcg_at_5": 0.54539, - "precision_at_1": 0.42829, - "precision_at_10": 0.07295, - "precision_at_20": 0.04041, - "precision_at_3": 0.19384, - "precision_at_5": 0.1298, - "recall_at_1": 0.42829, - "recall_at_10": 0.72954, - "recall_at_20": 0.80812, - "recall_at_3": 0.58153, - "recall_at_5": 0.64898 - } - ] - }, - "task_name": "LEMBQMSumRetrievalChunked" -} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_8192/overlap_256/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-soft-boundary/embed_size_8192/overlap_256/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json deleted file mode 100644 index 55d3863..0000000 --- a/results-soft-boundary/embed_size_8192/overlap_256/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 42.03281593322754, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.8896, - "map_at_1": 0.79464, - "map_at_10": 0.86285, - "map_at_3": 0.85466, - "map_at_5": 0.86032, - "mrr_at_1": 0.7946428571428571, - "mrr_at_10": 0.8628472222222223, - "mrr_at_3": 0.8546626984126985, - "mrr_at_5": 0.8603174603174603, - "ndcg_at_1": 0.79464, - "ndcg_at_10": 0.8896, - "ndcg_at_3": 0.87369, - "ndcg_at_5": 0.88368, - "precision_at_1": 0.79464, - "precision_at_10": 0.09702, - "precision_at_3": 0.30952, - "precision_at_5": 0.19048, - "recall_at_1": 0.79464, - "recall_at_10": 0.97024, - "recall_at_3": 0.92857, - "recall_at_5": 0.95238 - } - ] - }, - "task_name": "LEMBSummScreenFDRetrievalChunked" -} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_8192/overlap_256/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-soft-boundary/embed_size_8192/overlap_256/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json deleted file mode 100644 index 3ac7a2e..0000000 --- a/results-soft-boundary/embed_size_8192/overlap_256/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 37.988229513168335, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.94608, - "map_at_1": 0.90667, - "map_at_10": 0.93403, - "map_at_3": 0.93, - "map_at_5": 0.93233, - "mrr_at_1": 0.9066666666666666, - "mrr_at_10": 0.9340317460317461, - "mrr_at_3": 0.93, - "mrr_at_5": 0.9323333333333333, - "ndcg_at_1": 0.90667, - "ndcg_at_10": 0.94608, - "ndcg_at_3": 0.9377, - "ndcg_at_5": 0.94186, - "precision_at_1": 0.90667, - "precision_at_10": 0.09833, - "precision_at_3": 0.32, - "precision_at_5": 0.194, - "recall_at_1": 0.90667, - "recall_at_10": 0.98333, - "recall_at_3": 0.96, - "recall_at_5": 0.97 - } - ] - }, - "task_name": "LEMBWikimQARetrievalChunked" -} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_8192/overlap_256/no_model_name_available/no_revision_available/model_meta.json b/results-soft-boundary/embed_size_8192/overlap_256/no_model_name_available/no_revision_available/model_meta.json deleted file mode 100644 index 40ff52f..0000000 --- a/results-soft-boundary/embed_size_8192/overlap_256/no_model_name_available/no_revision_available/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_8192/overlap_32/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json b/results-soft-boundary/embed_size_8192/overlap_32/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json deleted file mode 100644 index fb2986d..0000000 --- a/results-soft-boundary/embed_size_8192/overlap_32/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 59.76712894439697, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.57078, - "map_at_1": 0.42829, - "map_at_10": 0.52124, - "map_at_20": 0.52661, - "map_at_3": 0.4952, - "map_at_5": 0.51055, - "mrr_at_1": 0.42829076620825146, - "mrr_at_10": 0.5212370453529589, - "mrr_at_20": 0.5266118548320644, - "mrr_at_3": 0.4951975551189696, - "mrr_at_5": 0.5105544640908098, - "ndcg_at_1": 0.42829, - "ndcg_at_10": 0.57078, - "ndcg_at_20": 0.59065, - "ndcg_at_3": 0.51719, - "ndcg_at_5": 0.54492, - "precision_at_1": 0.42829, - "precision_at_10": 0.07282, - "precision_at_20": 0.04037, - "precision_at_3": 0.19363, - "precision_at_5": 0.12967, - "recall_at_1": 0.42829, - "recall_at_10": 0.72823, - "recall_at_20": 0.80747, - "recall_at_3": 0.58088, - "recall_at_5": 0.64833 - } - ] - }, - "task_name": "LEMBQMSumRetrievalChunked" -} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_8192/overlap_32/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-soft-boundary/embed_size_8192/overlap_32/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json deleted file mode 100644 index e28ae82..0000000 --- a/results-soft-boundary/embed_size_8192/overlap_32/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 41.46819353103638, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.8896, - "map_at_1": 0.79464, - "map_at_10": 0.86285, - "map_at_3": 0.85466, - "map_at_5": 0.86032, - "mrr_at_1": 0.7946428571428571, - "mrr_at_10": 0.8628472222222223, - "mrr_at_3": 0.8546626984126985, - "mrr_at_5": 0.8603174603174603, - "ndcg_at_1": 0.79464, - "ndcg_at_10": 0.8896, - "ndcg_at_3": 0.87369, - "ndcg_at_5": 0.88368, - "precision_at_1": 0.79464, - "precision_at_10": 0.09702, - "precision_at_3": 0.30952, - "precision_at_5": 0.19048, - "recall_at_1": 0.79464, - "recall_at_10": 0.97024, - "recall_at_3": 0.92857, - "recall_at_5": 0.95238 - } - ] - }, - "task_name": "LEMBSummScreenFDRetrievalChunked" -} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_8192/overlap_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-soft-boundary/embed_size_8192/overlap_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json deleted file mode 100644 index 760c521..0000000 --- a/results-soft-boundary/embed_size_8192/overlap_32/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 37.77802062034607, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.94608, - "map_at_1": 0.90667, - "map_at_10": 0.93403, - "map_at_3": 0.93, - "map_at_5": 0.93233, - "mrr_at_1": 0.9066666666666666, - "mrr_at_10": 0.9340317460317461, - "mrr_at_3": 0.93, - "mrr_at_5": 0.9323333333333333, - "ndcg_at_1": 0.90667, - "ndcg_at_10": 0.94608, - "ndcg_at_3": 0.9377, - "ndcg_at_5": 0.94186, - "precision_at_1": 0.90667, - "precision_at_10": 0.09833, - "precision_at_3": 0.32, - "precision_at_5": 0.194, - "recall_at_1": 0.90667, - "recall_at_10": 0.98333, - "recall_at_3": 0.96, - "recall_at_5": 0.97 - } - ] - }, - "task_name": "LEMBWikimQARetrievalChunked" -} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_8192/overlap_32/no_model_name_available/no_revision_available/model_meta.json b/results-soft-boundary/embed_size_8192/overlap_32/no_model_name_available/no_revision_available/model_meta.json deleted file mode 100644 index 40ff52f..0000000 --- a/results-soft-boundary/embed_size_8192/overlap_32/no_model_name_available/no_revision_available/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_8192/overlap_512/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json b/results-soft-boundary/embed_size_8192/overlap_512/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json deleted file mode 100644 index ac77fe1..0000000 --- a/results-soft-boundary/embed_size_8192/overlap_512/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 61.178690671920776, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.57081, - "map_at_1": 0.42764, - "map_at_10": 0.52091, - "map_at_20": 0.52621, - "map_at_3": 0.49476, - "map_at_5": 0.51012, - "mrr_at_1": 0.42763588736083824, - "mrr_at_10": 0.5209119447822788, - "mrr_at_20": 0.5262128450125312, - "mrr_at_3": 0.4947609692206942, - "mrr_at_5": 0.5101178781925344, - "ndcg_at_1": 0.42764, - "ndcg_at_10": 0.57081, - "ndcg_at_20": 0.59048, - "ndcg_at_3": 0.51686, - "ndcg_at_5": 0.54459, - "precision_at_1": 0.42764, - "precision_at_10": 0.07295, - "precision_at_20": 0.04041, - "precision_at_3": 0.19363, - "precision_at_5": 0.12967, - "recall_at_1": 0.42764, - "recall_at_10": 0.72954, - "recall_at_20": 0.80812, - "recall_at_3": 0.58088, - "recall_at_5": 0.64833 - } - ] - }, - "task_name": "LEMBQMSumRetrievalChunked" -} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_8192/overlap_512/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-soft-boundary/embed_size_8192/overlap_512/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json deleted file mode 100644 index 8496845..0000000 --- a/results-soft-boundary/embed_size_8192/overlap_512/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 42.5116925239563, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.8896, - "map_at_1": 0.79464, - "map_at_10": 0.86285, - "map_at_3": 0.85466, - "map_at_5": 0.86032, - "mrr_at_1": 0.7946428571428571, - "mrr_at_10": 0.8628472222222223, - "mrr_at_3": 0.8546626984126985, - "mrr_at_5": 0.8603174603174603, - "ndcg_at_1": 0.79464, - "ndcg_at_10": 0.8896, - "ndcg_at_3": 0.87369, - "ndcg_at_5": 0.88368, - "precision_at_1": 0.79464, - "precision_at_10": 0.09702, - "precision_at_3": 0.30952, - "precision_at_5": 0.19048, - "recall_at_1": 0.79464, - "recall_at_10": 0.97024, - "recall_at_3": 0.92857, - "recall_at_5": 0.95238 - } - ] - }, - "task_name": "LEMBSummScreenFDRetrievalChunked" -} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_8192/overlap_512/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-soft-boundary/embed_size_8192/overlap_512/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json deleted file mode 100644 index 3c8224f..0000000 --- a/results-soft-boundary/embed_size_8192/overlap_512/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 38.475874185562134, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.94608, - "map_at_1": 0.90667, - "map_at_10": 0.93403, - "map_at_3": 0.93, - "map_at_5": 0.93233, - "mrr_at_1": 0.9066666666666666, - "mrr_at_10": 0.9340317460317461, - "mrr_at_3": 0.93, - "mrr_at_5": 0.9323333333333333, - "ndcg_at_1": 0.90667, - "ndcg_at_10": 0.94608, - "ndcg_at_3": 0.9377, - "ndcg_at_5": 0.94186, - "precision_at_1": 0.90667, - "precision_at_10": 0.09833, - "precision_at_3": 0.32, - "precision_at_5": 0.194, - "recall_at_1": 0.90667, - "recall_at_10": 0.98333, - "recall_at_3": 0.96, - "recall_at_5": 0.97 - } - ] - }, - "task_name": "LEMBWikimQARetrievalChunked" -} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_8192/overlap_512/no_model_name_available/no_revision_available/model_meta.json b/results-soft-boundary/embed_size_8192/overlap_512/no_model_name_available/no_revision_available/model_meta.json deleted file mode 100644 index 40ff52f..0000000 --- a/results-soft-boundary/embed_size_8192/overlap_512/no_model_name_available/no_revision_available/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_8192/overlap_64/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json b/results-soft-boundary/embed_size_8192/overlap_64/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json deleted file mode 100644 index c0e2dd5..0000000 --- a/results-soft-boundary/embed_size_8192/overlap_64/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 59.825618267059326, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.5716, - "map_at_1": 0.43026, - "map_at_10": 0.52234, - "map_at_20": 0.52777, - "map_at_3": 0.49651, - "map_at_5": 0.51164, - "mrr_at_1": 0.4302554027504912, - "mrr_at_10": 0.5223448820698329, - "mrr_at_20": 0.5277702261925865, - "mrr_at_3": 0.49650731281379606, - "mrr_at_5": 0.5116350141890417, - "ndcg_at_1": 0.43026, - "ndcg_at_10": 0.5716, - "ndcg_at_20": 0.59165, - "ndcg_at_3": 0.51833, - "ndcg_at_5": 0.54572, - "precision_at_1": 0.43026, - "precision_at_10": 0.07282, - "precision_at_20": 0.04041, - "precision_at_3": 0.19384, - "precision_at_5": 0.12967, - "recall_at_1": 0.43026, - "recall_at_10": 0.72823, - "recall_at_20": 0.80812, - "recall_at_3": 0.58153, - "recall_at_5": 0.64833 - } - ] - }, - "task_name": "LEMBQMSumRetrievalChunked" -} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_8192/overlap_64/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-soft-boundary/embed_size_8192/overlap_64/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json deleted file mode 100644 index d244a55..0000000 --- a/results-soft-boundary/embed_size_8192/overlap_64/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 41.537845849990845, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.8896, - "map_at_1": 0.79464, - "map_at_10": 0.86285, - "map_at_3": 0.85466, - "map_at_5": 0.86032, - "mrr_at_1": 0.7946428571428571, - "mrr_at_10": 0.8628472222222223, - "mrr_at_3": 0.8546626984126985, - "mrr_at_5": 0.8603174603174603, - "ndcg_at_1": 0.79464, - "ndcg_at_10": 0.8896, - "ndcg_at_3": 0.87369, - "ndcg_at_5": 0.88368, - "precision_at_1": 0.79464, - "precision_at_10": 0.09702, - "precision_at_3": 0.30952, - "precision_at_5": 0.19048, - "recall_at_1": 0.79464, - "recall_at_10": 0.97024, - "recall_at_3": 0.92857, - "recall_at_5": 0.95238 - } - ] - }, - "task_name": "LEMBSummScreenFDRetrievalChunked" -} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_8192/overlap_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-soft-boundary/embed_size_8192/overlap_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json deleted file mode 100644 index da6d73d..0000000 --- a/results-soft-boundary/embed_size_8192/overlap_64/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 38.01535248756409, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.94608, - "map_at_1": 0.90667, - "map_at_10": 0.93403, - "map_at_3": 0.93, - "map_at_5": 0.93233, - "mrr_at_1": 0.9066666666666666, - "mrr_at_10": 0.9340317460317461, - "mrr_at_3": 0.93, - "mrr_at_5": 0.9323333333333333, - "ndcg_at_1": 0.90667, - "ndcg_at_10": 0.94608, - "ndcg_at_3": 0.9377, - "ndcg_at_5": 0.94186, - "precision_at_1": 0.90667, - "precision_at_10": 0.09833, - "precision_at_3": 0.32, - "precision_at_5": 0.194, - "recall_at_1": 0.90667, - "recall_at_10": 0.98333, - "recall_at_3": 0.96, - "recall_at_5": 0.97 - } - ] - }, - "task_name": "LEMBWikimQARetrievalChunked" -} \ No newline at end of file diff --git a/results-soft-boundary/embed_size_8192/overlap_64/no_model_name_available/no_revision_available/model_meta.json b/results-soft-boundary/embed_size_8192/overlap_64/no_model_name_available/no_revision_available/model_meta.json deleted file mode 100644 index 40ff52f..0000000 --- a/results-soft-boundary/embed_size_8192/overlap_64/no_model_name_available/no_revision_available/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file diff --git a/results-truncation/embed_size_8192/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json b/results-truncation/embed_size_8192/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json deleted file mode 100644 index eb44b65..0000000 --- a/results-truncation/embed_size_8192/no_model_name_available/no_revision_available/LEMBQMSumRetrievalChunked.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 33.62538528442383, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.48383, - "map_at_1": 0.32547, - "map_at_10": 0.42617, - "map_at_20": 0.43314, - "map_at_3": 0.39413, - "map_at_5": 0.41246, - "mrr_at_1": 0.3254747871643746, - "mrr_at_10": 0.4261722851113814, - "mrr_at_20": 0.43314067378880616, - "mrr_at_3": 0.3941279196681947, - "mrr_at_5": 0.41246452739576517, - "ndcg_at_1": 0.32547, - "ndcg_at_10": 0.48383, - "ndcg_at_20": 0.5096, - "ndcg_at_3": 0.41762, - "ndcg_at_5": 0.45059, - "precision_at_1": 0.32547, - "precision_at_10": 0.06686, - "precision_at_20": 0.03857, - "precision_at_3": 0.16197, - "precision_at_5": 0.11316, - "recall_at_1": 0.32547, - "recall_at_10": 0.66863, - "recall_at_20": 0.77145, - "recall_at_3": 0.48592, - "recall_at_5": 0.56582 - } - ] - }, - "task_name": "LEMBQMSumRetrievalChunked" -} \ No newline at end of file diff --git a/results-truncation/embed_size_8192/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json b/results-truncation/embed_size_8192/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json deleted file mode 100644 index ec08f1f..0000000 --- a/results-truncation/embed_size_8192/no_model_name_available/no_revision_available/LEMBSummScreenFDRetrievalChunked.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 42.96391987800598, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.88557, - "map_at_1": 0.78571, - "map_at_10": 0.85666, - "map_at_20": 0.85776, - "map_at_3": 0.84821, - "map_at_5": 0.85446, - "mrr_at_1": 0.7857142857142857, - "mrr_at_10": 0.8566633597883598, - "mrr_at_20": 0.8577582800797087, - "mrr_at_3": 0.8482142857142857, - "mrr_at_5": 0.8544642857142858, - "ndcg_at_1": 0.78571, - "ndcg_at_10": 0.88557, - "ndcg_at_20": 0.88941, - "ndcg_at_3": 0.86883, - "ndcg_at_5": 0.87998, - "precision_at_1": 0.78571, - "precision_at_10": 0.09732, - "precision_at_20": 0.0494, - "precision_at_3": 0.30952, - "precision_at_5": 0.19107, - "recall_at_1": 0.78571, - "recall_at_10": 0.97321, - "recall_at_20": 0.9881, - "recall_at_3": 0.92857, - "recall_at_5": 0.95536 - } - ] - }, - "task_name": "LEMBSummScreenFDRetrievalChunked" -} \ No newline at end of file diff --git a/results-truncation/embed_size_8192/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json b/results-truncation/embed_size_8192/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json deleted file mode 100644 index 7d1c0fb..0000000 --- a/results-truncation/embed_size_8192/no_model_name_available/no_revision_available/LEMBWikimQARetrievalChunked.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "dataset_revision": "10039a580487dacecf79db69166e17ace3ede392", - "evaluation_time": 36.08832788467407, - "kg_co2_emissions": null, - "mteb_version": "1.14.20", - "scores": { - "test": [ - { - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.88074, - "map_at_1": 0.83667, - "map_at_10": 0.86607, - "map_at_20": 0.86737, - "map_at_3": 0.85944, - "map_at_5": 0.86378, - "mrr_at_1": 0.8366666666666667, - "mrr_at_10": 0.8660687830687831, - "mrr_at_20": 0.8673737974987974, - "mrr_at_3": 0.8594444444444443, - "mrr_at_5": 0.8637777777777778, - "ndcg_at_1": 0.83667, - "ndcg_at_10": 0.88074, - "ndcg_at_20": 0.8857, - "ndcg_at_3": 0.86726, - "ndcg_at_5": 0.87529, - "precision_at_1": 0.83667, - "precision_at_10": 0.09267, - "precision_at_20": 0.04733, - "precision_at_3": 0.29667, - "precision_at_5": 0.182, - "recall_at_1": 0.83667, - "recall_at_10": 0.92667, - "recall_at_20": 0.94667, - "recall_at_3": 0.89, - "recall_at_5": 0.91 - } - ] - }, - "task_name": "LEMBWikimQARetrievalChunked" -} \ No newline at end of file diff --git a/results-truncation/embed_size_8192/no_model_name_available/no_revision_available/model_meta.json b/results-truncation/embed_size_8192/no_model_name_available/no_revision_available/model_meta.json deleted file mode 100644 index 40ff52f..0000000 --- a/results-truncation/embed_size_8192/no_model_name_available/no_revision_available/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": null, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file From 9fc7d52bb0a4b322869e52f1287f39cc354a71f8 Mon Sep 17 00:00:00 2001 From: dannyjameswilliams Date: Thu, 26 Sep 2024 20:29:57 +0100 Subject: [PATCH 18/30] added plotting files for results, requires running them first --- experiments/__pycache__/lib.cpython-312.pyc | Bin 0 -> 7157 bytes .../explanatory_contextual_retrieval.py | 187 ++++++++++++++++++ experiments/lib.py | 145 ++++++++++++++ plot_chunk_size_experiments.py | 86 ++++++++ plot_soft_boundary_experiments.py | 158 +++++++++++++++ 5 files changed, 576 insertions(+) create mode 100644 experiments/__pycache__/lib.cpython-312.pyc create mode 100644 experiments/explanatory_contextual_retrieval.py create mode 100644 experiments/lib.py create mode 100644 plot_chunk_size_experiments.py create mode 100644 plot_soft_boundary_experiments.py diff --git a/experiments/__pycache__/lib.cpython-312.pyc b/experiments/__pycache__/lib.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..878501d89a2c0e1bfb349fe89bcb635475768079 GIT binary patch literal 7157 zcmds6Z)_7umaleq`@dtH1l##X%m6bu!TCctZdqnPY{&o$1O~$Hk=tEw5l$l!>Gnf7&>f#nth73X`?4pUbh0-y8>??tH`1(h+HZ!;X|${R zaPL*S?U3;x&3(L5{OVQJtLo~i_ulWl&c6i$UIHZ({qL#2Y9r)xY}iGpF&lpi%zYvg znN!Im$7;u<18Rrr$NCUBd3d~nbXO3!_oRQ+JEiftq<>9yS4?y5pxCPFrkI&XN%Oq<=c08_)%e*Ye z;z#^~Lw3q8*LOQk6!|B<$M|e03qnzt^;rC7`ncAR z{T;x`0Y+PpUL^70cwx_sVdV604-QJX^njE#W~i1+rv_45X7JkZU`Ek(X>cK%fI^w{NscmL4-k;Kr!#L#eIo0Q9`>6B!owd~-mu4UtV!84bDH4|5r z`9i3^aN?Rmbr34}Q(D$gvPQy~&nbmSN|o}mk{C(MNa?Hj1T1}EXk_S5)5J}PDdcs9 z-5AwW#dOtX!kvj3c3<9%G?z5+zI4-VZy9dbHKRx}JkJzxOEd^CP4PlZ!BFOmxM(^w z-E>YXu#an|n=0?;72N>ooTeMJ9R%YdMSC#$JXI;!1I&~qLqb_Sr)70T$2*E)`Qzom zOR!6QP?oaU`N88_DxZPZ(+9t=(W`^!sWz*mpmjzvlqAzZ(8uI?DW}ju75WlPRiSTC znS;KxHCIhv8OY6>+iP1(){ij>Crfw0v$w%dSApCl74PuP(cf*4+`oM9a;bO!+V+u3 zPrMTCd*Kp-&YLHye$v`s33qspw`W-@`}VGc%f6wKFth>BBdN`*jlO{y!9M|WpA?CK zBCIqsf!Vrh+foGP*k)1ifjT>i-2Y(>?v*w8O@`RagusAo&jH53jmHJkkgh~cm1x?S zr*Y`t2|xW95V*Ux&W|oUy!_zu!yi5P(aMRno&)8!12@lnP4}n1z}Pd%(c!=ziIfa)UiXiifC<}^n$}Nkx#7P&pU9skN z-LjkU9&o{)d2ifrwxjCWnmd-cqR3!c6!p*aFxz9kGGyy<5=;mtOwXIXY*GhHP_ijS zPi?+>X!@>IG>fG04+RrR_00EG5(07321>4 z<1XnwBq#wM!~y|2z=lmhSJbIZLO6?!HO$+(NOv-s&ZdoIve47)GzRS9y|94(7Lc2y z+Csuzi^3gmrQ;2#T7DPVQ5re^abye{{z})Ii^3=FD7!z?E!UjJE2iawW!NeG4=`w2 zvrum>v-NXmjNp0B)=y+pKWT<1aH_R6ra3S^($qvb*#VdZsP#mAT)W8y-hq1pH@&sU z?ldX#Kj;32TM!H|X{*3JIbgoiewyq;EP}hM8z`*ix1}337^r9b5l^>6mvxnRd8Q+0 zOr%K+Zdou}sAA-4HVLYxYvA==IZ2kmrJ7!Y=Cc+!nG7pbB|B}*#5sBxS3^hkh{RoV z7*`aTax@-c=>?vR>C^H?E^j22%oQE9YCldszy`qQYX(C<-Sjh10!0B)-3(YQYo@+c zp?KI>A=*@zVB{dIt=|Cx4kFTZzj&{>9*LJD@s+vN1HU@*{75O8T2E@_q*jW=*CM%6 z`29s63Vv|!gNnbs($!mOk3t*%sy-5ZbAvde-bFuXWXG;0;i30|w;bKS`~x71fls^o zD*le8!%q)92!I8p1RlRkI$GFsDUvUZJz8jc+2xyrv`g4By_}_WTzWArD z-KCzXwbtp9f4b7S<6+-}zV*)ja%ca_*jnfQ#b71UvFNX~Mb_JPmD_f`B%J3Iw>Vny zN7wzmWqjTaNwr2OY6L}pWAcD8CZEn4bOiel>e$=Tw~<3kv`(NOIV5kwPd^Rh z7O8s4o5Pjf*b9$Oe5=ydUFD(vEV8>QLJd2O?W($vbCb@V4V0;N2gKOY=mvqRy2CAY zFO7Ubpt7EWtEb>sFLeN>(6t4Cw`~Dnu?E0S1Yp?(z$-x3BLEZ=7aH&*)@t0_M8DHcR?{|TDc~vW1DfC`8d<9su_0Z z3@n=i^iSA3G~%(%&@6M!pyDyi!14-2Bf!@Ii2-~)9N1)!!jpP}1oCzJ2T13~rAMGB zUW}};ylorYkyyR&#*OcvJa=X+dHmeyrL$w>6AKHCv9|}COgO1fC6?A>#*A&O*Ol0H zh_um&^)wtGSn>6=F_X?Rnd>uJaeyX2(|-G2*a4od($Ahl2o;^yD5#bP3Q6yk{U&i zX;ZOu2C~>|;KgFPBGJ@L3}DmJ4jio?!$9mI4H{8&n(d+GQNVcxXf_B0FOWXOW-afWksd$Au47RKACg`WUqz5qi-4k?68!g!(e zs}@lR-MFzuH%%@U=dEPdMU|W?r4;(-u!emj^%u8f0^nDul&mS{mb(Fe4S|gHx7I?(*F$H@p);kkKQ3LFUJK1EIxErc zhwcaN|WB#!8evKm4kgHcb|1*EM7bci@tJW7;Lf|`z0{nEWkGg)+ZwEBn$4M z*fj2O+cypI$v%irM7Uw+_{6&arQurat4&SmilHYn5I|t)+luF_=af`hQfr?`SiDQo zD&m3#0X{4ANaobMZnjuU*elTQqM*kLcI$^}I%}U>0<2Ef9&0+8OVZKKC2Bx80x=A# zj*f81d^Qp*nF`kS0~Lhqb4H>6wTfar&=%Hl(GP%Fy0YWmj`dJ~In-bAZ(C_!-BaaB z>%I-*X!R~S0n9sk*Q1GYGyzHyY6HFL-1Cw+JUefHcX8~)5eSl(dY1F$j>L*o?ihO7 z^GshI{zbk#Jhm#8hfkK;Pc4pB{H=G--afmu=M#UdD#BEZWx|UAren>c;C~0C`evF1 zaV&b0Ek))44{P40mYUTuu_20|17#ld=Se-9eFbfsrfA2>T|NadbjtC=cF<6aq9em+9GaHI|8MC6e!^%g| zzr=kE!kD^S*`tbLX(SR725P5(Ges(8rxid%CCkG7xU-Q~#tTEiV9_}lL`<}X2ua+)ce4Z_Z`_{sj z04sf!6Q_T5gfBi*J~Y-Nd&`l%E8CyCKZzXrb@(DmdNa zFaPk;Ln0I0A6`01_XSSJPXW){I!1ehe>t>m^bLUp_{}Zu|5M&cF)O18hV(R$EeLDn zbUR{a8u!*skrPCOXP}EQQJtR!yzcvl~GZaB`9)9{yfj}DOCOwBMd-lEXbc*kC zEDuA5C1S{cpN(`@U5s@@Qntb5s^8|t{^ea&0@c&D4XmnOj~HGatrDo7%Fq5}wQV)E z8vVu5^1;&^*jEj^tn>}4YU|eY4R;l9GsSE^lbfeIp+(Wgz^&!y- z|GdCP{TNI?ghg2$!tMj60BvBk4!Nj_1|P^^fkitt9vey$&LP zC`3OA1oCN)+i(-E?Q;_TE$RO)+4o;$$A6N;Wpem)68sP1sR}`EczM?bf$HTOJU8&A W_bsmb1p)Hq1b39{eo25ZwfJ9q<52nl literal 0 HcmV?d00001 diff --git a/experiments/explanatory_contextual_retrieval.py b/experiments/explanatory_contextual_retrieval.py new file mode 100644 index 0000000..fb6ccc4 --- /dev/null +++ b/experiments/explanatory_contextual_retrieval.py @@ -0,0 +1,187 @@ +# experiments/explanatory_contextual_retrieval.py +# +# a simple example with a trivial piece of text to showcase the late chunking method against +# contextual retrieval method. contextual retrieval manually inserts context to each +# chunk, i.e. forces context to be around each chunk. so works as a good comparison +# to late chunking to see if the similarities are similar (which they appear to be) +# +# pip requirements: +# accelerate? + +from chunked_pooling.wrappers import load_model +from transformers import AutoModel, AutoTokenizer, pipeline + +import numpy as np + +import chunked_pooling +from chunked_pooling import chunked_pooling +from chunked_pooling.chunking import Chunker + +from typing import List, Tuple +from transformers import AutoModel, AutoTokenizer, pipeline + +import requests +import os + +def request_anthropic_api(prompt: str): + url = "https://api.anthropic.com/v1/messages" + headers = { + "x-api-key": os.getenv("ANTHROPIC_API_KEY"), + "anthropic-version": "2023-06-01", + "content-type": "application/json" + } + data = { + "model": "claude-3-haiku-20240307", + "max_tokens": 2048, + "messages": [ + {"role": "user", "content": prompt} + ] + } + response = requests.post(url, headers=headers, json=data) + return response.json()["content"][0]["text"] + +def cosine_similarity(vector1, vector2): + vector1_norm = vector1 / np.linalg.norm(vector1) + vector2_norm = vector2 / np.linalg.norm(vector2) + return np.dot(vector1_norm, vector2_norm) + +class LateChunkingEmbedder: + + def __init__(self, + model: AutoModel, + tokenizer: AutoTokenizer, + chunking_strategy: str = "sentences", + n_sentences: int = 1 + ): + + self.model = model + self.tokenizer = tokenizer + + self.chunker = Chunker(chunking_strategy = chunking_strategy) + self.n_sentences = n_sentences + + + def run(self, document: str): + annotations = [self.chunker.chunk(text=document, tokenizer=self.tokenizer, n_sentences=self.n_sentences)] + model_inputs = self.tokenizer( + document, + return_tensors='pt', + padding=True, + truncation=True, + max_length=8192, + ) + model_outputs = self.model(**model_inputs) + self.output_embs = chunked_pooling( + model_outputs, annotations, max_length=8192, + )[0] + return self.output_embs + + def query(self, query: str): + if "output_embs" not in dir(self): + raise ValueError("no embeddings calculated, use .run(document) to create chunk embeddings") + query_embedding = self.model.encode(query) + similarities = [] + for emb in self.output_embs: + similarities.append(cosine_similarity(query_embedding, emb)) + + return similarities + + +class ContextualRetrievalEmbedder(): + def __init__(self, + model: AutoModel, + tokenizer: AutoTokenizer, + llm_name: str = "meta-llama/Meta-Llama-3.1-8B", + chunking_strategy: str = "fixed" + ): + # self.llm = pipeline( + # "text-generation", model=llm_name, model_kwargs={"torch_dtype": torch.bfloat16}, device_map="auto", + # max_length = 1000 + # ) + self.llm = request_anthropic_api + + self.prompt = """ + + {{WHOLE_DOCUMENT}} + + Here is the chunk we want to situate within the whole document + + {{CHUNK_CONTENT}} + + Please give a short succinct context to situate this chunk within the overall document for the purposes of improving search retrieval of the chunk. Answer only with the succinct context and nothing else. + """.strip() + + self.model = model + self.tokenizer = tokenizer + + self.chunker = Chunker(chunking_strategy = chunking_strategy) + + + def _add_context(self, chunk: str, document: str): + prompt = self.prompt.replace("{{WHOLE_DOCUMENT}}", document).replace("{{CHUNK_CONTENT}}", chunk) + extra_context = self.llm(prompt) + return extra_context + " " + chunk + + def _tokens_to_text(self, text: str, annotations: List[Tuple[int, int]]): + tokens = self.tokenizer.encode_plus( + text, return_offsets_mapping=True, add_special_tokens=False + ) + token_offsets = tokens.offset_mapping + chunks = [] + for start, end in annotations: + chunk = text[token_offsets[start][0]:token_offsets[end-1][1]] + chunks.append(chunk) + return chunks + + def run(self, document: str): + annotations = [self.chunker.chunk(text=document, tokenizer=self.tokenizer, n_sentences=1)] + self.chunks = self._tokens_to_text(text=document, annotations=annotations[0]) + self.chunks = [self._add_context(chunk, document) for chunk in self.chunks] + + model_outputs = self.model.encode(self.chunks) + self.output_embs = [model_outputs[i, :] for i in range(len(self.chunks))] + return self.output_embs + + def query(self, query: str): + if "output_embs" not in dir(self): + raise ValueError("no embeddings calculated, use .run(document) to create chunk embeddings") + query_embedding = self.model.encode(query) + similarities = [] + for emb in self.output_embs: + similarities.append(cosine_similarity(query_embedding, emb)) + + return similarities + + + +if __name__ == "__main__": + + text = """ + The recent SEC filing provided insights into ACME Corp's performance for Q2 2023. + It highlighted a 3% revenue growth over the previous quarter. + The company, which had a revenue of $314 million in the prior quarter, showed steady progress. + They attributed this growth to strategic initiatives and operational efficiencies. + The report emphasized the company's resilience and ability to navigate market challenges, reflecting positively on their financial health and future prospects. + """.strip().replace("\n", "") + + + # llm_model_name = "microsoft/Phi-3.5-mini-instruct" + embedding_model_name = "jinaai/jina-embeddings-v2-small-en" + + embedding_model, has_instructions = load_model(embedding_model_name) + embedding_tokenizer = AutoTokenizer.from_pretrained(embedding_model_name, trust_remote_code=True) + + + cr = ContextualRetrievalEmbedder(embedding_model, embedding_tokenizer, chunking_strategy="sentences") + cr.run(text); + cr_cosine_similarities = cr.query("What is ACME Corp's revenue growth for Q2 2023?") + + lc = LateChunkingEmbedder(embedding_model, embedding_tokenizer) + lc.run(text) + lc_cosine_similarities = lc.query("What is ACME Corp's revenue growth for Q2 2023?") + + # import pandas as pd + for i, (cr_similarity, lc_similarity) in enumerate(zip(cr_cosine_similarities, lc_cosine_similarities)): + print(f"{text.split('.')[:-1][i].strip()}") + print(f"Similarities: Contextual Retrieval: {cr_similarity:.4f} | Late Chunking: {lc_similarity:.4f}") + print("") \ No newline at end of file diff --git a/experiments/lib.py b/experiments/lib.py new file mode 100644 index 0000000..f60133f --- /dev/null +++ b/experiments/lib.py @@ -0,0 +1,145 @@ +# pip requirements: +# accelerate? + +import numpy as np + +import chunked_pooling +from chunked_pooling import chunked_pooling +from chunked_pooling.chunking import Chunker + +from typing import List, Tuple +from transformers import AutoModel, AutoTokenizer, pipeline + +import requests +import os + +def request_anthropic_api(prompt: str): + url = "https://api.anthropic.com/v1/messages" + headers = { + "x-api-key": os.getenv("ANTHROPIC_API_KEY"), + "anthropic-version": "2023-06-01", + "content-type": "application/json" + } + data = { + "model": "claude-3-haiku-20240307", + "max_tokens": 2048, + "messages": [ + {"role": "user", "content": prompt} + ] + } + response = requests.post(url, headers=headers, json=data) + return response.json()["content"][0]["text"] + +def cosine_similarity(vector1, vector2): + vector1_norm = vector1 / np.linalg.norm(vector1) + vector2_norm = vector2 / np.linalg.norm(vector2) + return np.dot(vector1_norm, vector2_norm) + +class LateChunkingEmbedder: + + def __init__(self, + model: AutoModel, + tokenizer: AutoTokenizer, + chunking_strategy: str = "sentences", + n_sentences: int = 1 + ): + + self.model = model + self.tokenizer = tokenizer + + self.chunker = Chunker(chunking_strategy = chunking_strategy) + self.n_sentences = n_sentences + + + def run(self, document: str): + annotations = [self.chunker.chunk(text=document, tokenizer=self.tokenizer, n_sentences=self.n_sentences)] + model_inputs = self.tokenizer( + document, + return_tensors='pt', + padding=True, + truncation=True, + max_length=8192, + ) + model_outputs = self.model(**model_inputs) + self.output_embs = chunked_pooling( + model_outputs, annotations, max_length=8192, + )[0] + return self.output_embs + + def query(self, query: str): + if "output_embs" not in dir(self): + raise ValueError("no embeddings calculated, use .run(document) to create chunk embeddings") + query_embedding = self.model.encode(query) + similarities = [] + for emb in self.output_embs: + similarities.append(cosine_similarity(query_embedding, emb)) + + return similarities + + +class ContextualRetrievalEmbedder(): + def __init__(self, + model: AutoModel, + tokenizer: AutoTokenizer, + llm_name: str = "meta-llama/Meta-Llama-3.1-8B", + chunking_strategy: str = "fixed" + ): + # self.llm = pipeline( + # "text-generation", model=llm_name, model_kwargs={"torch_dtype": torch.bfloat16}, device_map="auto", + # max_length = 1000 + # ) + self.llm = request_anthropic_api + + self.prompt = """ + + {{WHOLE_DOCUMENT}} + + Here is the chunk we want to situate within the whole document + + {{CHUNK_CONTENT}} + + Please give a short succinct context to situate this chunk within the overall document for the purposes of improving search retrieval of the chunk. Answer only with the succinct context and nothing else. + """.strip() + + self.model = model + self.tokenizer = tokenizer + + self.chunker = Chunker(chunking_strategy = chunking_strategy) + + + def _add_context(self, chunk: str, document: str): + prompt = self.prompt.replace("{{WHOLE_DOCUMENT}}", document).replace("{{CHUNK_CONTENT}}", chunk) + extra_context = self.llm(prompt) + return extra_context + " " + chunk + + def _tokens_to_text(self, text: str, annotations: List[Tuple[int, int]]): + tokens = self.tokenizer.encode_plus( + text, return_offsets_mapping=True, add_special_tokens=False + ) + token_offsets = tokens.offset_mapping + chunks = [] + for start, end in annotations: + chunk = text[token_offsets[start][0]:token_offsets[end-1][1]] + chunks.append(chunk) + return chunks + + def run(self, document: str): + annotations = [self.chunker.chunk(text=document, tokenizer=self.tokenizer, n_sentences=1)] + self.chunks = self._tokens_to_text(text=document, annotations=annotations[0]) + self.chunks = [self._add_context(chunk, document) for chunk in self.chunks] + + model_outputs = self.model.encode(self.chunks) + self.output_embs = [model_outputs[i, :] for i in range(len(self.chunks))] + return self.output_embs + + def query(self, query: str): + if "output_embs" not in dir(self): + raise ValueError("no embeddings calculated, use .run(document) to create chunk embeddings") + query_embedding = self.model.encode(query) + similarities = [] + for emb in self.output_embs: + similarities.append(cosine_similarity(query_embedding, emb)) + + return similarities + + \ No newline at end of file diff --git a/plot_chunk_size_experiments.py b/plot_chunk_size_experiments.py new file mode 100644 index 0000000..ce8b9d4 --- /dev/null +++ b/plot_chunk_size_experiments.py @@ -0,0 +1,86 @@ +import numpy as np +import matplotlib.pyplot as plt +import json +import os + +if __name__ == "__main__": + + lc_dir = "results-chunked-pooling" + nc_dir = "results-normal-pooling" + + # == Load results + + chunk_sizes = [8, 16, 32, 64, 128, 256, 512, 1024] + task_names = ["SummScreenFD", "WikimQA"] # "QMSum", + + lc_ndcgs = np.empty((len(task_names), len(chunk_sizes))) + nc_ndcgs = np.empty((len(task_names), len(chunk_sizes))) + + lc_maps = np.empty((len(task_names), len(chunk_sizes))) + nc_maps = np.empty((len(task_names), len(chunk_sizes))) + + for task_i, task_name in enumerate(task_names): + + task_name = f"LEMB{task_name}RetrievalChunked" + + for chunk_i, chunk_size in enumerate(chunk_sizes): + + lc_dir_chunk_i = os.path.join(lc_dir, f"chunk_size_{chunk_size}", "no_model_name_available", "no_revision_available", f"{task_name}.json") + with open(lc_dir_chunk_i, "r") as f: + results = json.load(f) + + lc_ndcgs[task_i, chunk_i] = results["scores"]["test"][0]["ndcg_at_10"] + lc_maps[task_i, chunk_i] = results["scores"]["test"][0]["map_at_10"] + + nc_dir_chunk_i = os.path.join(nc_dir, f"chunk_size_{chunk_size}", "no_model_name_available", "no_revision_available", f"{task_name}.json") + with open(nc_dir_chunk_i, "r") as f: + results = json.load(f) + + nc_ndcgs[task_i, chunk_i] = results["scores"]["test"][0]["ndcg_at_10"] + nc_maps[task_i, chunk_i] = results["scores"]["test"][0]["map_at_10"] + + # == Plot + plt.style.use('seaborn-v0_8-whitegrid') + plt.rcParams.update({'font.size': 14, 'font.family': 'serif'}) + + # -- NDCG + fig, ax = plt.subplots(len(task_names), 1, figsize=(12, 3*len(task_names)), sharex=True) + + for task_i in range(len(task_names)): + ax[task_i].plot(chunk_sizes, lc_ndcgs[task_i], label="Late Chunking", color="blue", marker='o', linestyle='-', linewidth=2, markersize=6) + ax[task_i].plot(chunk_sizes, nc_ndcgs[task_i], label="Naive Chunking", color="red", marker='o', linestyle='-', linewidth=2, markersize=6) + ax[task_i].set_title(f"Task: {task_names[task_i]}", fontsize=16) + ax[task_i].set_ylabel("nCDG@10", fontsize=14) + ax[task_i].legend(loc='best', fontsize=12, frameon=True) + + ax[task_i].set_xlabel("Chunk Size", fontsize=14) + + # log scale + ax[task_i].set_xscale('log') + + # set specific ticks + ax[task_i].set_xticks([8, 16, 32, 64, 128, 256, 512, 1024]) + ax[task_i].set_xticklabels([8, 16, 32, 64, 128, 256, 512, 1024]) + fig.suptitle("nCDG@10 for Different Chunk Sizes", fontsize=18) + fig.tight_layout(rect=[0, 0, 1, 0.96]) + + # -- mAP + fig, ax = plt.subplots(len(task_names), 1, figsize=(12, 3*len(task_names)), sharex=True) + + for task_i in range(len(task_names)): + ax[task_i].plot(chunk_sizes, lc_maps[task_i], label="Late Chunking", color="blue", marker='o', linestyle='-', linewidth=2, markersize=6) + ax[task_i].plot(chunk_sizes, nc_maps[task_i], label="Naive Chunking", color="red", marker='o', linestyle='-', linewidth=2, markersize=6) + ax[task_i].set_title(f"Task: {task_names[task_i]}", fontsize=16) + ax[task_i].set_ylabel("mAP@10", fontsize=14) + ax[task_i].legend(loc='best', fontsize=12, frameon=True) + + ax[task_i].set_xlabel("Chunk Size", fontsize=14) + + # log scale + ax[task_i].set_xscale('log') + + # set specific ticks + ax[task_i].set_xticks([8, 16, 32, 64, 128, 256, 512, 1024]) + ax[task_i].set_xticklabels([8, 16, 32, 64, 128, 256, 512, 1024]) + fig.suptitle("mAP@10 for Different Chunk Sizes", fontsize=18) + fig.tight_layout(rect=[0, 0, 1, 0.96]) \ No newline at end of file diff --git a/plot_soft_boundary_experiments.py b/plot_soft_boundary_experiments.py new file mode 100644 index 0000000..62145d4 --- /dev/null +++ b/plot_soft_boundary_experiments.py @@ -0,0 +1,158 @@ +import numpy as np +import matplotlib.pyplot as plt +import json +import os + +if __name__ == "__main__": + + task_name = "LEMBWikimQARetrievalChunked" + soft_boundary_dir = "results-soft-boundary" + hard_boundary_dir = "results-hard-boundary" + truncated_boundary_dir = "results-truncation" + + # == Load results + + embed_sizes = [1024, 2048, 4096, 8192] + overlap_sizes = [32, 64, 128, 256, 512] + task_names = ["QMSum", "SummScreenFD", "WikimQA"] + + soft_boundary_ndcgs = np.empty((len(task_names), len(embed_sizes), len(overlap_sizes))) + hard_boundary_ndcgs = np.empty((len(task_names), len(embed_sizes))) + + soft_boundary_maps = np.empty((len(task_names), len(embed_sizes), len(overlap_sizes))) + hard_boundary_maps = np.empty((len(task_names), len(embed_sizes))) + + for task_i, task_name in enumerate(task_names): + + task_name = f"LEMB{task_name}RetrievalChunked" + + for embed_i, embed_size in enumerate(embed_sizes): + + for overlap_i, overlap_size in enumerate(overlap_sizes): + + soft_dir = os.path.join(soft_boundary_dir, f"embed_size_{embed_size}", f"overlap_{overlap_size}", "no_model_name_available", "no_revision_available", f"{task_name}.json") + with open(soft_dir, "r") as f: + results = json.load(f) + + soft_boundary_ndcgs[task_i, embed_i, overlap_i] = results["scores"]["test"][0]["ndcg_at_10"] + soft_boundary_maps[task_i, embed_i, overlap_i] = results["scores"]["test"][0]["map_at_10"] + + soft_dir = os.path.join(hard_boundary_dir, f"embed_size_{embed_size}", "no_model_name_available", "no_revision_available", f"{task_name}.json") + with open(soft_dir, "r") as f: + results = json.load(f) + + hard_boundary_ndcgs[task_i, embed_i] = results["scores"]["test"][0]["ndcg_at_10"] + hard_boundary_maps[task_i, embed_i] = results["scores"]["test"][0]["map_at_10"] + + # truncation_dir = os.path.join(truncated_boundary_dir, "no_model_name_available", "no_revision_available", f"{task_name}.json") + # with open(truncation_dir, "r") as f: + # results = json.load(f) + + # truncated_boundary_ndcg = results["scores"]["test"][0]["ndcg_at_10"] + # truncated_boundary_map = results["scores"]["test"][0]["map_at_10"] + + + # == Plot (by task separately) + + # NDCG + plt.style.use('seaborn-v0_8-whitegrid') + plt.rcParams.update({'font.size': 14, 'font.family': 'serif'}) + + fig, ax = plt.subplots(len(task_names), 1, figsize=(12, 3*len(task_names)), sharex=True) + + # colors = plt.cm.get_cmap('plasma', len(embed_sizes))(np.linspace(0, 1, len(embed_sizes), endpoint=False)) + colors = ["red", "blue", "green", "orange"] + + for task_i in range(len(task_names)): + for embed_i, embed_size in enumerate(embed_sizes[:-1]): + + ax[task_i].plot([o - embed_i*5 for o in overlap_sizes], soft_boundary_ndcgs[task_i, embed_i], label=f"Soft Boundary ({embed_size})", color=colors[embed_i], marker='o', linestyle='-', linewidth=2, markersize=6) + ax[task_i].axhline(hard_boundary_ndcgs[task_i, embed_i], linestyle="--", label=f"Hard Boundary ({embed_size})", linewidth=2, color=colors[embed_i]) + + ax[task_i].set_title(f"Embed Size: {embed_size}", fontsize=16) + ax[task_i].set_ylabel("nCDG@10", fontsize=14) + ax[task_i].legend(loc='best', fontsize=12) + + ax[task_i].set_title("Task: " + task_names[task_i], fontsize=16) + + # ax.axhline(truncated_boundary_ndcg, label=f"Truncated Boundary (at 8192)", color="k", linestyle='-.', linewidth=2) + ax[task_i].set_xlabel("Overlap Size", fontsize=14) + + fig.suptitle("NDCG@10 for Different Embed Sizes and Overlap Sizes", fontsize=18) + fig.tight_layout(rect=[0, 0, 1, 0.96]) + + # MAP + plt.style.use('seaborn-v0_8-whitegrid') + plt.rcParams.update({'font.size': 14, 'font.family': 'serif'}) + + fig, ax = plt.subplots(len(task_names), 1, figsize=(12, 3*len(task_names)), sharex=True) + + colors = ["red", "blue", "green", "orange"] + + for task_i in range(len(task_names)): + for embed_i, embed_size in enumerate(embed_sizes[:-1]): + + ax[task_i].plot([o - embed_i*5 for o in overlap_sizes], soft_boundary_maps[task_i, embed_i], label=f"Soft Boundary ({embed_size})", color=colors[embed_i], marker='o', linestyle='-', linewidth=2, markersize=6) + ax[task_i].axhline(hard_boundary_maps[task_i, embed_i], linestyle="--", label=f"Hard Boundary ({embed_size})", linewidth=2, color=colors[embed_i]) + + ax[task_i].set_title(f"Embed Size: {embed_size}", fontsize=16) + ax[task_i].set_ylabel("nCDG@10", fontsize=14) + ax[task_i].legend(loc='best', fontsize=12) + + ax[task_i].set_title("Task: " + task_names[task_i], fontsize=16) + + # ax.axhline(truncated_boundary_map, label=f"Truncated Boundary (at 8192)", color="k", linestyle='-.', linewidth=2) + ax[task_i].set_xlabel("Overlap Size", fontsize=14) + + fig.suptitle("mAP@10 for Different Embed Sizes and Overlap Sizes", fontsize=18) + fig.tight_layout(rect=[0, 0, 1, 0.96]) + + # == Plot difference in ncdg and map + + plt.style.use('seaborn-v0_8-whitegrid') + plt.rcParams.update({'font.size': 14, 'font.family': 'serif'}) + + fig, ax = plt.subplots(2, 1, figsize=(12, 10)) + + colors = ["red", "blue", "green", "orange"] + markers = ["o", "s", "D", "X"] + + for task_i in range(len(task_names)): + for embed_i, embed_size in enumerate(embed_sizes): + diff_ndcgs = soft_boundary_ndcgs[task_i, embed_i] - hard_boundary_ndcgs[task_i, embed_i] + ax[0].plot([o - embed_i*5 for o in overlap_sizes], diff_ndcgs, color=colors[embed_i], marker=markers[task_i], linestyle='-', linewidth=1.25, markersize=6) + + for embed_i, embed_size in enumerate(embed_sizes): + ax[0].plot([], [], label=f"Num Tokens: {embed_size}", color=colors[embed_i], linestyle='-', linewidth=2) + + for task_i in range(len(task_names)): + ax[0].scatter([], [], label=f"Task: {task_names[task_i]}", marker=markers[task_i], color = "k") + + ax[0].axhline(0, color="black", linestyle="--", linewidth=2) + ax[0].set_title("Difference in nDCG@10 Between Soft and Hard Boundaries Across Tasks", fontsize=16) + ax[0].set_ylabel("Difference in nDCG@10", fontsize=14) + ax[0].set_xlabel("Overlap Size", fontsize=14) + + for task_i in range(len(task_names)): + for embed_i, embed_size in enumerate(embed_sizes): + diff_maps = soft_boundary_maps[task_i, embed_i] - hard_boundary_maps[task_i, embed_i] + ax[1].plot([o - embed_i*5 for o in overlap_sizes], diff_maps, color=colors[embed_i], marker=markers[task_i], linestyle='-', linewidth=1.25, markersize=6) + + for embed_i, embed_size in enumerate(embed_sizes): + ax[1].plot([], [], label=f"Num Tokens: {embed_size}", color=colors[embed_i], linestyle='-', linewidth=2) + + for task_i in range(len(task_names)): + ax[1].scatter([], [], label=f"Task: {task_names[task_i]}", marker=markers[task_i], color = "k") + + ax[1].axhline(0, color="black", linestyle="--", linewidth=2) + ax[1].set_title("Difference in mAP@10 Between Soft and Hard Boundaries Across Tasks", fontsize=16) + ax[1].set_ylabel("Difference in mAP@10", fontsize=14) + ax[1].set_xlabel("Overlap Size", fontsize=14) + + + ax[-1].legend(loc='best', fontsize=12, frameon=True) + + fig.tight_layout(rect=[0, 0, 1, 0.96]) + + + \ No newline at end of file From 7cff5704fa333dc6877d97b94c26b997d949cd81 Mon Sep 17 00:00:00 2001 From: dannyjameswilliams Date: Thu, 26 Sep 2024 20:30:08 +0100 Subject: [PATCH 19/30] renamed file --- parse_experiments.py | 196 ------------------------------------------- 1 file changed, 196 deletions(-) delete mode 100644 parse_experiments.py diff --git a/parse_experiments.py b/parse_experiments.py deleted file mode 100644 index 56ef77a..0000000 --- a/parse_experiments.py +++ /dev/null @@ -1,196 +0,0 @@ -# overlap size and soft boundary - -import numpy as np - -import matplotlib.pyplot as plt - -import json - - -import os - - - - -if __name__ == "__main__": - - task_name = "LEMBWikimQARetrievalChunked" - soft_boundary_dir = "results-soft-boundary" - hard_boundary_dir = "results-hard-boundary" - truncated_boundary_dir = "results-truncation" - - # == Load results - - embed_sizes = [1024, 2048, 4096, 8192] - overlap_sizes = [32, 64, 128, 256, 512] - task_names = ["QMSum", "SummScreenFD", "WikimQA"] - - soft_boundary_ndcgs = np.empty((len(task_names), len(embed_sizes), len(overlap_sizes))) - hard_boundary_ndcgs = np.empty((len(task_names), len(embed_sizes))) - - soft_boundary_maps = np.empty((len(task_names), len(embed_sizes), len(overlap_sizes))) - hard_boundary_maps = np.empty((len(task_names), len(embed_sizes))) - - for task_i, task_name in enumerate(task_names): - - task_name = f"LEMB{task_name}RetrievalChunked" - - for embed_i, embed_size in enumerate(embed_sizes): - - for overlap_i, overlap_size in enumerate(overlap_sizes): - - soft_dir = os.path.join(soft_boundary_dir, f"embed_size_{embed_size}", f"overlap_{overlap_size}", "no_model_name_available", "no_revision_available", f"{task_name}.json") - with open(soft_dir, "r") as f: - results = json.load(f) - - soft_boundary_ndcgs[task_i, embed_i, overlap_i] = results["scores"]["test"][0]["ndcg_at_10"] - soft_boundary_maps[task_i, embed_i, overlap_i] = results["scores"]["test"][0]["map_at_10"] - - soft_dir = os.path.join(hard_boundary_dir, f"embed_size_{embed_size}", "no_model_name_available", "no_revision_available", f"{task_name}.json") - with open(soft_dir, "r") as f: - results = json.load(f) - - hard_boundary_ndcgs[task_i, embed_i] = results["scores"]["test"][0]["ndcg_at_10"] - hard_boundary_maps[task_i, embed_i] = results["scores"]["test"][0]["map_at_10"] - - # truncation_dir = os.path.join(truncated_boundary_dir, "no_model_name_available", "no_revision_available", f"{task_name}.json") - # with open(truncation_dir, "r") as f: - # results = json.load(f) - - # truncated_boundary_ndcg = results["scores"]["test"][0]["ndcg_at_10"] - # truncated_boundary_map = results["scores"]["test"][0]["map_at_10"] - - - # == Plot (by task separately) - - # NDCG - plt.style.use('seaborn-v0_8-whitegrid') - plt.rcParams.update({'font.size': 14, 'font.family': 'serif'}) - - fig, ax = plt.subplots(len(task_names), 1, figsize=(12, 3*len(task_names)), sharex=True) - - # colors = plt.cm.get_cmap('plasma', len(embed_sizes))(np.linspace(0, 1, len(embed_sizes), endpoint=False)) - colors = ["red", "blue", "green", "orange"] - - for task_i in range(len(task_names)): - for embed_i, embed_size in enumerate(embed_sizes[:-1]): - - ax[task_i].plot([o - embed_i*5 for o in overlap_sizes], soft_boundary_ndcgs[task_i, embed_i], label=f"Soft Boundary ({embed_size})", color=colors[embed_i], marker='o', linestyle='-', linewidth=2, markersize=6) - ax[task_i].axhline(hard_boundary_ndcgs[task_i, embed_i], linestyle="--", label=f"Hard Boundary ({embed_size})", linewidth=2, color=colors[embed_i]) - - ax[task_i].set_title(f"Embed Size: {embed_size}", fontsize=16) - ax[task_i].set_ylabel("nCDG@10", fontsize=14) - ax[task_i].legend(loc='best', fontsize=12) - - ax[task_i].set_title("Task: " + task_names[task_i], fontsize=16) - - # ax.axhline(truncated_boundary_ndcg, label=f"Truncated Boundary (at 8192)", color="k", linestyle='-.', linewidth=2) - ax[task_i].set_xlabel("Overlap Size", fontsize=14) - - fig.suptitle("NDCG@10 for Different Embed Sizes and Overlap Sizes", fontsize=18) - fig.tight_layout(rect=[0, 0, 1, 0.96]) - - # MAP - plt.style.use('seaborn-v0_8-whitegrid') - plt.rcParams.update({'font.size': 14, 'font.family': 'serif'}) - - fig, ax = plt.subplots(len(task_names), 1, figsize=(12, 3*len(task_names)), sharex=True) - - # colors = plt.cm.get_cmap('plasma', len(embed_sizes))(np.linspace(0, 1, len(embed_sizes), endpoint=False)) - colors = ["red", "blue", "green", "orange"] - - for task_i in range(len(task_names)): - for embed_i, embed_size in enumerate(embed_sizes[:-1]): - - ax[task_i].plot([o - embed_i*5 for o in overlap_sizes], soft_boundary_maps[task_i, embed_i], label=f"Soft Boundary ({embed_size})", color=colors[embed_i], marker='o', linestyle='-', linewidth=2, markersize=6) - ax[task_i].axhline(hard_boundary_maps[task_i, embed_i], linestyle="--", label=f"Hard Boundary ({embed_size})", linewidth=2, color=colors[embed_i]) - - ax[task_i].set_title(f"Embed Size: {embed_size}", fontsize=16) - ax[task_i].set_ylabel("nCDG@10", fontsize=14) - ax[task_i].legend(loc='best', fontsize=12) - - ax[task_i].set_title("Task: " + task_names[task_i], fontsize=16) - - # ax.axhline(truncated_boundary_map, label=f"Truncated Boundary (at 8192)", color="k", linestyle='-.', linewidth=2) - ax[task_i].set_xlabel("Overlap Size", fontsize=14) - - fig.suptitle("mAP@10 for Different Embed Sizes and Overlap Sizes", fontsize=18) - fig.tight_layout(rect=[0, 0, 1, 0.96]) - - - # == plot with normalized ncdg for all tasks - plt.style.use('seaborn-v0_8-whitegrid') - plt.rcParams.update({'font.size': 14, 'font.family': 'serif'}) - - fig, ax = plt.subplots(1, 1, figsize=(12, 6)) - - # colors = plt.cm.get_cmap('plasma', len(embed_sizes))(np.linspace(0, 1, len(embed_sizes), endpoint=False)) - colors = ["red", "blue", "green", "orange"] - - for task_i in range(len(task_names)): - for embed_i, embed_size in enumerate(embed_sizes[:-1]): - ax.plot([o - embed_i*5 for o in overlap_sizes], soft_boundary_ndcgs[task_i, embed_i] / np.mean(soft_boundary_ndcgs[task_i, embed_i]), color=colors[embed_i], marker='o', linestyle='-', linewidth=1, markersize=4) - ax.axhline(hard_boundary_ndcgs[task_i, embed_i] / np.mean(hard_boundary_ndcgs[task_i, embed_i]), linestyle="--", linewidth=1, color=colors[embed_i]) - - for embed_i, embed_size in enumerate(embed_sizes[:-1]): - ax.plot([], [], label=f"Embed Size: {embed_size}", color=colors[embed_i], linestyle='-', linewidth=2) - - - - ax.set_title("NDCG@10 for Different Embed Sizes and Overlap Sizes Across Tasks (Normalized)", fontsize=16) - ax.set_ylabel("Normalized nCDG@10", fontsize=14) - ax.legend(loc='best', fontsize=12) - ax.set_xlabel("Overlap Size", fontsize=14) - - fig.tight_layout(rect=[0, 0, 1, 0.96]) - - # == - - - plt.style.use('seaborn-v0_8-whitegrid') - plt.rcParams.update({'font.size': 14, 'font.family': 'serif'}) - - fig, ax = plt.subplots(2, 1, figsize=(12, 10)) - - colors = ["red", "blue", "green", "orange"] - markers = ["o", "s", "D", "X"] - - for task_i in range(len(task_names)): - for embed_i, embed_size in enumerate(embed_sizes): - diff_ndcgs = soft_boundary_ndcgs[task_i, embed_i] - hard_boundary_ndcgs[task_i, embed_i] - ax[0].plot([o - embed_i*5 for o in overlap_sizes], diff_ndcgs, color=colors[embed_i], marker=markers[task_i], linestyle='-', linewidth=1.25, markersize=6) - - for embed_i, embed_size in enumerate(embed_sizes): - ax[0].plot([], [], label=f"Num Tokens: {embed_size}", color=colors[embed_i], linestyle='-', linewidth=2) - - for task_i in range(len(task_names)): - ax[0].scatter([], [], label=f"Task: {task_names[task_i]}", marker=markers[task_i], color = "k") - - ax[0].axhline(0, color="black", linestyle="--", linewidth=2) - ax[0].set_title("Difference in nDCG@10 Between Soft and Hard Boundaries Across Tasks", fontsize=16) - ax[0].set_ylabel("Difference in nDCG@10", fontsize=14) - ax[0].set_xlabel("Overlap Size", fontsize=14) - - for task_i in range(len(task_names)): - for embed_i, embed_size in enumerate(embed_sizes): - diff_maps = soft_boundary_maps[task_i, embed_i] - hard_boundary_maps[task_i, embed_i] - ax[1].plot([o - embed_i*5 for o in overlap_sizes], diff_maps, color=colors[embed_i], marker=markers[task_i], linestyle='-', linewidth=1.25, markersize=6) - - for embed_i, embed_size in enumerate(embed_sizes): - ax[1].plot([], [], label=f"Num Tokens: {embed_size}", color=colors[embed_i], linestyle='-', linewidth=2) - - for task_i in range(len(task_names)): - ax[1].scatter([], [], label=f"Task: {task_names[task_i]}", marker=markers[task_i], color = "k") - - ax[1].axhline(0, color="black", linestyle="--", linewidth=2) - ax[1].set_title("Difference in mAP@10 Between Soft and Hard Boundaries Across Tasks", fontsize=16) - ax[1].set_ylabel("Difference in mAP@10", fontsize=14) - ax[1].set_xlabel("Overlap Size", fontsize=14) - - - ax[-1].legend(loc='best', fontsize=12, frameon=True) - - fig.tight_layout(rect=[0, 0, 1, 0.96]) - - - \ No newline at end of file From b8319899bedb7c8b03d8aefe655a43df8a83f44a Mon Sep 17 00:00:00 2001 From: dannyjameswilliams Date: Thu, 26 Sep 2024 20:34:21 +0100 Subject: [PATCH 20/30] renamed file --- ...oundary.py => run_soft_vs_hard_boundary.py | 23 ------------------- 1 file changed, 23 deletions(-) rename soft_token_boundary.py => run_soft_vs_hard_boundary.py (85%) diff --git a/soft_token_boundary.py b/run_soft_vs_hard_boundary.py similarity index 85% rename from soft_token_boundary.py rename to run_soft_vs_hard_boundary.py index 3370bf0..31ba6cf 100644 --- a/soft_token_boundary.py +++ b/run_soft_vs_hard_boundary.py @@ -1,26 +1,3 @@ -# soft_token_boundary.py -# -# long context embedding models (like jina-embeddings-v2-base-en) have a limit of 8192 tokens -# for longer context documents, such as those in LongEmbed benchmarks, how do we embed them? -# -# method 1: truncate the document at the token boundary -# method 2: embed twice, before and after the truncation point as many times as needed -# method 3: same as method 2 but with overlap -# -# obviously method 1 is not great. any information after the 8192nd token is discarded. -# method 3 should perform the best, but how much better is it? -# -# and do we need to look into reducing the size of context window? -# -# let's use the WikimQA dataset to test these ideas, just method 2 and 3. -# -# let's also try to use the mteb benchmark to evaluate the results. - -from chunked_pooling.wrappers import load_model -from transformers import AutoModel, AutoTokenizer, pipeline -# from experiments.lib import ContextualRetrievalEmbedder, LateChunkingEmbedder - - import click import torch.cuda from mteb import MTEB From 1eccfe57258731bb499b7071085aa73c1616913e Mon Sep 17 00:00:00 2001 From: dannyjameswilliams Date: Thu, 26 Sep 2024 20:37:27 +0100 Subject: [PATCH 21/30] added plt.show() --- plot_chunk_size_experiments.py | 4 +++- plot_soft_boundary_experiments.py | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/plot_chunk_size_experiments.py b/plot_chunk_size_experiments.py index ce8b9d4..05ce4e3 100644 --- a/plot_chunk_size_experiments.py +++ b/plot_chunk_size_experiments.py @@ -63,6 +63,7 @@ ax[task_i].set_xticklabels([8, 16, 32, 64, 128, 256, 512, 1024]) fig.suptitle("nCDG@10 for Different Chunk Sizes", fontsize=18) fig.tight_layout(rect=[0, 0, 1, 0.96]) + plt.show() # -- mAP fig, ax = plt.subplots(len(task_names), 1, figsize=(12, 3*len(task_names)), sharex=True) @@ -83,4 +84,5 @@ ax[task_i].set_xticks([8, 16, 32, 64, 128, 256, 512, 1024]) ax[task_i].set_xticklabels([8, 16, 32, 64, 128, 256, 512, 1024]) fig.suptitle("mAP@10 for Different Chunk Sizes", fontsize=18) - fig.tight_layout(rect=[0, 0, 1, 0.96]) \ No newline at end of file + fig.tight_layout(rect=[0, 0, 1, 0.96]) + plt.show() \ No newline at end of file diff --git a/plot_soft_boundary_experiments.py b/plot_soft_boundary_experiments.py index 62145d4..ce3e162 100644 --- a/plot_soft_boundary_experiments.py +++ b/plot_soft_boundary_experiments.py @@ -106,6 +106,7 @@ fig.suptitle("mAP@10 for Different Embed Sizes and Overlap Sizes", fontsize=18) fig.tight_layout(rect=[0, 0, 1, 0.96]) + plt.show() # == Plot difference in ncdg and map @@ -153,6 +154,7 @@ ax[-1].legend(loc='best', fontsize=12, frameon=True) fig.tight_layout(rect=[0, 0, 1, 0.96]) + plt.show() \ No newline at end of file From 4219ca6a92a615c87e06b0dacb3370ef228377a3 Mon Sep 17 00:00:00 2001 From: dannyjameswilliams Date: Thu, 26 Sep 2024 20:39:06 +0100 Subject: [PATCH 22/30] renamed to macro chunking --- ..._boundary_experiments.py => plot_macro_chunking_experiments.py | 0 run_soft_vs_hard_boundary.py => run_macro_chunking_experiments.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename plot_soft_boundary_experiments.py => plot_macro_chunking_experiments.py (100%) rename run_soft_vs_hard_boundary.py => run_macro_chunking_experiments.py (100%) diff --git a/plot_soft_boundary_experiments.py b/plot_macro_chunking_experiments.py similarity index 100% rename from plot_soft_boundary_experiments.py rename to plot_macro_chunking_experiments.py diff --git a/run_soft_vs_hard_boundary.py b/run_macro_chunking_experiments.py similarity index 100% rename from run_soft_vs_hard_boundary.py rename to run_macro_chunking_experiments.py From ad9f37c49a0a23fcca316ca422c62214252da7f1 Mon Sep 17 00:00:00 2001 From: dannyjameswilliams Date: Thu, 26 Sep 2024 20:42:38 +0100 Subject: [PATCH 23/30] moved file --- experiments/__pycache__/lib.cpython-312.pyc | Bin 7157 -> 0 bytes experiments/lib.py | 145 ------------------ ....py => explanatory_contextual_retrieval.py | 0 3 files changed, 145 deletions(-) delete mode 100644 experiments/__pycache__/lib.cpython-312.pyc delete mode 100644 experiments/lib.py rename experiments/explanatory_contextual_retrieval.py => explanatory_contextual_retrieval.py (100%) diff --git a/experiments/__pycache__/lib.cpython-312.pyc b/experiments/__pycache__/lib.cpython-312.pyc deleted file mode 100644 index 878501d89a2c0e1bfb349fe89bcb635475768079..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 7157 zcmds6Z)_7umaleq`@dtH1l##X%m6bu!TCctZdqnPY{&o$1O~$Hk=tEw5l$l!>Gnf7&>f#nth73X`?4pUbh0-y8>??tH`1(h+HZ!;X|${R zaPL*S?U3;x&3(L5{OVQJtLo~i_ulWl&c6i$UIHZ({qL#2Y9r)xY}iGpF&lpi%zYvg znN!Im$7;u<18Rrr$NCUBd3d~nbXO3!_oRQ+JEiftq<>9yS4?y5pxCPFrkI&XN%Oq<=c08_)%e*Ye z;z#^~Lw3q8*LOQk6!|B<$M|e03qnzt^;rC7`ncAR z{T;x`0Y+PpUL^70cwx_sVdV604-QJX^njE#W~i1+rv_45X7JkZU`Ek(X>cK%fI^w{NscmL4-k;Kr!#L#eIo0Q9`>6B!owd~-mu4UtV!84bDH4|5r z`9i3^aN?Rmbr34}Q(D$gvPQy~&nbmSN|o}mk{C(MNa?Hj1T1}EXk_S5)5J}PDdcs9 z-5AwW#dOtX!kvj3c3<9%G?z5+zI4-VZy9dbHKRx}JkJzxOEd^CP4PlZ!BFOmxM(^w z-E>YXu#an|n=0?;72N>ooTeMJ9R%YdMSC#$JXI;!1I&~qLqb_Sr)70T$2*E)`Qzom zOR!6QP?oaU`N88_DxZPZ(+9t=(W`^!sWz*mpmjzvlqAzZ(8uI?DW}ju75WlPRiSTC znS;KxHCIhv8OY6>+iP1(){ij>Crfw0v$w%dSApCl74PuP(cf*4+`oM9a;bO!+V+u3 zPrMTCd*Kp-&YLHye$v`s33qspw`W-@`}VGc%f6wKFth>BBdN`*jlO{y!9M|WpA?CK zBCIqsf!Vrh+foGP*k)1ifjT>i-2Y(>?v*w8O@`RagusAo&jH53jmHJkkgh~cm1x?S zr*Y`t2|xW95V*Ux&W|oUy!_zu!yi5P(aMRno&)8!12@lnP4}n1z}Pd%(c!=ziIfa)UiXiifC<}^n$}Nkx#7P&pU9skN z-LjkU9&o{)d2ifrwxjCWnmd-cqR3!c6!p*aFxz9kGGyy<5=;mtOwXIXY*GhHP_ijS zPi?+>X!@>IG>fG04+RrR_00EG5(07321>4 z<1XnwBq#wM!~y|2z=lmhSJbIZLO6?!HO$+(NOv-s&ZdoIve47)GzRS9y|94(7Lc2y z+Csuzi^3gmrQ;2#T7DPVQ5re^abye{{z})Ii^3=FD7!z?E!UjJE2iawW!NeG4=`w2 zvrum>v-NXmjNp0B)=y+pKWT<1aH_R6ra3S^($qvb*#VdZsP#mAT)W8y-hq1pH@&sU z?ldX#Kj;32TM!H|X{*3JIbgoiewyq;EP}hM8z`*ix1}337^r9b5l^>6mvxnRd8Q+0 zOr%K+Zdou}sAA-4HVLYxYvA==IZ2kmrJ7!Y=Cc+!nG7pbB|B}*#5sBxS3^hkh{RoV z7*`aTax@-c=>?vR>C^H?E^j22%oQE9YCldszy`qQYX(C<-Sjh10!0B)-3(YQYo@+c zp?KI>A=*@zVB{dIt=|Cx4kFTZzj&{>9*LJD@s+vN1HU@*{75O8T2E@_q*jW=*CM%6 z`29s63Vv|!gNnbs($!mOk3t*%sy-5ZbAvde-bFuXWXG;0;i30|w;bKS`~x71fls^o zD*le8!%q)92!I8p1RlRkI$GFsDUvUZJz8jc+2xyrv`g4By_}_WTzWArD z-KCzXwbtp9f4b7S<6+-}zV*)ja%ca_*jnfQ#b71UvFNX~Mb_JPmD_f`B%J3Iw>Vny zN7wzmWqjTaNwr2OY6L}pWAcD8CZEn4bOiel>e$=Tw~<3kv`(NOIV5kwPd^Rh z7O8s4o5Pjf*b9$Oe5=ydUFD(vEV8>QLJd2O?W($vbCb@V4V0;N2gKOY=mvqRy2CAY zFO7Ubpt7EWtEb>sFLeN>(6t4Cw`~Dnu?E0S1Yp?(z$-x3BLEZ=7aH&*)@t0_M8DHcR?{|TDc~vW1DfC`8d<9su_0Z z3@n=i^iSA3G~%(%&@6M!pyDyi!14-2Bf!@Ii2-~)9N1)!!jpP}1oCzJ2T13~rAMGB zUW}};ylorYkyyR&#*OcvJa=X+dHmeyrL$w>6AKHCv9|}COgO1fC6?A>#*A&O*Ol0H zh_um&^)wtGSn>6=F_X?Rnd>uJaeyX2(|-G2*a4od($Ahl2o;^yD5#bP3Q6yk{U&i zX;ZOu2C~>|;KgFPBGJ@L3}DmJ4jio?!$9mI4H{8&n(d+GQNVcxXf_B0FOWXOW-afWksd$Au47RKACg`WUqz5qi-4k?68!g!(e zs}@lR-MFzuH%%@U=dEPdMU|W?r4;(-u!emj^%u8f0^nDul&mS{mb(Fe4S|gHx7I?(*F$H@p);kkKQ3LFUJK1EIxErc zhwcaN|WB#!8evKm4kgHcb|1*EM7bci@tJW7;Lf|`z0{nEWkGg)+ZwEBn$4M z*fj2O+cypI$v%irM7Uw+_{6&arQurat4&SmilHYn5I|t)+luF_=af`hQfr?`SiDQo zD&m3#0X{4ANaobMZnjuU*elTQqM*kLcI$^}I%}U>0<2Ef9&0+8OVZKKC2Bx80x=A# zj*f81d^Qp*nF`kS0~Lhqb4H>6wTfar&=%Hl(GP%Fy0YWmj`dJ~In-bAZ(C_!-BaaB z>%I-*X!R~S0n9sk*Q1GYGyzHyY6HFL-1Cw+JUefHcX8~)5eSl(dY1F$j>L*o?ihO7 z^GshI{zbk#Jhm#8hfkK;Pc4pB{H=G--afmu=M#UdD#BEZWx|UAren>c;C~0C`evF1 zaV&b0Ek))44{P40mYUTuu_20|17#ld=Se-9eFbfsrfA2>T|NadbjtC=cF<6aq9em+9GaHI|8MC6e!^%g| zzr=kE!kD^S*`tbLX(SR725P5(Ges(8rxid%CCkG7xU-Q~#tTEiV9_}lL`<}X2ua+)ce4Z_Z`_{sj z04sf!6Q_T5gfBi*J~Y-Nd&`l%E8CyCKZzXrb@(DmdNa zFaPk;Ln0I0A6`01_XSSJPXW){I!1ehe>t>m^bLUp_{}Zu|5M&cF)O18hV(R$EeLDn zbUR{a8u!*skrPCOXP}EQQJtR!yzcvl~GZaB`9)9{yfj}DOCOwBMd-lEXbc*kC zEDuA5C1S{cpN(`@U5s@@Qntb5s^8|t{^ea&0@c&D4XmnOj~HGatrDo7%Fq5}wQV)E z8vVu5^1;&^*jEj^tn>}4YU|eY4R;l9GsSE^lbfeIp+(Wgz^&!y- z|GdCP{TNI?ghg2$!tMj60BvBk4!Nj_1|P^^fkitt9vey$&LP zC`3OA1oCN)+i(-E?Q;_TE$RO)+4o;$$A6N;Wpem)68sP1sR}`EczM?bf$HTOJU8&A W_bsmb1p)Hq1b39{eo25ZwfJ9q<52nl diff --git a/experiments/lib.py b/experiments/lib.py deleted file mode 100644 index f60133f..0000000 --- a/experiments/lib.py +++ /dev/null @@ -1,145 +0,0 @@ -# pip requirements: -# accelerate? - -import numpy as np - -import chunked_pooling -from chunked_pooling import chunked_pooling -from chunked_pooling.chunking import Chunker - -from typing import List, Tuple -from transformers import AutoModel, AutoTokenizer, pipeline - -import requests -import os - -def request_anthropic_api(prompt: str): - url = "https://api.anthropic.com/v1/messages" - headers = { - "x-api-key": os.getenv("ANTHROPIC_API_KEY"), - "anthropic-version": "2023-06-01", - "content-type": "application/json" - } - data = { - "model": "claude-3-haiku-20240307", - "max_tokens": 2048, - "messages": [ - {"role": "user", "content": prompt} - ] - } - response = requests.post(url, headers=headers, json=data) - return response.json()["content"][0]["text"] - -def cosine_similarity(vector1, vector2): - vector1_norm = vector1 / np.linalg.norm(vector1) - vector2_norm = vector2 / np.linalg.norm(vector2) - return np.dot(vector1_norm, vector2_norm) - -class LateChunkingEmbedder: - - def __init__(self, - model: AutoModel, - tokenizer: AutoTokenizer, - chunking_strategy: str = "sentences", - n_sentences: int = 1 - ): - - self.model = model - self.tokenizer = tokenizer - - self.chunker = Chunker(chunking_strategy = chunking_strategy) - self.n_sentences = n_sentences - - - def run(self, document: str): - annotations = [self.chunker.chunk(text=document, tokenizer=self.tokenizer, n_sentences=self.n_sentences)] - model_inputs = self.tokenizer( - document, - return_tensors='pt', - padding=True, - truncation=True, - max_length=8192, - ) - model_outputs = self.model(**model_inputs) - self.output_embs = chunked_pooling( - model_outputs, annotations, max_length=8192, - )[0] - return self.output_embs - - def query(self, query: str): - if "output_embs" not in dir(self): - raise ValueError("no embeddings calculated, use .run(document) to create chunk embeddings") - query_embedding = self.model.encode(query) - similarities = [] - for emb in self.output_embs: - similarities.append(cosine_similarity(query_embedding, emb)) - - return similarities - - -class ContextualRetrievalEmbedder(): - def __init__(self, - model: AutoModel, - tokenizer: AutoTokenizer, - llm_name: str = "meta-llama/Meta-Llama-3.1-8B", - chunking_strategy: str = "fixed" - ): - # self.llm = pipeline( - # "text-generation", model=llm_name, model_kwargs={"torch_dtype": torch.bfloat16}, device_map="auto", - # max_length = 1000 - # ) - self.llm = request_anthropic_api - - self.prompt = """ - - {{WHOLE_DOCUMENT}} - - Here is the chunk we want to situate within the whole document - - {{CHUNK_CONTENT}} - - Please give a short succinct context to situate this chunk within the overall document for the purposes of improving search retrieval of the chunk. Answer only with the succinct context and nothing else. - """.strip() - - self.model = model - self.tokenizer = tokenizer - - self.chunker = Chunker(chunking_strategy = chunking_strategy) - - - def _add_context(self, chunk: str, document: str): - prompt = self.prompt.replace("{{WHOLE_DOCUMENT}}", document).replace("{{CHUNK_CONTENT}}", chunk) - extra_context = self.llm(prompt) - return extra_context + " " + chunk - - def _tokens_to_text(self, text: str, annotations: List[Tuple[int, int]]): - tokens = self.tokenizer.encode_plus( - text, return_offsets_mapping=True, add_special_tokens=False - ) - token_offsets = tokens.offset_mapping - chunks = [] - for start, end in annotations: - chunk = text[token_offsets[start][0]:token_offsets[end-1][1]] - chunks.append(chunk) - return chunks - - def run(self, document: str): - annotations = [self.chunker.chunk(text=document, tokenizer=self.tokenizer, n_sentences=1)] - self.chunks = self._tokens_to_text(text=document, annotations=annotations[0]) - self.chunks = [self._add_context(chunk, document) for chunk in self.chunks] - - model_outputs = self.model.encode(self.chunks) - self.output_embs = [model_outputs[i, :] for i in range(len(self.chunks))] - return self.output_embs - - def query(self, query: str): - if "output_embs" not in dir(self): - raise ValueError("no embeddings calculated, use .run(document) to create chunk embeddings") - query_embedding = self.model.encode(query) - similarities = [] - for emb in self.output_embs: - similarities.append(cosine_similarity(query_embedding, emb)) - - return similarities - - \ No newline at end of file diff --git a/experiments/explanatory_contextual_retrieval.py b/explanatory_contextual_retrieval.py similarity index 100% rename from experiments/explanatory_contextual_retrieval.py rename to explanatory_contextual_retrieval.py From ada2a9141591eba1dc5ea82c3d9ce850bd2cfc96 Mon Sep 17 00:00:00 2001 From: Danny James Williams <56155783+dannyjameswilliams@users.noreply.github.com> Date: Tue, 1 Oct 2024 15:36:35 +0100 Subject: [PATCH 24/30] Update chunked_pooling/chunking.py according to comment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Michael Günther --- chunked_pooling/chunking.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/chunked_pooling/chunking.py b/chunked_pooling/chunking.py index 09e1ff5..4585aa7 100644 --- a/chunked_pooling/chunking.py +++ b/chunked_pooling/chunking.py @@ -150,8 +150,8 @@ def chunk( tokenizer=tokenizer, ) elif chunking_strategy == "fixed": - if chunk_size < 8: - raise ValueError("Chunk size must be >= 8.") + if chunk_size < 4: + raise ValueError("Chunk size must be >= 4.") return self.chunk_by_tokens(text, chunk_size, tokenizer) elif chunking_strategy == "sentences": return self.chunk_by_sentences(text, n_sentences, tokenizer) From d4f99ce480e831d98f2e6d0318f348fc69387bb6 Mon Sep 17 00:00:00 2001 From: dannyjameswilliams Date: Wed, 2 Oct 2024 09:49:30 +0100 Subject: [PATCH 25/30] updated main experiment file with long late chunking --- chunked_pooling/chunked_eval_tasks.py | 79 ---------- chunked_pooling/mteb_chunked_eval.py | 58 ++----- plot_chunk_size_experiments.py | 88 ----------- plot_macro_chunking_experiments.py | 160 -------------------- run_chunked_eval.py | 26 +++- run_macro_chunking_experiments.py | 209 -------------------------- 6 files changed, 36 insertions(+), 584 deletions(-) delete mode 100644 plot_chunk_size_experiments.py delete mode 100644 plot_macro_chunking_experiments.py delete mode 100644 run_macro_chunking_experiments.py diff --git a/chunked_pooling/chunked_eval_tasks.py b/chunked_pooling/chunked_eval_tasks.py index 45ff1d1..8cd18fb 100644 --- a/chunked_pooling/chunked_eval_tasks.py +++ b/chunked_pooling/chunked_eval_tasks.py @@ -297,85 +297,6 @@ def load_data(self, **kwargs): self.data_loaded = True -class LEMBNarrativeQARetrievalChunked(AbsTaskChunkedRetrieval): - """ - modified from https://github.com/embeddings-benchmark/mteb/blob/main/mteb/tasks/Retrieval/eng/LEMBWikimQARetrieval.py - """ - - _EVAL_SPLIT = "test" - - metadata = TaskMetadata( - name="LEMBNarrativeQARetrievalChunked", - dataset={ - "path": "dwzhu/LongEmbed", - "revision": "10039a580487dacecf79db69166e17ace3ede392", - "name": "LEMBNarrativeQARetrieval", - }, - reference="https://huggingface.co/datasets/dwzhu/LongEmbed", - description=("narrativeqa subset of dwzhu/LongEmbed dataset."), - type="Retrieval", - category="s2p", - modalities=["text"], - eval_splits=[_EVAL_SPLIT], - eval_langs=["eng-Latn"], - main_score="ndcg_at_10", - date=("1950-01-01", "2019-12-31"), - domains=None, - socioeconomic_status=None, - n_samples=None, - avg_character_length=None, - form=None, - text_creation=None, - task_subtypes=["Article retrieval"], - license="not specified", - annotations_creators="derived", - dialect=[], - sample_creation="found", - bibtex_citation=""" - @inproceedings{ho2020constructing, - title={Constructing A Multi-hop QA Dataset for Comprehensive Evaluation of Reasoning Steps}, - author={Ho, Xanh and Nguyen, Anh-Khoa Duong and Sugawara, Saku and Aizawa, Akiko}, - booktitle={Proceedings of the 28th International Conference on Computational Linguistics}, - pages={6609--6625}, - year={2020} - } - """, - descriptive_stats={ - "n_samples": {_EVAL_SPLIT: 500}, - "avg_character_length": { - "test": { - "average_document_length": 326753.53, - "average_query_length": 47.89, - "num_documents": 300, - "num_queries": 300, - "average_relevant_docs_per_query": 1.0, - } - }, - }, - ) - - def load_data(self, **kwargs): - if self.data_loaded: - return - - dataset_dict = {**self.metadata.dataset} - dataset_dict['name'] = 'narrativeqa' - - query_list = datasets.load_dataset(**dataset_dict)["queries"] - queries = {row["qid"]: row["text"] for row in query_list} - - corpus_list = datasets.load_dataset(**dataset_dict)["corpus"] - corpus = {row["doc_id"]: {"text": row["text"]} for row in corpus_list} - - qrels_list = datasets.load_dataset(**dataset_dict)["qrels"] - qrels = {row["qid"]: {row["doc_id"]: 1} for row in qrels_list} - - self.corpus = {self._EVAL_SPLIT: corpus} - self.queries = {self._EVAL_SPLIT: queries} - self.relevant_docs = {self._EVAL_SPLIT: qrels} - - self.data_loaded = True - class LEMBSummScreenFDRetrievalChunked(AbsTaskChunkedRetrieval): """ modified from https://github.com/embeddings-benchmark/mteb/blob/main/mteb/tasks/Retrieval/eng/LEMBWikimQARetrieval.py diff --git a/chunked_pooling/mteb_chunked_eval.py b/chunked_pooling/mteb_chunked_eval.py index df1b28c..827fc1c 100644 --- a/chunked_pooling/mteb_chunked_eval.py +++ b/chunked_pooling/mteb_chunked_eval.py @@ -27,9 +27,8 @@ def __init__( model_has_instructions: bool = False, embedding_model_name: Optional[str] = None, # for semantic chunking truncate_max_length: Optional[int] = 8192, - soft_boundary_embed_size: Optional[int] = 0, - soft_boundary_overlap_size: Optional[int] = 512, - hard_boundary_embed_size: Optional[int] = 0, + long_late_chunking_embed_size: Optional[int] = 0, + long_late_chunking_overlap_size: Optional[int] = 512, **kwargs, ): super().__init__(**kwargs) @@ -54,12 +53,8 @@ def __init__( } self.truncate_max_length = truncate_max_length - if soft_boundary_embed_size > 0 and hard_boundary_embed_size > 0: - raise ValueError('Cannot use both soft and hard boundaries') - - self.soft_boundary_embed_size = soft_boundary_embed_size - self.soft_boundary_overlap_size = soft_boundary_overlap_size - self.hard_boundary_embed_size = hard_boundary_embed_size + self.long_late_chunking_embed_size = long_late_chunking_embed_size + self.long_late_chunking_overlap_size = long_late_chunking_overlap_size def load_data(self, **kwargs): self.retrieval_task.load_data(**kwargs) @@ -124,15 +119,15 @@ def _truncate_documents(self, corpus): v['text'] = v['text'][: last_token_span[1]] return corpus - def _embed_with_soft_boundary(self, model, model_inputs): + def _embed_with_overlap(self, model, model_inputs): len_tokens = len(model_inputs["input_ids"][0]) - if len_tokens > self.soft_boundary_embed_size: + if len_tokens > self.long_late_chunking_embed_size: indices = [] - for i in range(0, len_tokens, self.soft_boundary_embed_size - self.soft_boundary_overlap_size): + for i in range(0, len_tokens, self.long_late_chunking_embed_size - self.long_late_chunking_overlap_size): start = i - end = min(i + self.soft_boundary_embed_size, len_tokens) + end = min(i + self.long_late_chunking_embed_size, len_tokens) indices.append((start, end)) else: indices = [(0, len_tokens)] @@ -146,38 +141,12 @@ def _embed_with_soft_boundary(self, model, model_inputs): model_output = model(**batch_inputs) if start > 0: - outputs.append(model_output[0][:, self.soft_boundary_overlap_size:]) + outputs.append(model_output[0][:, self.long_late_chunking_overlap_size:]) else: outputs.append(model_output[0]) return torch.cat(outputs, dim=1).to(model.device) - def _embed_with_hard_boundary(self, model, model_inputs): - - len_tokens = len(model_inputs["input_ids"][0]) - - if len_tokens > self.hard_boundary_embed_size: - indices = [] - for i in range(0, len_tokens, self.hard_boundary_embed_size): - start = i - end = min(i + self.hard_boundary_embed_size, len_tokens) - indices.append((start, end)) - else: - indices = [(0, len_tokens)] - - outputs = [] - for start, end in indices: - - batch_inputs = {k: v[:, start:end] for k, v in model_inputs.items()} - - with torch.no_grad(): - model_output = model(**batch_inputs) - - outputs.append(model_output[0]) - - return torch.cat(outputs, dim=1).to(model.device) - - def _evaluate_monolingual( self, model, @@ -253,13 +222,8 @@ def _evaluate_monolingual( k: v.to(model.device) for k, v in model_inputs.items() } - if self.soft_boundary_embed_size > 0: - model_outputs = self._embed_with_soft_boundary(model, model_inputs) - output_embs = chunked_pooling( - [model_outputs], annotations, max_length=None - ) - elif self.hard_boundary_embed_size > 0: - model_outputs = self._embed_with_hard_boundary(model, model_inputs) + if self.long_late_chunking_embed_size > 0: + model_outputs = self._embed_with_overlap(model, model_inputs) output_embs = chunked_pooling( [model_outputs], annotations, max_length=None ) diff --git a/plot_chunk_size_experiments.py b/plot_chunk_size_experiments.py deleted file mode 100644 index 05ce4e3..0000000 --- a/plot_chunk_size_experiments.py +++ /dev/null @@ -1,88 +0,0 @@ -import numpy as np -import matplotlib.pyplot as plt -import json -import os - -if __name__ == "__main__": - - lc_dir = "results-chunked-pooling" - nc_dir = "results-normal-pooling" - - # == Load results - - chunk_sizes = [8, 16, 32, 64, 128, 256, 512, 1024] - task_names = ["SummScreenFD", "WikimQA"] # "QMSum", - - lc_ndcgs = np.empty((len(task_names), len(chunk_sizes))) - nc_ndcgs = np.empty((len(task_names), len(chunk_sizes))) - - lc_maps = np.empty((len(task_names), len(chunk_sizes))) - nc_maps = np.empty((len(task_names), len(chunk_sizes))) - - for task_i, task_name in enumerate(task_names): - - task_name = f"LEMB{task_name}RetrievalChunked" - - for chunk_i, chunk_size in enumerate(chunk_sizes): - - lc_dir_chunk_i = os.path.join(lc_dir, f"chunk_size_{chunk_size}", "no_model_name_available", "no_revision_available", f"{task_name}.json") - with open(lc_dir_chunk_i, "r") as f: - results = json.load(f) - - lc_ndcgs[task_i, chunk_i] = results["scores"]["test"][0]["ndcg_at_10"] - lc_maps[task_i, chunk_i] = results["scores"]["test"][0]["map_at_10"] - - nc_dir_chunk_i = os.path.join(nc_dir, f"chunk_size_{chunk_size}", "no_model_name_available", "no_revision_available", f"{task_name}.json") - with open(nc_dir_chunk_i, "r") as f: - results = json.load(f) - - nc_ndcgs[task_i, chunk_i] = results["scores"]["test"][0]["ndcg_at_10"] - nc_maps[task_i, chunk_i] = results["scores"]["test"][0]["map_at_10"] - - # == Plot - plt.style.use('seaborn-v0_8-whitegrid') - plt.rcParams.update({'font.size': 14, 'font.family': 'serif'}) - - # -- NDCG - fig, ax = plt.subplots(len(task_names), 1, figsize=(12, 3*len(task_names)), sharex=True) - - for task_i in range(len(task_names)): - ax[task_i].plot(chunk_sizes, lc_ndcgs[task_i], label="Late Chunking", color="blue", marker='o', linestyle='-', linewidth=2, markersize=6) - ax[task_i].plot(chunk_sizes, nc_ndcgs[task_i], label="Naive Chunking", color="red", marker='o', linestyle='-', linewidth=2, markersize=6) - ax[task_i].set_title(f"Task: {task_names[task_i]}", fontsize=16) - ax[task_i].set_ylabel("nCDG@10", fontsize=14) - ax[task_i].legend(loc='best', fontsize=12, frameon=True) - - ax[task_i].set_xlabel("Chunk Size", fontsize=14) - - # log scale - ax[task_i].set_xscale('log') - - # set specific ticks - ax[task_i].set_xticks([8, 16, 32, 64, 128, 256, 512, 1024]) - ax[task_i].set_xticklabels([8, 16, 32, 64, 128, 256, 512, 1024]) - fig.suptitle("nCDG@10 for Different Chunk Sizes", fontsize=18) - fig.tight_layout(rect=[0, 0, 1, 0.96]) - plt.show() - - # -- mAP - fig, ax = plt.subplots(len(task_names), 1, figsize=(12, 3*len(task_names)), sharex=True) - - for task_i in range(len(task_names)): - ax[task_i].plot(chunk_sizes, lc_maps[task_i], label="Late Chunking", color="blue", marker='o', linestyle='-', linewidth=2, markersize=6) - ax[task_i].plot(chunk_sizes, nc_maps[task_i], label="Naive Chunking", color="red", marker='o', linestyle='-', linewidth=2, markersize=6) - ax[task_i].set_title(f"Task: {task_names[task_i]}", fontsize=16) - ax[task_i].set_ylabel("mAP@10", fontsize=14) - ax[task_i].legend(loc='best', fontsize=12, frameon=True) - - ax[task_i].set_xlabel("Chunk Size", fontsize=14) - - # log scale - ax[task_i].set_xscale('log') - - # set specific ticks - ax[task_i].set_xticks([8, 16, 32, 64, 128, 256, 512, 1024]) - ax[task_i].set_xticklabels([8, 16, 32, 64, 128, 256, 512, 1024]) - fig.suptitle("mAP@10 for Different Chunk Sizes", fontsize=18) - fig.tight_layout(rect=[0, 0, 1, 0.96]) - plt.show() \ No newline at end of file diff --git a/plot_macro_chunking_experiments.py b/plot_macro_chunking_experiments.py deleted file mode 100644 index ce3e162..0000000 --- a/plot_macro_chunking_experiments.py +++ /dev/null @@ -1,160 +0,0 @@ -import numpy as np -import matplotlib.pyplot as plt -import json -import os - -if __name__ == "__main__": - - task_name = "LEMBWikimQARetrievalChunked" - soft_boundary_dir = "results-soft-boundary" - hard_boundary_dir = "results-hard-boundary" - truncated_boundary_dir = "results-truncation" - - # == Load results - - embed_sizes = [1024, 2048, 4096, 8192] - overlap_sizes = [32, 64, 128, 256, 512] - task_names = ["QMSum", "SummScreenFD", "WikimQA"] - - soft_boundary_ndcgs = np.empty((len(task_names), len(embed_sizes), len(overlap_sizes))) - hard_boundary_ndcgs = np.empty((len(task_names), len(embed_sizes))) - - soft_boundary_maps = np.empty((len(task_names), len(embed_sizes), len(overlap_sizes))) - hard_boundary_maps = np.empty((len(task_names), len(embed_sizes))) - - for task_i, task_name in enumerate(task_names): - - task_name = f"LEMB{task_name}RetrievalChunked" - - for embed_i, embed_size in enumerate(embed_sizes): - - for overlap_i, overlap_size in enumerate(overlap_sizes): - - soft_dir = os.path.join(soft_boundary_dir, f"embed_size_{embed_size}", f"overlap_{overlap_size}", "no_model_name_available", "no_revision_available", f"{task_name}.json") - with open(soft_dir, "r") as f: - results = json.load(f) - - soft_boundary_ndcgs[task_i, embed_i, overlap_i] = results["scores"]["test"][0]["ndcg_at_10"] - soft_boundary_maps[task_i, embed_i, overlap_i] = results["scores"]["test"][0]["map_at_10"] - - soft_dir = os.path.join(hard_boundary_dir, f"embed_size_{embed_size}", "no_model_name_available", "no_revision_available", f"{task_name}.json") - with open(soft_dir, "r") as f: - results = json.load(f) - - hard_boundary_ndcgs[task_i, embed_i] = results["scores"]["test"][0]["ndcg_at_10"] - hard_boundary_maps[task_i, embed_i] = results["scores"]["test"][0]["map_at_10"] - - # truncation_dir = os.path.join(truncated_boundary_dir, "no_model_name_available", "no_revision_available", f"{task_name}.json") - # with open(truncation_dir, "r") as f: - # results = json.load(f) - - # truncated_boundary_ndcg = results["scores"]["test"][0]["ndcg_at_10"] - # truncated_boundary_map = results["scores"]["test"][0]["map_at_10"] - - - # == Plot (by task separately) - - # NDCG - plt.style.use('seaborn-v0_8-whitegrid') - plt.rcParams.update({'font.size': 14, 'font.family': 'serif'}) - - fig, ax = plt.subplots(len(task_names), 1, figsize=(12, 3*len(task_names)), sharex=True) - - # colors = plt.cm.get_cmap('plasma', len(embed_sizes))(np.linspace(0, 1, len(embed_sizes), endpoint=False)) - colors = ["red", "blue", "green", "orange"] - - for task_i in range(len(task_names)): - for embed_i, embed_size in enumerate(embed_sizes[:-1]): - - ax[task_i].plot([o - embed_i*5 for o in overlap_sizes], soft_boundary_ndcgs[task_i, embed_i], label=f"Soft Boundary ({embed_size})", color=colors[embed_i], marker='o', linestyle='-', linewidth=2, markersize=6) - ax[task_i].axhline(hard_boundary_ndcgs[task_i, embed_i], linestyle="--", label=f"Hard Boundary ({embed_size})", linewidth=2, color=colors[embed_i]) - - ax[task_i].set_title(f"Embed Size: {embed_size}", fontsize=16) - ax[task_i].set_ylabel("nCDG@10", fontsize=14) - ax[task_i].legend(loc='best', fontsize=12) - - ax[task_i].set_title("Task: " + task_names[task_i], fontsize=16) - - # ax.axhline(truncated_boundary_ndcg, label=f"Truncated Boundary (at 8192)", color="k", linestyle='-.', linewidth=2) - ax[task_i].set_xlabel("Overlap Size", fontsize=14) - - fig.suptitle("NDCG@10 for Different Embed Sizes and Overlap Sizes", fontsize=18) - fig.tight_layout(rect=[0, 0, 1, 0.96]) - - # MAP - plt.style.use('seaborn-v0_8-whitegrid') - plt.rcParams.update({'font.size': 14, 'font.family': 'serif'}) - - fig, ax = plt.subplots(len(task_names), 1, figsize=(12, 3*len(task_names)), sharex=True) - - colors = ["red", "blue", "green", "orange"] - - for task_i in range(len(task_names)): - for embed_i, embed_size in enumerate(embed_sizes[:-1]): - - ax[task_i].plot([o - embed_i*5 for o in overlap_sizes], soft_boundary_maps[task_i, embed_i], label=f"Soft Boundary ({embed_size})", color=colors[embed_i], marker='o', linestyle='-', linewidth=2, markersize=6) - ax[task_i].axhline(hard_boundary_maps[task_i, embed_i], linestyle="--", label=f"Hard Boundary ({embed_size})", linewidth=2, color=colors[embed_i]) - - ax[task_i].set_title(f"Embed Size: {embed_size}", fontsize=16) - ax[task_i].set_ylabel("nCDG@10", fontsize=14) - ax[task_i].legend(loc='best', fontsize=12) - - ax[task_i].set_title("Task: " + task_names[task_i], fontsize=16) - - # ax.axhline(truncated_boundary_map, label=f"Truncated Boundary (at 8192)", color="k", linestyle='-.', linewidth=2) - ax[task_i].set_xlabel("Overlap Size", fontsize=14) - - fig.suptitle("mAP@10 for Different Embed Sizes and Overlap Sizes", fontsize=18) - fig.tight_layout(rect=[0, 0, 1, 0.96]) - plt.show() - - # == Plot difference in ncdg and map - - plt.style.use('seaborn-v0_8-whitegrid') - plt.rcParams.update({'font.size': 14, 'font.family': 'serif'}) - - fig, ax = plt.subplots(2, 1, figsize=(12, 10)) - - colors = ["red", "blue", "green", "orange"] - markers = ["o", "s", "D", "X"] - - for task_i in range(len(task_names)): - for embed_i, embed_size in enumerate(embed_sizes): - diff_ndcgs = soft_boundary_ndcgs[task_i, embed_i] - hard_boundary_ndcgs[task_i, embed_i] - ax[0].plot([o - embed_i*5 for o in overlap_sizes], diff_ndcgs, color=colors[embed_i], marker=markers[task_i], linestyle='-', linewidth=1.25, markersize=6) - - for embed_i, embed_size in enumerate(embed_sizes): - ax[0].plot([], [], label=f"Num Tokens: {embed_size}", color=colors[embed_i], linestyle='-', linewidth=2) - - for task_i in range(len(task_names)): - ax[0].scatter([], [], label=f"Task: {task_names[task_i]}", marker=markers[task_i], color = "k") - - ax[0].axhline(0, color="black", linestyle="--", linewidth=2) - ax[0].set_title("Difference in nDCG@10 Between Soft and Hard Boundaries Across Tasks", fontsize=16) - ax[0].set_ylabel("Difference in nDCG@10", fontsize=14) - ax[0].set_xlabel("Overlap Size", fontsize=14) - - for task_i in range(len(task_names)): - for embed_i, embed_size in enumerate(embed_sizes): - diff_maps = soft_boundary_maps[task_i, embed_i] - hard_boundary_maps[task_i, embed_i] - ax[1].plot([o - embed_i*5 for o in overlap_sizes], diff_maps, color=colors[embed_i], marker=markers[task_i], linestyle='-', linewidth=1.25, markersize=6) - - for embed_i, embed_size in enumerate(embed_sizes): - ax[1].plot([], [], label=f"Num Tokens: {embed_size}", color=colors[embed_i], linestyle='-', linewidth=2) - - for task_i in range(len(task_names)): - ax[1].scatter([], [], label=f"Task: {task_names[task_i]}", marker=markers[task_i], color = "k") - - ax[1].axhline(0, color="black", linestyle="--", linewidth=2) - ax[1].set_title("Difference in mAP@10 Between Soft and Hard Boundaries Across Tasks", fontsize=16) - ax[1].set_ylabel("Difference in mAP@10", fontsize=14) - ax[1].set_xlabel("Overlap Size", fontsize=14) - - - ax[-1].legend(loc='best', fontsize=12, frameon=True) - - fig.tight_layout(rect=[0, 0, 1, 0.96]) - plt.show() - - - \ No newline at end of file diff --git a/run_chunked_eval.py b/run_chunked_eval.py index 88494bd..5f9dd33 100644 --- a/run_chunked_eval.py +++ b/run_chunked_eval.py @@ -10,6 +10,8 @@ DEFAULT_CHUNK_SIZE = 256 DEFAULT_N_SENTENCES = 5 BATCH_SIZE = 1 +DEFAULT_LONG_LATE_CHUNKING_OVERLAP_SIZE = 256 +DEFAULT_LONG_LATE_CHUNKING_EMBED_SIZE = 8192 # set to 0 to disable long late chunking @click.command() @@ -39,7 +41,7 @@ '--truncate-max-length', default=None, type=int, - help='Maximum number of tokens; By default, no truncation is done.', + help='Maximum number of tokens; By default, no truncation is done. If defined, Long Late Chunking algorithm is disabled.', ) @click.option( '--chunk-size', @@ -53,6 +55,18 @@ type=int, help='Number of sentences per chunk for sentence strategy.', ) +@click.option( + '--long-late-chunking-embed-size', + default=DEFAULT_LONG_LATE_CHUNKING_EMBED_SIZE, + type=int, + help='Token length of the embeddings that come before/after soft boundaries (i.e. overlapping embeddings). Above zero, overlap is used between neighbouring embeddings.', +) +@click.option( + '--long-late-chunking-overlap-size', + default=DEFAULT_LONG_LATE_CHUNKING_OVERLAP_SIZE, + type=int, + help='Number of tokens per chunk for fixed strategy.', +) def main( model_name, strategy, @@ -62,11 +76,19 @@ def main( truncate_max_length, chunk_size, n_sentences, + long_late_chunking_embed_size, + long_late_chunking_overlap_size ): try: task_cls = globals()[task_name] except: raise ValueError(f'Unknown task name: {task_name}') + + if truncate_max_length is not None and (long_late_chunking_embed_size > 0): + long_late_chunking_embed_size = 0 + print(f'Long Late Chunking algorithm will be disabled because truncate max length is defined, hence documents are truncated.') + + assert (long_late_chunking_embed_size > 0 or truncate_max_length is not None), 'Define either long late chunking or truncation to handle documents.' model, has_instructions = load_model(model_name) @@ -92,6 +114,8 @@ def main( tokenizer=tokenizer, prune_size=None, truncate_max_length=truncate_max_length, + long_late_chunking_embed_size=long_late_chunking_embed_size, + long_late_chunking_overlap_size=long_late_chunking_overlap_size, **chunking_args, ) ] diff --git a/run_macro_chunking_experiments.py b/run_macro_chunking_experiments.py deleted file mode 100644 index 31ba6cf..0000000 --- a/run_macro_chunking_experiments.py +++ /dev/null @@ -1,209 +0,0 @@ -import click -import torch.cuda -from mteb import MTEB -from transformers import AutoModel, AutoTokenizer - -from chunked_pooling.chunked_eval_tasks import ( - LEMBWikimQARetrievalChunked, - LEMBNeedleRetrievalChunked, - LEMBNarrativeQARetrievalChunked, - LEMBQMSumRetrievalChunked, - LEMBSummScreenFDRetrievalChunked -) -from chunked_pooling.wrappers import load_model - -DEFAULT_CHUNKING_STRATEGY = 'fixed' -DEFAULT_CHUNK_SIZE = 256 -DEFAULT_N_SENTENCES = 5 -BATCH_SIZE = 1 -DEFAULT_SOFT_BOUNDARY_EMBED_SIZE = 8192 -DEFAULT_HARD_BOUNDARY_EMBED_SIZE = 0 - - - -@click.command() -@click.option( - '--model-name', - default='jinaai/jina-embeddings-v2-small-en', - help='The name of the model to use.', -) -@click.option( - '--strategy', - default=DEFAULT_CHUNKING_STRATEGY, - help='The chunking strategy to be applied.', -) -@click.option( - '--task-name', default='LEMBWikimQARetrievalChunked', help='The evaluation task to perform.' -) -@click.option( - '--eval-split', default='test', help='The name of the evaluation split in the task.' -) -@click.option( - '--chunking-model', - default=None, - required=False, - help='The name of the model used for semantic chunking.', -) -@click.option( - '--truncate-max-length', - default=8192, - type=int, - help='Maximum number of tokens; By default, no truncation is done.', -) -@click.option( - '--chunk-size', - default=DEFAULT_CHUNK_SIZE, - type=int, - help='Number of tokens per chunk for fixed strategy.', -) -@click.option( - '--n-sentences', - default=DEFAULT_N_SENTENCES, - type=int, - help='Number of sentences per chunk for sentence strategy.', -) -@click.option( - '--soft-boundary-embed-size', - default=DEFAULT_SOFT_BOUNDARY_EMBED_SIZE, - type=int, - help='Token length of the embeddings that come before/after soft boundaries (i.e. overlapping embeddings). Above zero, soft boundaries are used.', -) -@click.option( - '--hard-boundary-embed-size', - default=DEFAULT_HARD_BOUNDARY_EMBED_SIZE, - type=int, - help='Token length of the embeddings that come before/after hard boundaries. Above zero, hard boundaries are used.', -) - -def main( - model_name, - strategy, - task_name, - eval_split, - chunking_model, - truncate_max_length, - chunk_size, - n_sentences, - soft_boundary_embed_size, - hard_boundary_embed_size, -): - try: - task_cls = globals()[task_name] - except: - raise ValueError(f'Unknown task name: {task_name}') - - model, has_instructions = load_model(model_name) - - tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True) - - chunking_args = { - 'chunk_size': chunk_size, - 'n_sentences': n_sentences, - 'chunking_strategy': strategy, - 'model_has_instructions': has_instructions, - 'embedding_model_name': chunking_model if chunking_model else model_name, - } - - if torch.cuda.is_available(): - model = model.cuda() - - model.eval() - - overlap_sizes = [32, 64, 128, 256, 512] - for overlap_size in overlap_sizes: - - - # Evaluate with soft boundary - tasks = [ - task_cls( - chunked_pooling_enabled=True, - tokenizer=tokenizer, - prune_size=None, - truncate_max_length=0, - soft_boundary_embed_size=soft_boundary_embed_size, - soft_boundary_overlap_size=overlap_size, - hard_boundary_embed_size=0, - **chunking_args, - ) - ] - - evaluation = MTEB( - tasks=tasks, - chunked_pooling_enabled=True, - tokenizer=tokenizer, - prune_size=None, - **chunking_args, - ) - evaluation.run( - model, - output_folder=f'results-soft-boundary/embed_size_{soft_boundary_embed_size}/overlap_{overlap_size}', - eval_splits=[eval_split], - overwrite_results=True, - batch_size=BATCH_SIZE, - encode_kwargs={'batch_size': BATCH_SIZE}, - ) - - - # Evaluate with hard boundary - tasks = [ - task_cls( - chunked_pooling_enabled=True, - tokenizer=tokenizer, - prune_size=None, - truncate_max_length=0, - soft_boundary_embed_size=0, - hard_boundary_embed_size=hard_boundary_embed_size, - **chunking_args, - ) - ] - - evaluation = MTEB( - tasks=tasks, - chunked_pooling_enabled=True, - tokenizer=tokenizer, - prune_size=None, - **chunking_args, - ) - - evaluation.run( - model, - output_folder=f'results-hard-boundary/embed_size_{hard_boundary_embed_size}', - eval_splits=[eval_split], - overwrite_results=True, - batch_size=BATCH_SIZE, - encode_kwargs={'batch_size': BATCH_SIZE}, - ) - - # Evaluate with no boundary (truncation) - tasks = [ - task_cls( - chunked_pooling_enabled=True, - tokenizer=tokenizer, - prune_size=None, - truncate_max_length=truncate_max_length, - soft_boundary_embed_size=0, - hard_boundary_embed_size=0, - **chunking_args, - ) - ] - - evaluation = MTEB( - tasks=tasks, - chunked_pooling_enabled=True, - tokenizer=tokenizer, - prune_size=None, - **chunking_args, - ) - evaluation.run( - model, - output_folder=f'results-truncation/embed_size_{truncate_max_length}', - eval_splits=[eval_split], - overwrite_results=True, - batch_size=BATCH_SIZE, - encode_kwargs={'batch_size': BATCH_SIZE}, - ) - - x=1 - -if __name__ == '__main__': - main() From 936ae517df4ba96ea2e1c4d42aa369d2a5c543b9 Mon Sep 17 00:00:00 2001 From: dannyjameswilliams Date: Wed, 2 Oct 2024 09:51:30 +0100 Subject: [PATCH 26/30] remove redundant macro chunking file --- run_chunked_eval_with_macro_chunks.py | 170 -------------------------- 1 file changed, 170 deletions(-) delete mode 100644 run_chunked_eval_with_macro_chunks.py diff --git a/run_chunked_eval_with_macro_chunks.py b/run_chunked_eval_with_macro_chunks.py deleted file mode 100644 index f990f6e..0000000 --- a/run_chunked_eval_with_macro_chunks.py +++ /dev/null @@ -1,170 +0,0 @@ -import click -import torch.cuda -from mteb import MTEB -from transformers import AutoModel, AutoTokenizer - -from chunked_pooling.chunked_eval_tasks import * -from chunked_pooling.wrappers import load_model - -DEFAULT_CHUNKING_STRATEGY = 'fixed' -DEFAULT_N_SENTENCES = 5 -BATCH_SIZE = 1 -DEFAULT_OVERLAP_SIZE = 256 -DEFAULT_SOFT_BOUNDARY_EMBED_SIZE = 8192 # set to 0 to disable soft boundaries -DEFAULT_HARD_BOUNDARY_EMBED_SIZE = 0 # set to 0 to disable hard boundaries - - -@click.command() -@click.option( - '--model-name', - default='jinaai/jina-embeddings-v2-small-en', - help='The name of the model to use.', -) -@click.option( - '--strategy', - default=DEFAULT_CHUNKING_STRATEGY, - help='The chunking strategy to be applied.', -) -@click.option( - '--task-name', default='LEMBWikimQARetrievalChunked', help='The evaluation task to perform.' -) -@click.option( - '--eval-split', default='test', help='The name of the evaluation split in the task.' -) -@click.option( - '--chunking-model', - default=None, - required=False, - help='The name of the model used for semantic chunking.', -) -@click.option( - '--truncate-max-length', - default=None, - type=int, - help='Maximum number of tokens; By default, no truncation is done.', -) -@click.option( - '--n-sentences', - default=DEFAULT_N_SENTENCES, - type=int, - help='Number of sentences per chunk for sentence strategy.', -) -@click.option( - '--soft-boundary-embed-size', - default=DEFAULT_SOFT_BOUNDARY_EMBED_SIZE, - type=int, - help='Token length of the embeddings that come before/after soft boundaries (i.e. overlapping embeddings). Above zero, soft boundaries are used.', -) -@click.option( - '--hard-boundary-embed-size', - default=DEFAULT_HARD_BOUNDARY_EMBED_SIZE, - type=int, - help='Token length of the embeddings that come before/after hard boundaries. Above zero, hard boundaries are used.', -) -@click.option( - '--overlap-size', - default=DEFAULT_OVERLAP_SIZE, - type=int, - help='Number of tokens per chunk for fixed strategy.', -) - -def main( - model_name, - strategy, - task_name, - eval_split, - chunking_model, - truncate_max_length, - n_sentences, - soft_boundary_embed_size, - hard_boundary_embed_size, - overlap_size, -): - try: - task_cls = globals()[task_name] - except: - raise ValueError(f'Unknown task name: {task_name}') - - model, has_instructions = load_model(model_name) - - tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True) - - if torch.cuda.is_available(): - model = model.cuda() - - model.eval() - - chunk_sizes = [8, 16, 32, 64, 128, 256, 512, 1024] - for chunk_size in chunk_sizes: - - print(f'\n\n\n\nEvaluating chunk size: {chunk_size} ({chunk_sizes.index(chunk_size) + 1}/{len(chunk_sizes)}) \n\n\n\n') - - chunking_args = { - 'chunk_size': chunk_size, - 'n_sentences': n_sentences, - 'chunking_strategy': strategy, - 'model_has_instructions': has_instructions, - 'embedding_model_name': chunking_model if chunking_model else model_name, - } - - # == Late Chunking == - # tasks = [ - # task_cls( - # chunked_pooling_enabled=True, - # tokenizer=tokenizer, - # prune_size=None, - # truncate_max_length=truncate_max_length, - # soft_boundary_embed_size=soft_boundary_embed_size, - # soft_boundary_overlap_size=overlap_size, - # hard_boundary_embed_size=hard_boundary_embed_size, - # **chunking_args, - # ) - # ] - - # evaluation = MTEB( - # tasks=tasks, - # chunked_pooling_enabled=True, - # tokenizer=tokenizer, - # prune_size=None, - # **chunking_args, - # ) - # evaluation.run( - # model, - # output_folder=f'results-chunked-pooling/chunk_size_{chunk_size}', - # eval_splits=[eval_split], - # overwrite_results=True, - # batch_size=BATCH_SIZE, - # encode_kwargs={'batch_size': BATCH_SIZE}, - # ) - - # == Naive Chunking == - # naive chunking does not need soft boundaries because chunk size is guaranteed to be <8192 tokens - tasks = [ - task_cls( - chunked_pooling_enabled=False, - tokenizer=tokenizer, - prune_size=None, - truncate_max_length=truncate_max_length, - **chunking_args, - ) - ] - - evaluation = MTEB( - tasks=tasks, - chunked_pooling_enabled=False, - tokenizer=tokenizer, - prune_size=None, - **chunking_args, - ) - evaluation.run( - model, - output_folder=f'results-normal-pooling/chunk_size_{chunk_size}', - eval_splits=[eval_split], - overwrite_results=True, - batch_size=BATCH_SIZE, - encode_kwargs={'batch_size': BATCH_SIZE}, - ) - - -if __name__ == '__main__': - main() From 1ed7fb6dccc92b3d92d421810efec9ebaab03fab Mon Sep 17 00:00:00 2001 From: dannyjameswilliams Date: Wed, 2 Oct 2024 10:00:20 +0100 Subject: [PATCH 27/30] updated default to truncation (8192) --- run_chunked_eval.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/run_chunked_eval.py b/run_chunked_eval.py index 5f9dd33..f4c0e8c 100644 --- a/run_chunked_eval.py +++ b/run_chunked_eval.py @@ -11,7 +11,8 @@ DEFAULT_N_SENTENCES = 5 BATCH_SIZE = 1 DEFAULT_LONG_LATE_CHUNKING_OVERLAP_SIZE = 256 -DEFAULT_LONG_LATE_CHUNKING_EMBED_SIZE = 8192 # set to 0 to disable long late chunking +DEFAULT_LONG_LATE_CHUNKING_EMBED_SIZE = 0 # set to 0 to disable long late chunking +DEFAULT_TRUNCATE_MAX_LENGTH = 8192 @click.command() @@ -39,9 +40,9 @@ ) @click.option( '--truncate-max-length', - default=None, + default=DEFAULT_TRUNCATE_MAX_LENGTH, type=int, - help='Maximum number of tokens; By default, no truncation is done. If defined, Long Late Chunking algorithm is disabled.', + help='Maximum number of tokens; by default, truncation to 8192 tokens. If None, Long Late Chunking algorithm should be enabled.', ) @click.option( '--chunk-size', From abab7fa1e59d3b68e0ed07b63873050a41af3e35 Mon Sep 17 00:00:00 2001 From: dannyjameswilliams Date: Wed, 2 Oct 2024 10:01:37 +0100 Subject: [PATCH 28/30] updated error message/print statement --- run_chunked_eval.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/run_chunked_eval.py b/run_chunked_eval.py index f4c0e8c..adbe7fa 100644 --- a/run_chunked_eval.py +++ b/run_chunked_eval.py @@ -86,11 +86,9 @@ def main( raise ValueError(f'Unknown task name: {task_name}') if truncate_max_length is not None and (long_late_chunking_embed_size > 0): - long_late_chunking_embed_size = 0 - print(f'Long Late Chunking algorithm will be disabled because truncate max length is defined, hence documents are truncated.') + truncate_max_length = None + print(f'Truncation is disabled because Long Late Chunking algorithm is enabled.') - assert (long_late_chunking_embed_size > 0 or truncate_max_length is not None), 'Define either long late chunking or truncation to handle documents.' - model, has_instructions = load_model(model_name) tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True) From b46d469b9644494bb1a1d6978094a1336c1e2cb5 Mon Sep 17 00:00:00 2001 From: dannyjameswilliams Date: Wed, 2 Oct 2024 10:40:48 +0100 Subject: [PATCH 29/30] changed how local llm is loaded --- explanatory_contextual_retrieval.py | 35 ++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/explanatory_contextual_retrieval.py b/explanatory_contextual_retrieval.py index fb6ccc4..7af904e 100644 --- a/explanatory_contextual_retrieval.py +++ b/explanatory_contextual_retrieval.py @@ -9,8 +9,8 @@ # accelerate? from chunked_pooling.wrappers import load_model -from transformers import AutoModel, AutoTokenizer, pipeline - +from transformers import AutoModel, AutoTokenizer, pipeline, AutoModelForCausalLM +import torch import numpy as np import chunked_pooling @@ -40,6 +40,23 @@ def request_anthropic_api(prompt: str): response = requests.post(url, headers=headers, json=data) return response.json()["content"][0]["text"] +def setup_local_llm(llm_name): + + model = AutoModelForCausalLM.from_pretrained(llm_name, trust_remote_code=True) + tokenizer = AutoTokenizer.from_pretrained(llm_name, trust_remote_code=True) + + def llm(prompt): + messages = [{"role": "user", "content": prompt}] + inputs = tokenizer.apply_chat_template(messages, add_generation_prompt=True, return_tensors="pt") + inputs = inputs.to(model.device) + outputs = model.generate(inputs, max_new_tokens=512) + text_output = tokenizer.batch_decode(outputs)[0] + if "<|assistant|>" in text_output: + text_output = text_output.split("<|assistant|>")[1].strip() + return text_output + + return llm + def cosine_similarity(vector1, vector2): vector1_norm = vector1 / np.linalg.norm(vector1) vector2_norm = vector2 / np.linalg.norm(vector2) @@ -94,11 +111,9 @@ def __init__(self, llm_name: str = "meta-llama/Meta-Llama-3.1-8B", chunking_strategy: str = "fixed" ): - # self.llm = pipeline( - # "text-generation", model=llm_name, model_kwargs={"torch_dtype": torch.bfloat16}, device_map="auto", - # max_length = 1000 - # ) - self.llm = request_anthropic_api + + self.llm = setup_local_llm(llm_name) + # self.llm = request_anthropic_api self.prompt = """ @@ -164,15 +179,13 @@ def query(self, query: str): The report emphasized the company's resilience and ability to navigate market challenges, reflecting positively on their financial health and future prospects. """.strip().replace("\n", "") - - # llm_model_name = "microsoft/Phi-3.5-mini-instruct" + llm_model_name = "microsoft/Phi-3.5-mini-instruct" embedding_model_name = "jinaai/jina-embeddings-v2-small-en" embedding_model, has_instructions = load_model(embedding_model_name) embedding_tokenizer = AutoTokenizer.from_pretrained(embedding_model_name, trust_remote_code=True) - - cr = ContextualRetrievalEmbedder(embedding_model, embedding_tokenizer, chunking_strategy="sentences") + cr = ContextualRetrievalEmbedder(embedding_model, embedding_tokenizer, llm_model_name, chunking_strategy="sentences") cr.run(text); cr_cosine_similarities = cr.query("What is ACME Corp's revenue growth for Q2 2023?") From b64c2a6a03252cf866201c72a07c8fe4262bde2c Mon Sep 17 00:00:00 2001 From: dannyjameswilliams Date: Wed, 2 Oct 2024 10:45:06 +0100 Subject: [PATCH 30/30] removed comment on pip and update default model to phi --- explanatory_contextual_retrieval.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/explanatory_contextual_retrieval.py b/explanatory_contextual_retrieval.py index 7af904e..269b518 100644 --- a/explanatory_contextual_retrieval.py +++ b/explanatory_contextual_retrieval.py @@ -4,9 +4,6 @@ # contextual retrieval method. contextual retrieval manually inserts context to each # chunk, i.e. forces context to be around each chunk. so works as a good comparison # to late chunking to see if the similarities are similar (which they appear to be) -# -# pip requirements: -# accelerate? from chunked_pooling.wrappers import load_model from transformers import AutoModel, AutoTokenizer, pipeline, AutoModelForCausalLM @@ -108,7 +105,7 @@ class ContextualRetrievalEmbedder(): def __init__(self, model: AutoModel, tokenizer: AutoTokenizer, - llm_name: str = "meta-llama/Meta-Llama-3.1-8B", + llm_name: str = "microsoft/Phi-3.5-mini-instruct", chunking_strategy: str = "fixed" ):