Automated Leaderboard Update

embeddings-benchmark · Sep 18, 2024 · f2faa72 · f2faa72
1 parent d93f299
commit f2faa72
Show file tree

Hide file tree

Showing 20 changed files with 600 additions and 615 deletions.
diff --git a/all_data_tasks/27/default.jsonl b/all_data_tasks/27/default.jsonl
diff --git a/all_data_tasks/28/default.jsonl b/all_data_tasks/28/default.jsonl
diff --git a/all_data_tasks/31/default.jsonl b/all_data_tasks/31/default.jsonl
diff --git a/all_data_tasks/32/default.jsonl b/all_data_tasks/32/default.jsonl
diff --git a/all_data_tasks/33/default.jsonl b/all_data_tasks/33/default.jsonl
diff --git a/all_data_tasks/34/default.jsonl b/all_data_tasks/34/default.jsonl
diff --git a/all_data_tasks/35/default.jsonl b/all_data_tasks/35/default.jsonl
diff --git a/all_data_tasks/36/default.jsonl b/all_data_tasks/36/default.jsonl
diff --git a/all_data_tasks/37/default.jsonl b/all_data_tasks/37/default.jsonl
diff --git a/boards_data/pl/data_overall/default.jsonl b/boards_data/pl/data_overall/default.jsonl
@@ -35,7 +35,7 @@
 {"index":31,"Rank":46,"Model":"<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https:\/\/huggingface.co\/intfloat\/multilingual-e5-large-instruct\">multilingual-e5-large-instruct<\/a>","Model Size (Million Parameters)":560,"Memory Usage (GB, fp32)":2.09,"Embedding Dimensions":1024,"Max Tokens":514,"Average (26 datasets)":"","Classification Average (7 datasets)":67.02,"Clustering Average (1 datasets)":"","PairClassification Average (4 datasets)":"","Retrieval Average (11 datasets)":"","STS Average (3 datasets)":70.31}
 {"index":35,"Rank":47,"Model":"<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https:\/\/huggingface.co\/izhx\/udever-bloom-1b1\">udever-bloom-1b1<\/a>","Model Size (Million Parameters)":1065,"Memory Usage (GB, fp32)":3.97,"Embedding Dimensions":1536,"Max Tokens":2048,"Average (26 datasets)":"","Classification Average (7 datasets)":"","Clustering Average (1 datasets)":"","PairClassification Average (4 datasets)":"","Retrieval Average (11 datasets)":"","STS Average (3 datasets)":""}
 {"index":36,"Rank":48,"Model":"<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https:\/\/huggingface.co\/izhx\/udever-bloom-560m\">udever-bloom-560m<\/a>","Model Size (Million Parameters)":559,"Memory Usage (GB, fp32)":2.08,"Embedding Dimensions":1024,"Max Tokens":2048,"Average (26 datasets)":"","Classification Average (7 datasets)":"","Clustering Average (1 datasets)":"","PairClassification Average (4 datasets)":"","Retrieval Average (11 datasets)":"","STS Average (3 datasets)":""}
-{"index":37,"Rank":49,"Model":"<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https:\/\/huggingface.co\/jinaai\/jina-embeddings-v3\">jina-embeddings-v3<\/a>","Model Size (Million Parameters)":572,"Memory Usage (GB, fp32)":2.13,"Embedding Dimensions":1024,"Max Tokens":8194,"Average (26 datasets)":"","Classification Average (7 datasets)":70.81,"Clustering Average (1 datasets)":"","PairClassification Average (4 datasets)":"","Retrieval Average (11 datasets)":"","STS Average (3 datasets)":72.77}
+{"index":37,"Rank":49,"Model":"<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https:\/\/huggingface.co\/jinaai\/jina-embeddings-v3\">jina-embeddings-v3<\/a>","Model Size (Million Parameters)":572,"Memory Usage (GB, fp32)":2.13,"Embedding Dimensions":1024,"Max Tokens":8194,"Average (26 datasets)":"","Classification Average (7 datasets)":70.81,"Clustering Average (1 datasets)":43.66,"PairClassification Average (4 datasets)":83.7,"Retrieval Average (11 datasets)":"","STS Average (3 datasets)":72.77}
 {"index":46,"Rank":50,"Model":"<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https:\/\/huggingface.co\/sentence-transformers\/all-MiniLM-L12-v2\">all-MiniLM-L12-v2<\/a>","Model Size (Million Parameters)":33,"Memory Usage (GB, fp32)":0.12,"Embedding Dimensions":384,"Max Tokens":512,"Average (26 datasets)":"","Classification Average (7 datasets)":39.36,"Clustering Average (1 datasets)":"","PairClassification Average (4 datasets)":"","Retrieval Average (11 datasets)":"","STS Average (3 datasets)":51.99}
 {"index":47,"Rank":51,"Model":"<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https:\/\/huggingface.co\/sentence-transformers\/all-MiniLM-L6-v2\">all-MiniLM-L6-v2<\/a>","Model Size (Million Parameters)":23,"Memory Usage (GB, fp32)":0.09,"Embedding Dimensions":384,"Max Tokens":512,"Average (26 datasets)":"","Classification Average (7 datasets)":40.07,"Clustering Average (1 datasets)":"","PairClassification Average (4 datasets)":"","Retrieval Average (11 datasets)":"","STS Average (3 datasets)":52.88}
 {"index":48,"Rank":52,"Model":"<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https:\/\/huggingface.co\/sentence-transformers\/all-mpnet-base-v2\">all-mpnet-base-v2<\/a>","Model Size (Million Parameters)":110,"Memory Usage (GB, fp32)":0.41,"Embedding Dimensions":768,"Max Tokens":514,"Average (26 datasets)":"","Classification Average (7 datasets)":39.84,"Clustering Average (1 datasets)":"","PairClassification Average (4 datasets)":"","Retrieval Average (11 datasets)":"","STS Average (3 datasets)":50.48}

diff --git a/boards_data/pl/data_tasks/Clustering/default.jsonl b/boards_data/pl/data_tasks/Clustering/default.jsonl
diff --git a/boards_data/pl/data_tasks/PairClassification/default.jsonl b/boards_data/pl/data_tasks/PairClassification/default.jsonl
diff --git a/boards_data/ru/data_overall/default.jsonl b/boards_data/ru/data_overall/default.jsonl
diff --git a/boards_data/ru/data_tasks/Classification/default.jsonl b/boards_data/ru/data_tasks/Classification/default.jsonl
diff --git a/boards_data/ru/data_tasks/Clustering/default.jsonl b/boards_data/ru/data_tasks/Clustering/default.jsonl
diff --git a/boards_data/ru/data_tasks/MultilabelClassification/default.jsonl b/boards_data/ru/data_tasks/MultilabelClassification/default.jsonl
diff --git a/boards_data/ru/data_tasks/PairClassification/default.jsonl b/boards_data/ru/data_tasks/PairClassification/default.jsonl
diff --git a/boards_data/ru/data_tasks/Reranking/default.jsonl b/boards_data/ru/data_tasks/Reranking/default.jsonl
diff --git a/boards_data/ru/data_tasks/Retrieval/default.jsonl b/boards_data/ru/data_tasks/Retrieval/default.jsonl
diff --git a/boards_data/ru/data_tasks/STS/default.jsonl b/boards_data/ru/data_tasks/STS/default.jsonl