diff --git a/promptsource/machine_translate.py b/promptsource/machine_translate.py new file mode 100644 index 000000000..f71f2000e --- /dev/null +++ b/promptsource/machine_translate.py @@ -0,0 +1,111 @@ +import os +import re + +from promptsource.templates import Template, TemplateCollection + + +### XNLI + +PROMPTS = [ + "GPT-3 style", + "can we infer", + "justified in saying", + "guaranteed/possible/impossible", + "MNLI crowdsource", +] + +LANGS = [ + "ar", + "es", + "fr", + "hi", + "sw", + "ur", + "vi", + "zh", + + "bg", + "de", + "el", + "ru", + "th", + "tr", +] + +SOURCE_DATASET = TARGET_DATASET = "xnli" +SOURCE_LANG = "en" + + +# Path to key +os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "/Users/niklasmuennighoff/Desktop/gcp_translate_key.json" + + +def translate(target, text): + """Translates text into the target language. + Target must be an ISO 639-1 language code. + See https://g.co/cloud/translate/v2/translate-reference#supported_languages + (pip install --upgrade google-api-python-client) + pip install google-cloud-translate + """ + import six + from google.cloud import translate_v2 as translate + + translate_client = translate.Client() + if isinstance(text, six.binary_type): + text = text.decode("utf-8") + # Text can also be a sequence of strings, in which case this method + # will return a sequence of results for each text. + # By default format_ is html, which would return " instead of " + result = translate_client.translate(text, source_language="en", target_language=target, format_="text") + print("Text: {}".format(result["input"])) + print("Translation: {}".format(result["translatedText"])) + # If not providing source_language + # print("Detected source language: {}".format(result["detectedSourceLanguage"])) + return result["translatedText"] + + +def normalize_string(zh_string, en_string): + """ + This is not specific to zh just to give an example & help Codex understand it :-) + Replaces the content in brackets in zh_string with the content in brackets from en_string. + All else is left the same in zh_string. + Args: + zh_string: {{前提}} 问题：{{假设}} 对、错或两者都不是？ ||| {{ answer_choices[标签] }} + en_string: {{premise}} Question: {{hypothesis}} True, False, or Neither? ||| {{ answer_choices[label] }} + Returns: + zh_string_normalized: {{premise}} 问题：{{hypothesis}} 对、错或两者都不是？ ||| {{ answer_choices[label] }} + """ + zh_string_normalized = zh_string + # Find all the content in brackets in zh_string + # For only double brackets {{(.*?)}}, but we do single brackets as well + zh_bracket_content = re.findall(r"{(.*?)}", zh_string) + # Find all the content in brackets in en_string + # For only double brackets {{(.*?)}}, but we do single brackets as well + en_bracket_content = re.findall(r"{(.*?)}", en_string) + # Replace the content in brackets in zh_string with the content in brackets from en_string + for i in range(len(zh_bracket_content)): + zh_string_normalized = zh_string_normalized.replace(zh_bracket_content[i], en_bracket_content[i]) + return zh_string_normalized + + +template_collection = TemplateCollection() +source_templates = template_collection.get_dataset(SOURCE_DATASET, SOURCE_LANG) + +for lang in LANGS: + target_templates = template_collection.get_dataset(TARGET_DATASET, lang) + for uid, template in source_templates.templates.items(): + if template.name.strip() not in PROMPTS: + continue + print(f"Translating {template.name.strip()} to {lang}") + answer_choices = [] + if template.answer_choices is not None: + choices = template.answer_choices.split("|||") + for c in choices: + answer_choices.append(normalize_string(translate(lang, c.strip()), c.strip())) + or_jinja = template.jinja.strip() + jinja = normalize_string(translate(lang, or_jinja), or_jinja) + template_name = template.name.strip() + f"_{lang}mt" + target_template = Template( + template_name, jinja=jinja, reference="", answer_choices=" ||| ".join(answer_choices) + ) + target_templates.add_template(target_template) diff --git a/promptsource/templates.py b/promptsource/templates.py index 2b9eb341e..ac8628509 100644 --- a/promptsource/templates.py +++ b/promptsource/templates.py @@ -235,6 +235,212 @@ "zu": "Zulu", } +# These are the metrics with which templates can be tagged +METRICS = { + "BLEU", + "ROUGE", + "Squad", + "Trivia QA", + "Accuracy", + "Pearson Correlation", + "Spearman Correlation", + "MultiRC", + "AUC", + "COQA F1", + "Edit Distance", + "Mean Reciprocal Rank", + "Other", +} + +# These are the languages with which templates can be tagged. Keys are ISO 639-1 +# tags, which are the actual tags we use. Values are English names shown in the +# UI for convenience. +LANGUAGES = { + "ab": "Abkhazian", + "aa": "Afar", + "af": "Afrikaans", + "ak": "Akan", + "sq": "Albanian", + "am": "Amharic", + "ar": "Arabic", + "an": "Aragonese", + "hy": "Armenian", + "as": "Assamese", + "av": "Avaric", + "ae": "Avestan", + "ay": "Aymara", + "az": "Azerbaijani", + "bm": "Bambara", + "ba": "Bashkir", + "eu": "Basque", + "be": "Belarusian", + "bn": "Bengali", + "bi": "Bislama", + "bs": "Bosnian", + "br": "Breton", + "bg": "Bulgarian", + "my": "Burmese", + "ca": "Catalan, Valencian", + "ch": "Chamorro", + "ce": "Chechen", + "ny": "Chichewa, Chewa, Nyanja", + "zh": "Chinese", + "cu": "Church Slavic, Old Slavonic, Church Slavonic, Old Bulgarian, Old Church Slavonic", + "cv": "Chuvash", + "kw": "Cornish", + "co": "Corsican", + "cr": "Cree", + "hr": "Croatian", + "cs": "Czech", + "da": "Danish", + "dv": "Divehi, Dhivehi, Maldivian", + "nl": "Dutch, Flemish", + "dz": "Dzongkha", + "en": "English", + "eo": "Esperanto", + "et": "Estonian", + "ee": "Ewe", + "fo": "Faroese", + "fj": "Fijian", + "fi": "Finnish", + "fr": "French", + "fy": "Western Frisian", + "ff": "Fulah", + "gd": "Gaelic, Scottish Gaelic", + "gl": "Galician", + "lg": "Ganda", + "ka": "Georgian", + "de": "German", + "el": "Greek, Modern (1453–)", + "kl": "Kalaallisut, Greenlandic", + "gn": "Guarani", + "gu": "Gujarati", + "ht": "Haitian, Haitian Creole", + "ha": "Hausa", + "he": "Hebrew", + "hz": "Herero", + "hi": "Hindi", + "ho": "Hiri Motu", + "hu": "Hungarian", + "is": "Icelandic", + "io": "Ido", + "ig": "Igbo", + "id": "Indonesian", + "ia": "Interlingua (International Auxiliary Language Association)", + "ie": "Interlingue, Occidental", + "iu": "Inuktitut", + "ik": "Inupiaq", + "ga": "Irish", + "it": "Italian", + "ja": "Japanese", + "jv": "Javanese", + "kn": "Kannada", + "kr": "Kanuri", + "ks": "Kashmiri", + "kk": "Kazakh", + "km": "Central Khmer", + "ki": "Kikuyu, Gikuyu", + "rw": "Kinyarwanda", + "ky": "Kirghiz, Kyrgyz", + "kv": "Komi", + "kg": "Kongo", + "ko": "Korean", + "kj": "Kuanyama, Kwanyama", + "ku": "Kurdish", + "lo": "Lao", + "la": "Latin", + "lv": "Latvian", + "li": "Limburgan, Limburger, Limburgish", + "ln": "Lingala", + "lt": "Lithuanian", + "lu": "Luba-Katanga", + "lb": "Luxembourgish, Letzeburgesch", + "mk": "Macedonian", + "mg": "Malagasy", + "ms": "Malay", + "ml": "Malayalam", + "mt": "Maltese", + "gv": "Manx", + "mi": "Maori", + "mr": "Marathi", + "mh": "Marshallese", + "mn": "Mongolian", + "na": "Nauru", + "nv": "Navajo, Navaho", + "nd": "North Ndebele", + "nr": "South Ndebele", + "ng": "Ndonga", + "ne": "Nepali", + "no": "Norwegian", + "nb": "Norwegian Bokmål", + "nn": "Norwegian Nynorsk", + "ii": "Sichuan Yi, Nuosu", + "oc": "Occitan", + "oj": "Ojibwa", + "or": "Oriya", + "om": "Oromo", + "os": "Ossetian, Ossetic", + "pi": "Pali", + "ps": "Pashto, Pushto", + "fa": "Persian", + "pl": "Polish", + "pt": "Portuguese", + "pa": "Punjabi, Panjabi", + "qu": "Quechua", + "ro": "Romanian, Moldavian, Moldovan", + "rm": "Romansh", + "rn": "Rundi", + "ru": "Russian", + "se": "Northern Sami", + "sm": "Samoan", + "sg": "Sango", + "sa": "Sanskrit", + "sc": "Sardinian", + "sr": "Serbian", + "sn": "Shona", + "sd": "Sindhi", + "si": "Sinhala, Sinhalese", + "sk": "Slovak", + "sl": "Slovenian", + "so": "Somali", + "st": "Southern Sotho", + "es": "Spanish, Castilian", + "su": "Sundanese", + "sw": "Swahili", + "ss": "Swati", + "sv": "Swedish", + "tl": "Tagalog", + "ty": "Tahitian", + "tg": "Tajik", + "ta": "Tamil", + "tt": "Tatar", + "te": "Telugu", + "th": "Thai", + "bo": "Tibetan", + "ti": "Tigrinya", + "to": "Tonga (Tonga Islands)", + "ts": "Tsonga", + "tn": "Tswana", + "tr": "Turkish", + "tk": "Turkmen", + "tw": "Twi", + "ug": "Uighur, Uyghur", + "uk": "Ukrainian", + "ur": "Urdu", + "uz": "Uzbek", + "ve": "Venda", + "vi": "Vietnamese", + "vo": "Volapük", + "wa": "Walloon", + "cy": "Welsh", + "wo": "Wolof", + "xh": "Xhosa", + "yi": "Yiddish", + "yo": "Yoruba", + "za": "Zhuang, Chuang", + "zu": "Zulu", +} + def highlight(input): return "" + input + "" @@ -617,7 +823,7 @@ def write_to_file(self) -> None: # We only create the folder if a template is written if not os.path.exists(self.folder_path): os.makedirs(self.folder_path) - yaml.dump(self.format_for_dump(), open(self.yaml_path, "w")) + yaml.dump(self.format_for_dump(), open(self.yaml_path, "w"),allow_unicode=True) def add_template(self, template: "Template") -> None: """ diff --git a/promptsource/templates/xnli/ar/templates.yaml b/promptsource/templates/xnli/ar/templates.yaml new file mode 100644 index 000000000..f9c03f918 --- /dev/null +++ b/promptsource/templates/xnli/ar/templates.yaml @@ -0,0 +1,194 @@ +dataset: xnli +subset: ar +templates: + 16836edd-898d-4508-92c0-e683f34aa41e: !Template + answer_choices: نعم ||| ربما ||| لا + id: 16836edd-898d-4508-92c0-e683f34aa41e + jinja: " {{premise}}\n بناء على النص السابق هل يصح القول بأن \"{{hypothesis}}\"\ + \ فرضية صحيحة؟ أجب بنعم أو لا أو ربما. ||| {{ answer_choices[label] }}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - ar + metrics: + - Accuracy + original_task: true + name: based on the previous passage_arht + reference: '' + 45e348b8-1dc4-485c-a182-6277069fe5a7: !Template + answer_choices: صحيحة ||| لا يمكن الجزم ||| خاطئة + id: 45e348b8-1dc4-485c-a182-6277069fe5a7 + jinja: '{{premise}} بناء على المعلومات السابقة, هل الدعوى التالية: "{{hypothesis}}" + {{"صحيحة"}} أو {{"خاطئة"}} أو {{"لا يمكن الجزم"}}؟ ||| {{ answer_choices[label] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - ar + metrics: + - Accuracy + original_task: true + name: claim true/false/inconclusive_arht + reference: Sanh et al. 2021 + 70af8d9d-cfab-45a4-8536-2ae00444392e: !Template + answer_choices: نعم ||| يمكن ||| رقم + id: 70af8d9d-cfab-45a4-8536-2ae00444392e + jinja: افترض {{premise}} هل يمكننا استنتاج أن "{{hypothesis}}"؟ نعم + ، لا ، أو ربما؟ ||| {{ answer_choices[label] }} + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: can we infer_armt + reference: '' + 8b75ff67-bb66-413b-a33d-325707b035d7: !Template + answer_choices: نعم ||| ربما ||| لا + id: 8b75ff67-bb66-413b-a33d-325707b035d7 + jinja: 'اذا كان لدينا المقدمة التالية {{premise}} هل يمكن أن نتوصل إلى الفرضية + التالية "{{hypothesis}}"؟ أجب بـ نعم أو لا أو ربما؟ ||| {{ answer_choices[label] + }} ' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - ar + metrics: + - Accuracy + original_task: true + name: can we infer_arht + reference: Webson & Pavlick 2021 + 8bda8e36-c881-4c9a-a3a9-eec68388a6f6: !Template + answer_choices: نعم ||| ربما ||| لا + id: 8bda8e36-c881-4c9a-a3a9-eec68388a6f6 + jinja: '{{premise}} هل الفرضية التالية صحيحة "{{hypothesis}}" بناء على ما سبق؟ + أجب بنعم أو ﻻ أو ربما ؟ ||| {{ answer_choices[label] }} ' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - ar + metrics: + - Accuracy + original_task: true + name: justified in saying_arht + reference: Webson & Pavlick 2021 + 9756f1a3-3607-43c3-a513-958271bd770b: !Template + answer_choices: صحيح ||| غير حاسم ||| غير صحيح + id: 9756f1a3-3607-43c3-a513-958271bd770b + jinja: '{{premise}} استخدام الوصف أعلاه وما تعرفه عن العالم ، "{{hypothesis}}" + هو بالتأكيد صحيح أم غير صحيح أم غير حاسم؟ ||| {{ answer_choices[label] }}' + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: MNLI crowdsource_armt + reference: '' + 97d2f061-06b0-4aa3-af53-871a2b06748f: !Template + answer_choices: صح ||| غير ذلك ||| خطأ + id: 97d2f061-06b0-4aa3-af53-871a2b06748f + jinja: '{{premise}} سؤال: {{hypothesis}} صح ، خطأ إو غير ذلك ||| {{ answer_choices[label] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - ar + metrics: + - Accuracy + original_task: true + name: GPT-3 style_arht + reference: 'Same as reported in Figure G7 of the GPT-3 paper, except that there + is no task identifying tokens like "anli R1: ".' + 97ddd82d-e5d6-4c56-be33-c490b1c81350: !Template + answer_choices: حقيقي ||| لا هذا ولا ذاك ||| خطأ شنيع + id: 97ddd82d-e5d6-4c56-be33-c490b1c81350 + jinja: '{{premise}} سؤال: {{hypothesis}} صواب أم خطأ أم لا؟ ||| {{ answer_choices[label] + }}' + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: GPT-3 style_armt + reference: '' + b1c0d748-3969-4df6-9836-59c23806faf2: !Template + answer_choices: نعم ||| يمكن ||| رقم + id: b1c0d748-3969-4df6-9836-59c23806faf2 + jinja: '{{premise}} هل يحق لنا أن نقول إن "{{hypothesis}}"؟ نعم ، لا + ، أو ربما؟ ||| {{ answer_choices[label] }}' + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: justified in saying_armt + reference: '' + ba194be7-4818-4e24-b5f9-bbcccaa4e166: !Template + answer_choices: نعم ||| ربما ||| لا + id: ba194be7-4818-4e24-b5f9-bbcccaa4e166 + jinja: 'بناء على المقدمة التالية {{premise}} هل يمكن أن تليها الفرضية التالية: + {{hypothesis}} أجب بـ نعم أو ربما أو لا ؟ ||| {{ answer_choices[label] }}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - ar + metrics: + - Accuracy + original_task: true + name: does it follow that_arht + reference: Sanh et al. 2021 + d9e1809e-fa44-4f21-8456-3bbd30ff5c87: !Template + answer_choices: مضمون ||| المستطاع ||| غير ممكن + id: d9e1809e-fa44-4f21-8456-3bbd30ff5c87 + jinja: افترض أنه صحيح أن {{premise}} لذلك ، "{{hypothesis}}" {{"guaranteed"}} + ، {{"possible"}} ، أو {{"impossible"}}؟ ||| {{ answer_choices[label] }} + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: guaranteed/possible/impossible_armt + reference: '' + e44c2947-f8e0-49ea-9770-e59f0581a921: !Template + answer_choices: مؤكدة ||| ممكنة ||| مستحيلة + id: e44c2947-f8e0-49ea-9770-e59f0581a921 + jinja: "اذا كانت لدينا المقدمة الصحيحة التالية {{premise}} \nفإن, \"{{hypothesis}}\"\ + \ هي دعوى {{\"مؤكدة\"}} أو {{\"ممكنة\"}} أو {{\"مستحيلة\"}}؟ ||| {{ answer_choices[label]\ + \ }}\n" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - ar + metrics: + - Accuracy + original_task: true + name: guaranteed/possible/impossible_arht + reference: Sanh et al. 2021 + ea368462-3a66-4222-9de1-05d66037a708: !Template + answer_choices: صحيحة ||| لا يمكن الجزم ||| غير صحيحة + id: ea368462-3a66-4222-9de1-05d66037a708 + jinja: " {{premise}}\n بإستخدام الوصف بالأعلى وحسب ما تعرفه من المعرفة العامة\ + \ فقط هل يمكن القول بأن الفرضية التالية: \"{{hypothesis}}\" بلا شك صحيحة أو\ + \ خاطئة أو لا يمكن الجزم؟ ||| {{ answer_choices[label] }}\n" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - ar + metrics: + - Accuracy + original_task: true + name: MNLI crowdsource_arht + reference: Adapted from Williams et al. 2018's instructions to crowdsourcing workers. + ef61016d-6451-4462-b69c-cb2ac7856b32: !Template + answer_choices: صحيحة ||| لا يمكن الجزم ||| خاطئة + id: ef61016d-6451-4462-b69c-cb2ac7856b32 + jinja: 'اذا كانت المقدمة التالية صحيحة: {{premise}} فإن الفرضية التالية: "{{hypothesis}}" {{"صحيحة"}} + أو {{"خاطئة"}} أو {{"لايمكن الجزم"}}؟ ||| {{ answer_choices[label] }} ' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - ar + metrics: + - Accuracy + original_task: true + name: take the following as truth_arht + reference: Sanh et al. 2021 diff --git a/promptsource/templates/xnli/bg/templates.yaml b/promptsource/templates/xnli/bg/templates.yaml new file mode 100644 index 000000000..a463798e3 --- /dev/null +++ b/promptsource/templates/xnli/bg/templates.yaml @@ -0,0 +1,69 @@ +dataset: xnli +subset: bg +templates: + 116e9cdd-0507-48a4-bdfc-b449da21882d: !Template + answer_choices: Вярно ||| Нито едното ||| Невярно + id: 116e9cdd-0507-48a4-bdfc-b449da21882d + jinja: '{{premise}} + + Въпрос: {{hypothesis}} Вярно, невярно или нито едно от двете? ||| {{ answer_choices[label] + }}' + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: GPT-3 style_bgmt + reference: '' + 3cc01b61-d8a3-4522-aaf0-2442b144f970: !Template + answer_choices: да ||| Може би ||| Не + id: 3cc01b61-d8a3-4522-aaf0-2442b144f970 + jinja: Да предположим, че {{premise}} Можем ли да заключим, че "{{hypothesis}}"? + Да, не, или може би? ||| {{ answer_choices[label] }} + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: can we infer_bgmt + reference: '' + 49a5a603-d96a-4e6a-ac2f-07a6265af0c8: !Template + answer_choices: Правилно ||| Неубедителен ||| Неправилно + id: 49a5a603-d96a-4e6a-ac2f-07a6265af0c8 + jinja: '{{premise}} Използвайки само горното описание и това, което знаете за + света, „{{hypothesis}}“ определено е правилно, неправилно или неубедително? + ||| {{ answer_choices[label] }}' + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: MNLI crowdsource_bgmt + reference: '' + 59677618-2642-4986-ada2-f2a17e28b744: !Template + answer_choices: да ||| Може би ||| Не + id: 59677618-2642-4986-ada2-f2a17e28b744 + jinja: '{{premise}} Оправдани ли сме да кажем, че „{{hypothesis}}“? Да, не, или + може би? ||| {{ answer_choices[label] }}' + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: justified in saying_bgmt + reference: '' + 9894e00d-859b-494b-806c-7aad45b11a72: !Template + answer_choices: Гарантирано ||| Възможен ||| Невъзможен + id: 9894e00d-859b-494b-806c-7aad45b11a72 + jinja: 'Да приемем, че е вярно, че {{premise}} + + + Следователно „{{hypothesis}}“ е {{"guaranteed"}}, {{"possible"}} или {{"impossible"}}? + ||| {{ answer_choices[label] }}' + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: guaranteed/possible/impossible_bgmt + reference: '' diff --git a/promptsource/templates/xnli/de/templates.yaml b/promptsource/templates/xnli/de/templates.yaml new file mode 100644 index 000000000..53e2f6c3a --- /dev/null +++ b/promptsource/templates/xnli/de/templates.yaml @@ -0,0 +1,69 @@ +dataset: xnli +subset: de +templates: + 3e41cd49-8acf-4325-96e4-8f155a243cb1: !Template + answer_choices: Richtig ||| Nicht schlüssig ||| Falsch + id: 3e41cd49-8acf-4325-96e4-8f155a243cb1 + jinja: '{{premise}} Wenn Sie nur die obige Beschreibung und das, was Sie über + die Welt wissen, verwenden, ist "{{hypothesis}}" definitiv richtig, falsch oder + nicht schlüssig? ||| {{ answer_choices[label] }}' + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: MNLI crowdsource_demt + reference: '' + 45a92641-0ab4-4225-9d8e-160606ee6e3d: !Template + answer_choices: Ja ||| Vielleicht ||| Nein + id: 45a92641-0ab4-4225-9d8e-160606ee6e3d + jinja: '{{premise}} Sind wir berechtigt zu sagen, dass "{{hypothesis}}"? Ja, nein, + oder vielleicht? ||| {{ answer_choices[label] }}' + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: justified in saying_demt + reference: '' + 8338e809-b10a-4a38-bfa3-549948e7d5fb: !Template + answer_choices: WAHR ||| Weder ||| FALSCH + id: 8338e809-b10a-4a38-bfa3-549948e7d5fb + jinja: '{{premise}} + + Frage: {{hypothesis}} Wahr, falsch oder keines von beiden? ||| {{ answer_choices[label] + }}' + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: GPT-3 style_demt + reference: '' + c6db6225-f091-434e-acbf-90f0ded80883: !Template + answer_choices: Garantiert ||| Möglich ||| Unmöglich + id: c6db6225-f091-434e-acbf-90f0ded80883 + jinja: 'Angenommen, es ist wahr, dass {{premise}} + + + Daher ist "{{hypothesis}}" {{"guaranteed"}}, {{"possible"}} oder {{"impossible"}}? + ||| {{ answer_choices[label] }}' + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: guaranteed/possible/impossible_demt + reference: '' + f9937a08-1b2e-428b-a5b4-4108e28cc0c9: !Template + answer_choices: Ja ||| Vielleicht ||| Nein + id: f9937a08-1b2e-428b-a5b4-4108e28cc0c9 + jinja: Angenommen {{premise}} Können wir daraus schließen, dass "{{hypothesis}}"? + Ja, nein, oder vielleicht? ||| {{ answer_choices[label] }} + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: can we infer_demt + reference: '' diff --git a/promptsource/templates/xnli/el/templates.yaml b/promptsource/templates/xnli/el/templates.yaml new file mode 100644 index 000000000..f331f6660 --- /dev/null +++ b/promptsource/templates/xnli/el/templates.yaml @@ -0,0 +1,69 @@ +dataset: xnli +subset: el +templates: + 3085e6b0-81f2-4d50-b7dc-63b023bb6792: !Template + answer_choices: Ναί ||| Μπορεί ||| Οχι + id: 3085e6b0-81f2-4d50-b7dc-63b023bb6792 + jinja: Ας υποθέσουμε ότι {{premise}} Μπορούμε να συμπεράνουμε ότι "{{hypothesis}}"; + Ναι, όχι, ή μήπως; ||| {{ answer_choices[label] }} + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: can we infer_elmt + reference: '' + 3789dc3c-d422-4d7e-8826-9947c5af7d21: !Template + answer_choices: Ναί ||| Μπορεί ||| Οχι + id: 3789dc3c-d422-4d7e-8826-9947c5af7d21 + jinja: '{{premise}} Είμαστε δικαιολογημένοι που λέμε ότι "{{hypothesis}}"; Ναι, + όχι, ή μήπως; ||| {{ answer_choices[label] }}' + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: justified in saying_elmt + reference: '' + 8f474851-54d1-4f0e-98f2-1f4b83bb25ec: !Template + answer_choices: Σωστός ||| Μη τελεσίδικος ||| Ανακριβής + id: 8f474851-54d1-4f0e-98f2-1f4b83bb25ec + jinja: '{{premise}} Χρησιμοποιώντας μόνο την παραπάνω περιγραφή και όσα γνωρίζετε + για τον κόσμο, το "{{hypothesis}}" είναι σίγουρα σωστό, λανθασμένο ή ασαφές; + ||| {{ answer_choices[label] }}' + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: MNLI crowdsource_elmt + reference: '' + de3451f3-96de-4807-b73a-b840670f3a4d: !Template + answer_choices: Εγγυημένα ||| Δυνατόν ||| Αδύνατο + id: de3451f3-96de-4807-b73a-b840670f3a4d + jinja: 'Ας υποθέσουμε ότι είναι αλήθεια ότι {{premise}} + + + Επομένως, η "{{hypothesis}}" είναι {{"guaranteed"}}, {{"possible"}} ή {{"impossible"}}; + ||| {{ answer_choices[label] }}' + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: guaranteed/possible/impossible_elmt + reference: '' + e52a2994-df8f-49ab-b850-03276c36b1f6: !Template + answer_choices: Αληθής ||| κανενα απο τα δυο ||| Ψευδής + id: e52a2994-df8f-49ab-b850-03276c36b1f6 + jinja: '{{premise}} + + Ερώτηση: {{hypothesis}} Σωστό, Λάθος ή κανένα από τα δύο; ||| {{ answer_choices[label] + }}' + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: GPT-3 style_elmt + reference: '' diff --git a/promptsource/templates/xnli/es/templates.yaml b/promptsource/templates/xnli/es/templates.yaml new file mode 100644 index 000000000..ce89ab08d --- /dev/null +++ b/promptsource/templates/xnli/es/templates.yaml @@ -0,0 +1,140 @@ +dataset: xnli +subset: es +templates: + 0c39a992-f60b-45a9-bfd6-4ff81b491d28: !Template + answer_choices: Sí ||| Quizás ||| No + id: 0c39a992-f60b-45a9-bfd6-4ff81b491d28 + jinja: '{{premise}} ¿Estamos justificados al decir que "{{hypothesis}}"? + ¿Sí, no o tal vez? ||| {{ answer_choices[label] }}' + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: justified in saying_esmt + reference: '' + 2c30be6c-52f9-4754-9848-bcb4ee7a4d88: !Template + answer_choices: Verdadero ||| Ninguno de los dos ||| Falso + id: 2c30be6c-52f9-4754-9848-bcb4ee7a4d88 + jinja: '{{premise}} Pregunta: {{hypothesis}} ¿Verdadero, falso o ninguno? ||| + {{ answer_choices[label] }}' + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: GPT-3 style_esmt + reference: '' + 37d2f072-06b0-4aa3-af53-871a2b06748f: !Template + answer_choices: Verdadero ||| Ninguno ||| Falso + id: 37d2f072-06b0-4aa3-af53-871a2b06748f + jinja: '{{premise}} + + Pregunta: {{hypothesis}} Verdadero, Falso, o Ninguno? ||| {{ answer_choices[label] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: GPT-3 style_esht + reference: 'Same as reported in Figure G7 of the GPT-3 paper, except that there + is no task identifying tokens like "anli R1: ".' + 8426ac76-cfe2-4d3a-b57b-b7904b01cd6f: !Template + answer_choices: Correcto ||| Poco concluyente ||| Incorrecto + id: 8426ac76-cfe2-4d3a-b57b-b7904b01cd6f + jinja: '{{premise}} Usando solo la descripción anterior y lo que sabe sobre el + mundo, "{{hypothesis}}" es definitivamente correcta, incorrecta o + no concluyente. ||| {{ answer_choices[label] }}' + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: MNLI crowdsource_esmt + reference: '' + 9b75ff67-bb66-423b-a33d-325707b035d7: !Template + answer_choices: Sí ||| Tal vez ||| No + id: 9b75ff67-bb66-423b-a33d-325707b035d7 + jinja: 'Supongamos {{premise}} Podemos inferir que "{{hypothesis}}"? Si, no, o + tal vez? ||| {{ answer_choices[label] }} ' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: can we infer_esht + reference: Webson & Pavlick 2021 + 9bda8e36-c991-4c9a-a3a9-eec68388a6f6: !Template + answer_choices: Sí ||| Tal vez ||| No + id: 9bda8e36-c991-4c9a-a3a9-eec68388a6f6 + jinja: '{{premise}} Estamos justificados en decir que "{{hypothesis}}"? Si, no, + o tal vez? ||| {{ answer_choices[label] }} ' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: justified in saying_esht + reference: Webson & Pavlick 2021 + d44c2947-f8e0-49ea-9770-e59f0581a922: !Template + answer_choices: Garantizado ||| Posible ||| Imposible + id: d44c2947-f8e0-49ea-9770-e59f0581a922 + jinja: "Supongamos que es cierto que {{premise}} \n\npor lo tanto, \"{{hypothesis}}\"\ + \ es {{\"garantizado\"}}, {{\"posible\"}}, o {{\"imposible\"}}? ||| {{ answer_choices[label]\ + \ }}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: guaranteed/possible/impossible_esht + reference: Sanh et al. 2021 + da388462-3a66-4222-9de1-05d66037a708: !Template + answer_choices: Correcto ||| No concluyente ||| Incorrecto + id: da388462-3a66-4222-9de1-05d66037a708 + jinja: '{{premise}} Usando solo la descripción anterior y lo que sabe sobre el + mundo, "{{hypothesis}}" es definitivamente correcto, incorrecto o no concluyente? + ||| {{ answer_choices[label] }}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: MNLI crowdsource_esht + reference: Adapted from Williams et al. 2018's instructions to crowdsourcing workers. + df379cab-fa2a-44c9-8008-2fe610efe467: !Template + answer_choices: Sí ||| Quizás ||| No + id: df379cab-fa2a-44c9-8008-2fe610efe467 + jinja: Supongamos {{premise}} ¿Podemos inferir que "{{hypothesis}}"? + ¿Sí, no o tal vez? ||| {{ answer_choices[label] }} + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: can we infer_esmt + reference: '' + f31ba22f-e7e1-44e5-9d92-6b3bbe8ad97e: !Template + answer_choices: garantizado ||| Posible ||| Imposible + id: f31ba22f-e7e1-44e5-9d92-6b3bbe8ad97e + jinja: Supongamos que es cierto que {{premise}} Por lo tanto, "{{hypothesis}}" + es {{"guaranteed"}}, {{"possible"}} o {{"impossible"}}. ||| {{ answer_choices[label] + }} + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: guaranteed/possible/impossible_esmt + reference: '' diff --git a/promptsource/templates/xnli/fr/templates.yaml b/promptsource/templates/xnli/fr/templates.yaml new file mode 100644 index 000000000..48c4939b0 --- /dev/null +++ b/promptsource/templates/xnli/fr/templates.yaml @@ -0,0 +1,142 @@ +dataset: xnli +subset: fr +templates: + 0fd68cdf-0488-4a70-a075-74df1883b784: !Template + answer_choices: Oui ||| Peut-être ||| Non + id: 0fd68cdf-0488-4a70-a075-74df1883b784 + jinja: Supposons que {{premise}} Pouvons-nous déduire que "{{hypothesis}}" ? + Oui, non, ou peut-être ? ||| {{ answer_choices[label] }} + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: can we infer_frmt + reference: '' + 28cc3c2d-a9d1-4bb8-b9a0-52e9df17751d: !Template + answer_choices: Garanti ||| Possible ||| Impossible + id: 28cc3c2d-a9d1-4bb8-b9a0-52e9df17751d + jinja: Supposons qu'il est vrai que {{premise}} Par conséquent, "{{hypothesis}}" + est {{"guaranteed"}}, {{"possible"}} ou {{"impossible"}} ? ||| {{ answer_choices[label] + }} + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: guaranteed/possible/impossible_frmt + reference: '' + 37d2f761-06b0-4aa3-af53-871a2b06748f: !Template + answer_choices: Vrai ||| Ni l'un ni l'autre ||| Faux + id: 37d2f761-06b0-4aa3-af53-871a2b06748f + jinja: '{{premise}} + + Question: {{hypothesis}} Vrai, faux, ou ni l''un ni l''autre? ||| {{ answer_choices[label] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - fr + metrics: + - Accuracy + original_task: true + name: GPT-3 style_frht + reference: 'Same as reported in Figure G7 of the GPT-3 paper, except that there + is no task identifying tokens like "anli R1: ".' + 40fdc487-3935-47e2-a43f-413a1d86d7e1: !Template + answer_choices: Corriger ||| Non concluant ||| Incorrect + id: 40fdc487-3935-47e2-a43f-413a1d86d7e1 + jinja: '{{premise}} En utilisant uniquement la description ci-dessus et ce que + vous savez sur le monde, "{{hypothesis}}" est définitivement correcte, + incorrecte ou non concluante ? ||| {{ answer_choices[label] }}' + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: MNLI crowdsource_frmt + reference: '' + 96a70043-d49a-4cd3-b235-a3c8a3a985fd: !Template + answer_choices: Oui ||| Peut-être ||| Non + id: 96a70043-d49a-4cd3-b235-a3c8a3a985fd + jinja: '{{premise}} Sommes-nous justifiés de dire que "{{hypothesis}}" ? + Oui, non, ou peut-être ? ||| {{ answer_choices[label] }}' + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: justified in saying_frmt + reference: '' + 9b75ff67-bb62-413b-a33d-325707b035d7: !Template + answer_choices: Oui ||| Peut-être ||| Non + id: 9b75ff67-bb62-413b-a33d-325707b035d7 + jinja: 'Supposons « {{premise}} », peut-on inférer « {{hypothesis}} » ? Oui, non, + ou peut-être ? ||| {{ answer_choices[label] }} ' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - fr + metrics: + - Accuracy + original_task: true + name: can we infer_frht + reference: Webson & Pavlick 2021 + 2bda8e36-c881-4c9a-a3a9-eec68388a6f6: !Template + answer_choices: Oui ||| Peut-être ||| Non + id: 2bda8e36-c881-4c9a-a3a9-eec68388a6f6 + jinja: "{{premise}} \n\nPeut-on dire que « {{hypothesis}} » ? Oui, non, ou peut-être\ + \ ? ||| {{ answer_choices[label] }} " + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - fr + metrics: + - Accuracy + original_task: true + name: justified in saying_frht + reference: Webson & Pavlick 2021 + d44c2947-f8e0-49ea-9770-e59f0581a966: !Template + answer_choices: Garanti ||| Possible ||| Impossible + id: d44c2947-f8e0-49ea-9770-e59f0581a966 + jinja: 'Présumons que ce soit vrai que « {{premise}} » + + + Alors, « {{hypothesis}} » est garanti, possible, ou impossible? ||| {{ answer_choices[label] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - fr + metrics: + - Accuracy + original_task: true + name: guaranteed/possible/impossible_frht + reference: Sanh et al. 2021 + da368462-3a66-4222-9de1-05d66032a708: !Template + answer_choices: Correct ||| Inconcluant ||| Incorrect + id: da368462-3a66-4222-9de1-05d66032a708 + jinja: "{{premise}} \n\nEn utilisant la description ci-dessus et ce que vous savez\ + \ du monde, « {{hypothesis}} » est définitivement correct, incorrect, inconcluant\ + \ ? ||| {{ answer_choices[label] }}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - fr + metrics: + - Accuracy + original_task: true + name: MNLI crowdsource_frht + reference: Adapted from Williams et al. 2018's instructions to crowdsourcing workers. + df398791-bfd5-4774-897c-c1a9acb3801c: !Template + answer_choices: Vrai ||| Ni ||| Faux + id: df398791-bfd5-4774-897c-c1a9acb3801c + jinja: '{{premise}} Question : {{hypothesis}} Vrai, faux ou ni l'un ni l'autre ? + ||| {{ answer_choices[label] }}' + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: GPT-3 style_frmt + reference: '' diff --git a/promptsource/templates/xnli/hi/templates.yaml b/promptsource/templates/xnli/hi/templates.yaml new file mode 100644 index 000000000..76dca4406 --- /dev/null +++ b/promptsource/templates/xnli/hi/templates.yaml @@ -0,0 +1,139 @@ +dataset: xnli +subset: hi +templates: + 25a32b2a-f5e2-4e74-a5c9-828b99261b43: !Template + answer_choices: सही ||| दुविधा में पड़ा हुआ ||| ग़लत + id: 25a32b2a-f5e2-4e74-a5c9-828b99261b43 + jinja: '{{premise}} केवल उपरोक्त विवरण और दुनिया के बारे में आप जो जानते हैं उसका + उपयोग करते हुए, "{{hypothesis}}" निश्चित रूप से सही, गलत या अनिर्णायक + है? ||| {{ answer_choices[label] }}' + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: MNLI crowdsource_himt + reference: '' + 31ed3aca-e0c1-4ebb-a2a2-4d3063e72144: !Template + answer_choices: हाँ ||| शायद ||| नहीं + id: 31ed3aca-e0c1-4ebb-a2a2-4d3063e72144 + jinja: '{{premise}} क्या यह कहना उचित है कि "{{hypothesis}}"? हाँ, नहीं, + या शायद? ||| {{ answer_choices[label] }}' + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: justified in saying_himt + reference: '' + 37d2f061-06c0-4aa3-af53-871a2b06748f: !Template + answer_choices: सत्य ||| इनमें से कोई भी नहीं ||| असत्य + id: 37d2f061-06c0-4aa3-af53-871a2b06748f + jinja: '{{premise}} + + प्रश्न: "{{hypothesis}}" सत्य, असत्य, या इनमें से कोई भी नहीं है? ||| {{ answer_choices[label] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: GPT-3 style_hiht + reference: 'Same as reported in Figure G7 of the GPT-3 paper, except that there + is no task identifying tokens like "anli R1: ".' + 629827a5-5057-4dea-b382-4f3c8ca2b15b: !Template + answer_choices: हाँ ||| शायद ||| नहीं + id: 629827a5-5057-4dea-b382-4f3c8ca2b15b + jinja: मान लीजिए {{premise}} क्या हम उस "{{hypothesis}}" का अनुमान लगा + सकते हैं? हाँ, नहीं, या शायद? ||| {{ answer_choices[label] }} + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: can we infer_himt + reference: '' + 83bdae91-8a37-4575-a63f-b5f998e861df: !Template + answer_choices: गारंटी ||| संभव ||| असंभव + id: 83bdae91-8a37-4575-a63f-b5f998e861df + jinja: मान लें कि यह सच है कि {{premise}} इसलिए, "{{hypothesis}}" {{"guaranteed"}}, + {{"possible"}}, या {{"impossible"}} है? ||| {{ answer_choices[label] }} + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: guaranteed/possible/impossible_himt + reference: '' + 9b75ff67-bb46-413b-a33d-325707b035d7: !Template + answer_choices: हाँ ||| शायद ||| नहीं + id: 9b75ff67-bb46-413b-a33d-325707b035d7 + jinja: 'मान लीजिए "{{premise}}" क्या हम यह अनुमान लगा सकते हैं कि "{{hypothesis}}"? + हाँ, नहीं, या शायद? ||| {{ answer_choices[label] }} ' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: can we infer_hiht + reference: Webson & Pavlick 2021 + 9bdb8e36-c881-4c9a-a3a9-eec68388a6f6: !Template + answer_choices: हाँ ||| शायद ||| नहीं + id: 9bdb8e36-c881-4c9a-a3a9-eec68388a6f6 + jinja: '"{{premise}}" क्या यह कहना उचित है कि "{{hypothesis}}"? हाँ, नहीं, या + शायद? ||| {{ answer_choices[label] }} ' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: justified in saying_hiht + reference: Webson & Pavlick 2021 + a3bcd909-2598-4d7a-a2b5-dc369556b663: !Template + answer_choices: सत्य ||| न ||| असत्य + id: a3bcd909-2598-4d7a-a2b5-dc369556b663 + jinja: '{{premise}} प्रश्न: {{hypothesis}} सत्य, असत्य, या नहीं? ||| {{ answer_choices[label] + }}' + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: GPT-3 style_himt + reference: '' + d44c2947-f8e0-55ea-9770-e59f0581a921: !Template + answer_choices: निश्चित ||| संभव ||| असंभव + id: d44c2947-f8e0-55ea-9770-e59f0581a921 + jinja: "मान लें कि यह सच है कि \"{{premise}}\" \n\nइसलिए, क्या \"{{hypothesis}}\"\ + \ {{\"निश्चित\"}}, {{\"संभव\"}}, या {{\"असंभव\"}} है? ||| {{ answer_choices[label]\ + \ }}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: guaranteed/possible/impossible_hiht + reference: Sanh et al. 2021 + da368462-3a66-4233-9de1-05d66037a708: !Template + answer_choices: सही ||| अनिर्णीत ||| गलत + id: da368462-3a66-4233-9de1-05d66037a708 + jinja: '"{{premise}}" केवल उपरोक्त विवरण और दुनिया के बारे में आप जो जानते हैं + उसका उपयोग करते हुए, "{{hypothesis}}" सही, गलत या अनिर्णीत है? ||| {{ answer_choices[label] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: MNLI crowdsource_hiht + reference: Adapted from Williams et al. 2018's instructions to crowdsourcing workers. diff --git a/promptsource/templates/xnli/ru/templates.yaml b/promptsource/templates/xnli/ru/templates.yaml new file mode 100644 index 000000000..1aa40e8c2 --- /dev/null +++ b/promptsource/templates/xnli/ru/templates.yaml @@ -0,0 +1,69 @@ +dataset: xnli +subset: ru +templates: + 04f56ff4-14c6-4362-a599-ec2eb735f3d4: !Template + answer_choices: Гарантировано ||| Возможный ||| Невозможно + id: 04f56ff4-14c6-4362-a599-ec2eb735f3d4 + jinja: 'Предположим, что верно, что {{premise}} + + + Следовательно, "{{hypothesis}}" является {{"guaranteed"}}, {{"possible"}} или + {{"impossible"}}? ||| {{ answer_choices[label] }}' + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: guaranteed/possible/impossible_rumt + reference: '' + 2858121b-f5e9-4e91-a30a-fc2c7af1c223: !Template + answer_choices: Да ||| Может быть ||| Нет + id: 2858121b-f5e9-4e91-a30a-fc2c7af1c223 + jinja: Предположим, {{premise}} Можем ли мы сделать вывод, что "{{hypothesis}}"? + Да, нет, а может быть? ||| {{ answer_choices[label] }} + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: can we infer_rumt + reference: '' + 67b21e12-5fb0-4bb7-a0fd-3d2bc551d0b2: !Template + answer_choices: Да ||| Может быть ||| Нет + id: 67b21e12-5fb0-4bb7-a0fd-3d2bc551d0b2 + jinja: '{{premise}} Имеем ли мы право говорить, что "{{hypothesis}}"? Да, нет, + а может быть? ||| {{ answer_choices[label] }}' + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: justified in saying_rumt + reference: '' + 9e093809-e6e8-4d5e-af5f-81e69412206e: !Template + answer_choices: Правильный ||| Неубедительный ||| Неправильно + id: 9e093809-e6e8-4d5e-af5f-81e69412206e + jinja: '{{premise}} Используя только приведенное выше описание и то, что вы знаете + о мире, "{{hypothesis}}" определенно верна, неверна или неубедительна? ||| {{ + answer_choices[label] }}' + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: MNLI crowdsource_rumt + reference: '' + e7aa68b6-262a-4802-b6df-04b0fdab3de8: !Template + answer_choices: Истинный ||| Ни один ||| ЛОЖЬ + id: e7aa68b6-262a-4802-b6df-04b0fdab3de8 + jinja: '{{premise}} + + Вопрос: {{hypothesis}} верна, неверна или ни то, ни другое? ||| {{ answer_choices[label] + }}' + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: GPT-3 style_rumt + reference: '' diff --git a/promptsource/templates/xnli/sw/templates.yaml b/promptsource/templates/xnli/sw/templates.yaml new file mode 100644 index 000000000..f5be2d441 --- /dev/null +++ b/promptsource/templates/xnli/sw/templates.yaml @@ -0,0 +1,138 @@ +dataset: xnli +subset: sw +templates: + 336286e7-cd66-4318-b3e5-31e1ef304b34: !Template + answer_choices: Kweli ||| Wala ||| Uongo + id: 336286e7-cd66-4318-b3e5-31e1ef304b34 + jinja: '{{premise}} Swali: {{hypothesis}} Kweli, Si kweli au Sivyo? ||| {{ answer_choices[label] + }}' + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: GPT-3 style_swmt + reference: '' + 37d2f061-17b0-4aa3-af53-871a2b06748f: !Template + answer_choices: Ukweli ||| Wala ||| Uongo + id: 37d2f061-17b0-4aa3-af53-871a2b06748f + jinja: '{{premise}} Swali: {{hypothesis}} Ukweli, Uongo, au Wala ? ||| {{ answer_choices[label] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: GPT-3 style_swht + reference: 'Same as reported in Figure G7 of the GPT-3 paper, except that there + is no task identifying tokens like "anli R1: ".' + 95062749-0cf2-4211-a6e8-496bd809764e: !Template + answer_choices: Sahihi ||| Haijumuishi ||| Si sahihi + id: 95062749-0cf2-4211-a6e8-496bd809764e + jinja: '{{premise}} Kwa kutumia tu maelezo yaliyo hapo juu na kile unachojua kuhusu + ulimwengu, "{{hypothesis}}" ni sahihi, si sahihi, au si sahihi? ||| + {{ answer_choices[label] }}' + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: MNLI crowdsource_swmt + reference: '' + 9b75ff62-bb66-413b-a33d-325707b035d7: !Template + answer_choices: Ndio ||| Labda ||| Hapana + id: 9b75ff62-bb66-413b-a33d-325707b035d7 + jinja: Ijapo {{premise}} Tunaweza kisi kuwa "{{hypothesis}}"? Ndio, hapana, au + labda? ||| {{ answer_choices[label] }} + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: can we infer_swht + reference: Webson & Pavlick 2021 + 9bdc8e36-c881-4c9a-a3a9-eec68388a6f6: !Template + answer_choices: Ndio ||| Labda ||| Hapana + id: 9bdc8e36-c881-4c9a-a3a9-eec68388a6f6 + jinja: '{{premise}} Tunahaki ya kusema kuwa "{{hypothesis}}"? Ndio, hapana, au + labda? ||| {{ answer_choices[label] }} ' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: justified in saying_swht + reference: Webson & Pavlick 2021 + 9c0ceeeb-a2c3-4a00-a4b1-7604469a53df: !Template + answer_choices: Imehakikishwa ||| Inawezekana ||| Haiwezekani + id: 9c0ceeeb-a2c3-4a00-a4b1-7604469a53df + jinja: Chukulia kuwa ni kweli kwamba {{premise}} Kwa hivyo, "{{hypothesis}}" + ni {{"guaranteed"}}, {{"possible"}}, au {{"impossible"}}? ||| {{ answer_choices[label] + }} + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: guaranteed/possible/impossible_swmt + reference: '' + b0170084-2a14-4e52-a459-d509b8465611: !Template + answer_choices: Ndiyo ||| Labda ||| Hapana + id: b0170084-2a14-4e52-a459-d509b8465611 + jinja: '{{premise}} Je, tuna haki ya kusema kwamba "{{hypothesis}}"? + Ndio, hapana, au labda? ||| {{ answer_choices[label] }}' + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: justified in saying_swmt + reference: '' + d44c2947-f8e0-49ea-9771-e59f0581a921: !Template + answer_choices: Imehakikishwa ||| Inawezekana ||| Haiwezekani + id: d44c2947-f8e0-49ea-9771-e59f0581a921 + jinja: "Ukidhania ni kweli kuwa {{premise}} \n\nKwa hivyo, \"{{hypothesis}}\"\ + \ ni {{\"Imehakikishwa\"}}, {{\"Inawezekana\"}}, au {{\"Haiwezekani\"}}? |||\ + \ {{ answer_choices[label] }}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: guaranteed/possible/impossible_swht + reference: Sanh et al. 2021 + da368462-3a88-4222-9de1-05d66037a708: !Template + answer_choices: Sahihi ||| Isiohitimishwa ||| Isio Sahihi + id: da368462-3a88-4222-9de1-05d66037a708 + jinja: '{{premise}} Ukitumia maelezo yaliyo hapo juu pekee na unayojua kuhusu + dunia, "{{hypothesis}}" ni hakika sahihi, isio sahihi, au isiohitimishwa? ||| + {{ answer_choices[label] }}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: MNLI crowdsource_swht + reference: Adapted from Williams et al. 2018's instructions to crowdsourcing workers. + f988a0fe-9980-47ea-be07-8d1c51c37854: !Template + answer_choices: Ndiyo ||| Labda ||| Hapana + id: f988a0fe-9980-47ea-be07-8d1c51c37854 + jinja: Tuseme {{premise}} Je, tunaweza kukisia kuwa "{{hypothesis}}"? + Ndio, hapana, au labda? ||| {{ answer_choices[label] }} + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: can we infer_swmt + reference: '' diff --git a/promptsource/templates/xnli/th/templates.yaml b/promptsource/templates/xnli/th/templates.yaml new file mode 100644 index 000000000..a8257577e --- /dev/null +++ b/promptsource/templates/xnli/th/templates.yaml @@ -0,0 +1,68 @@ +dataset: xnli +subset: th +templates: + 3aedf6e8-57e7-4ae9-aab7-3ca6972bef1e: !Template + answer_choices: ใช่ ||| อาจจะ ||| ไม่ + id: 3aedf6e8-57e7-4ae9-aab7-3ca6972bef1e + jinja: '{{premise}} เรามีเหตุผลในการพูดว่า "{{hypothesis}}" หรือไม่ ใช่ ไม่ หรืออาจจะ? + ||| {{ answer_choices[label] }}' + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: justified in saying_thmt + reference: '' + 549bf8a4-c0a9-413d-9f19-0688a89f8099: !Template + answer_choices: ใช่ ||| อาจจะ ||| ไม่ + id: 549bf8a4-c0a9-413d-9f19-0688a89f8099 + jinja: สมมติว่า {{premise}} เราสามารถอนุมานว่า "{{hypothesis}}" ได้หรือไม่ ใช่ + ไม่ หรืออาจจะ? ||| {{ answer_choices[label] }} + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: can we infer_thmt + reference: '' + c50d6c82-c8c8-46df-beef-a7e4fc6fd6fd: !Template + answer_choices: รับประกัน ||| เป็นไปได้ ||| เป็นไปไม่ได้ + id: c50d6c82-c8c8-46df-beef-a7e4fc6fd6fd + jinja: 'สมมติว่าเป็นความจริงที่ {{premise}} + + + ดังนั้น "{{hypothesis}}" คือ {{"guaranteed"}}, {{"possible"}} หรือ {{"impossible"}}? + ||| {{ answer_choices[label] }}' + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: guaranteed/possible/impossible_thmt + reference: '' + db82374c-c0f4-48f2-8d8d-f9c682a732f4: !Template + answer_choices: จริง ||| ไม่ใช่ทั้งสองอย่าง ||| เท็จ + id: db82374c-c0f4-48f2-8d8d-f9c682a732f4 + jinja: '{{premise}} + + คำถาม: {{hypothesis}} จริง เท็จ หรือเปล่า? ||| {{ answer_choices[label] }}' + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: GPT-3 style_thmt + reference: '' + dc028dd3-14d3-4e24-b2f4-ebd3930a01e5: !Template + answer_choices: ถูกต้อง ||| สรุปไม่ได้ ||| ไม่ถูกต้อง + id: dc028dd3-14d3-4e24-b2f4-ebd3930a01e5 + jinja: '{{premise}} ใช้เฉพาะคำอธิบายข้างต้นและสิ่งที่คุณรู้เกี่ยวกับโลก "{{hypothesis}}" + นั้นถูกต้อง ไม่ถูกต้อง หรือไม่สามารถสรุปได้อย่างแน่นอน ||| {{ answer_choices[label] + }}' + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: MNLI crowdsource_thmt + reference: '' diff --git a/promptsource/templates/xnli/tr/templates.yaml b/promptsource/templates/xnli/tr/templates.yaml new file mode 100644 index 000000000..b8fd9cc75 --- /dev/null +++ b/promptsource/templates/xnli/tr/templates.yaml @@ -0,0 +1,69 @@ +dataset: xnli +subset: tr +templates: + 666128c0-95be-483a-90b9-2689df2cbffb: !Template + answer_choices: garantili ||| Mümkün ||| İmkansız + id: 666128c0-95be-483a-90b9-2689df2cbffb + jinja: '{{premise}}''in doğru olduğunu varsayın + + + Bu nedenle, "{{hypothesis}}" {{"guaranteed"}}, {{"possible"}} veya {{"impossible"}} + mı? ||| {{ answer_choices[label] }}' + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: guaranteed/possible/impossible_trmt + reference: '' + 84ebd457-2c6e-4723-9714-1aadcbe71ab7: !Template + answer_choices: Evet ||| Belki ||| Numara + id: 84ebd457-2c6e-4723-9714-1aadcbe71ab7 + jinja: Diyelim ki {{premise}} "{{hypothesis}}" sonucuna varabilir miyiz? Evet, + hayır, ya da belki? ||| {{ answer_choices[label] }} + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: can we infer_trmt + reference: '' + 85b26595-086d-4360-b0f0-440b080c3477: !Template + answer_choices: Doğru ||| Hiç biri ||| Yanlış + id: 85b26595-086d-4360-b0f0-440b080c3477 + jinja: '{{premise}} + + Soru: {{hypothesis}} Doğru, Yanlış veya Hiçbiri? ||| {{ answer_choices[label] + }}' + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: GPT-3 style_trmt + reference: '' + 85e6954d-eb7d-43f3-8a93-4dd2ecf3ced2: !Template + answer_choices: Doğru ||| sonuçsuz ||| Yanlış + id: 85e6954d-eb7d-43f3-8a93-4dd2ecf3ced2 + jinja: '{{premise}} Yalnızca yukarıdaki açıklamayı ve dünya hakkında bildiklerinizi + kullanarak, "{{hypothesis}}" kesinlikle doğru, yanlış veya sonuçsuz mu? ||| + {{ answer_choices[label] }}' + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: MNLI crowdsource_trmt + reference: '' + c51efa50-efe0-4624-8145-a974bf91916e: !Template + answer_choices: Evet ||| Belki ||| Numara + id: c51efa50-efe0-4624-8145-a974bf91916e + jinja: '{{premise}} "{{hypothesis}}" demekte haklı mıyız? Evet, hayır, ya da belki? + ||| {{ answer_choices[label] }}' + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: justified in saying_trmt + reference: '' diff --git a/promptsource/templates/xnli/ur/templates.yaml b/promptsource/templates/xnli/ur/templates.yaml new file mode 100644 index 000000000..7a4f8b589 --- /dev/null +++ b/promptsource/templates/xnli/ur/templates.yaml @@ -0,0 +1,139 @@ +dataset: xnli +subset: ur +templates: + 37d2f261-06b0-4aa3-af53-871a2b06748f: !Template + answer_choices: صحیح ||| کچھ نہیں ||| غلط + id: 37d2f261-06b0-4aa3-af53-871a2b06748f + jinja: '{{premise}} + + سوال: {{hypothesis}} صحیح، غلط, یا کچھ نہیں؟ ||| {{ answer_choices[label] }}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: GPT-3 style_urht + reference: 'Same as reported in Figure G7 of the GPT-3 paper, except that there + is no task identifying tokens like "anli R1: ".' + 443b5ab1-614e-4d3f-a782-6ccff4af2380: !Template + answer_choices: سچ ہے۔ ||| نہ ہی ||| جھوٹا۔ + id: 443b5ab1-614e-4d3f-a782-6ccff4af2380 + jinja: '{{premise}} سوال: {{hypothesis}} صحیح، غلط، یا نہ؟ ||| {{ answer_choices[label] + }}' + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: GPT-3 style_urmt + reference: '' + 5d0b8303-80b2-41a8-b355-23928b181c20: !Template + answer_choices: جی ہاں ||| شاید ||| نہیں + id: 5d0b8303-80b2-41a8-b355-23928b181c20 + jinja: '{{premise}} کیا ہم یہ کہنا جائز ہیں کہ "{{hypothesis}}"؟ ہاں، + نہیں، یا شاید؟ ||| {{ answer_choices[label] }}' + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: justified in saying_urmt + reference: '' + 9b75ff77-bb66-413b-a33d-325707b035d7: !Template + answer_choices: ہاں ||| ممکن ہے ||| نہیں + id: 9b75ff77-bb66-413b-a33d-325707b035d7 + jinja: 'فرض کرو {{premise}} ہم یہ نتیجہ اخذ کر سکتے ہیں "{{hypothesis}}"? ہاں + ، نہیں ، یا ممکن ہے ؟ ||| {{ answer_choices[label] }} ' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: can we infer_urht + reference: Webson & Pavlick 2021 + 9bda8e44-c881-4c9a-a3a9-eec68388a6f6: !Template + answer_choices: ہاں ||| ممکن ہے ||| نہیں + id: 9bda8e44-c881-4c9a-a3a9-eec68388a6f6 + jinja: '{{premise}} کیا ہمارا یہ کہنا درست ہے "{{hypothesis}}"? ہاں ، نہیں ، یا + ممکن ہے ؟ ||| {{ answer_choices[label] }} ' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: justified in saying_urht + reference: Webson & Pavlick 2021 + ccce5ce7-a6bd-48c0-9cc9-f6b77309133f: !Template + answer_choices: جی ہاں ||| شاید ||| نہیں + id: ccce5ce7-a6bd-48c0-9cc9-f6b77309133f + jinja: فرض کریں {{premise}} کیا ہم اندازہ لگا سکتے ہیں کہ "{{hypothesis}}"؟ + ہاں، نہیں، یا شاید؟ ||| {{ answer_choices[label] }} + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: can we infer_urmt + reference: '' + d374295a-c1d4-4742-82d5-290d42726ec4: !Template + answer_choices: گارنٹی شدہ ||| ممکن ||| ناممکن + id: d374295a-c1d4-4742-82d5-290d42726ec4 + jinja: فرض کریں کہ یہ درست ہے کہ {{premise}} لہذا، "{{hypothesis}}" + {{"guaranteed"}}، {{"possible"}}، یا {{"impossible"}} ہے؟ ||| {{ answer_choices[label] + }} + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: guaranteed/possible/impossible_urmt + reference: '' + d44c2947-f8e0-49ea-9880-e59f0581a921: !Template + answer_choices: یقینی ||| ممکن ||| نا ممکن + id: d44c2947-f8e0-49ea-9880-e59f0581a921 + jinja: "فرض کرو یہ صحیح ہے كہ {{premise}} \n\nاِس لیے, \"{{hypothesis}}\" {{\"\ + یقینی\"}}, {{\"ممکن\"}}, یا {{\"نا ممکن\"}} ہے ؟ ||| {{ answer_choices[label]\ + \ }}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: guaranteed/possible/impossible_urht + reference: Sanh et al. 2021 + da368462-3a66-4222-9de1-05d66037a718: !Template + answer_choices: بالکل صحیح ||| غیر یقینی ||| غلط + id: da368462-3a66-4222-9de1-05d66037a718 + jinja: '{{premise}} صرف مندرجہ بالا تشریح اور دنیا كے بارے میں اپنی معلومات كے + مطابق, "{{hypothesis}}" بالکل صحیح ہے ، غلط ہے ، یا غیر یقینی ہے ؟ ||| {{ answer_choices[label] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: MNLI crowdsource_urht + reference: Adapted from Williams et al. 2018's instructions to crowdsourcing workers. + db00c977-f972-4fd8-b841-c924d443d242: !Template + answer_choices: درست ||| بے نتیجہ ||| غلط + id: db00c977-f972-4fd8-b841-c924d443d242 + jinja: '{{premise}} صرف اوپر دی گئی تفصیل کا استعمال کرتے ہوئے اور جو آپ دنیا + کے بارے میں جانتے ہیں، "{{hypothesis}}" یقینی طور پر درست، غلط، یا + غیر نتیجہ خیز ہے؟ ||| {{ answer_choices[label] }}' + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: MNLI crowdsource_urmt + reference: '' diff --git a/promptsource/templates/xnli/vi/templates.yaml b/promptsource/templates/xnli/vi/templates.yaml new file mode 100644 index 000000000..4a2ae3696 --- /dev/null +++ b/promptsource/templates/xnli/vi/templates.yaml @@ -0,0 +1,139 @@ +dataset: xnli +subset: vi +templates: + 37d2f061-06b0-4aa3-af53-871a2b08848f: !Template + answer_choices: đúng ||| cả hai đều không ||| sai + id: 37d2f061-06b0-4aa3-af53-871a2b08848f + jinja: '{{premise}} + + Câu hỏi: {{hypothesis}} đúng, sai, hay cả hai đều không? ||| {{ answer_choices[label] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: GPT-3 style_viht + reference: 'Same as reported in Figure G7 of the GPT-3 paper, except that there + is no task identifying tokens like "anli R1: ".' + 53e31928-4df0-43bd-9748-66e114f60fa9: !Template + answer_choices: Đúng ||| Có lẽ ||| Không + id: 53e31928-4df0-43bd-9748-66e114f60fa9 + jinja: Giả sử {{premise}} Chúng ta có thể suy ra rằng "{{hypothesis}}" + không? Có, không, hoặc có thể? ||| {{ answer_choices[label] }} + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: can we infer_vimt + reference: '' + 9b75ff67-bb66-413b-a33d-325707b035d8: !Template + answer_choices: đúng ||| có thể ||| sai + id: 9b75ff67-bb66-413b-a33d-325707b035d8 + jinja: 'Giả sử với {{premise}}, chúng ta có thể suy ra "{{hypothesis}}"? đúng, + sai, hay có thể? ||| {{ answer_choices[label] }} ' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: can we infer_viht + reference: Webson & Pavlick 2021 + 9bda8e96-c881-4c9a-a3a9-eec68388a6f6: !Template + answer_choices: đúng ||| có thể ||| sai + id: 9bda8e96-c881-4c9a-a3a9-eec68388a6f6 + jinja: '{{premise}} chúng ta có thể chứng minh "{{hypothesis}}" hay không? đúng, + sai, hay có thể? ||| {{ answer_choices[label] }} ' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: justified in saying_viht + reference: Webson & Pavlick 2021 + b0a93f3e-ae86-48a2-bcc2-0382aaa1f3e7: !Template + answer_choices: Đảm bảo ||| Khả thi ||| Không thể nào + id: b0a93f3e-ae86-48a2-bcc2-0382aaa1f3e7 + jinja: Giả sử điều đó đúng là {{premise}} Do đó, "{{hypothesis}}" là + {{"guaranteed"}}, {{"possible"}} hay {{"impossible"}}? ||| {{ answer_choices[label] + }} + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: guaranteed/possible/impossible_vimt + reference: '' + b8c75d5c-df8f-4368-9fbd-69a7241a7266: !Template + answer_choices: Chính xác ||| Không xác định ||| Không đúng + id: b8c75d5c-df8f-4368-9fbd-69a7241a7266 + jinja: '{{premise}} Chỉ sử dụng mô tả ở trên và những gì bạn biết về thế giới, + "{{hypothesis}}" chắc chắn là đúng, không chính xác hay không thể + kết luận? ||| {{ answer_choices[label] }}' + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: MNLI crowdsource_vimt + reference: '' + c4ae89df-ca0a-49eb-b5d6-e003ec6149d2: !Template + answer_choices: Đúng ||| Có lẽ ||| Không + id: c4ae89df-ca0a-49eb-b5d6-e003ec6149d2 + jinja: '{{premise}} Chúng tôi có biện minh khi nói rằng "{{hypothesis}}" + không? Có, không, hoặc có thể? ||| {{ answer_choices[label] }}' + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: justified in saying_vimt + reference: '' + d44c2947-f8e5-49ea-9770-e59f0581a921: !Template + answer_choices: chắc chắn ||| có thể ||| không thể + id: d44c2947-f8e5-49ea-9770-e59f0581a921 + jinja: "Giả sử {{premise}} là đúng \n\nThì, \"{{hypothesis}}\" là {{\"chắc chắn\"\ + }}, {{\"có thể\"}}, or {{\"không thể\"}}? ||| {{ answer_choices[label] }}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: guaranteed/possible/impossible_viht + reference: Sanh et al. 2021 + da368473-3a66-4222-9de1-05d66037a708: !Template + answer_choices: chính xác ||| không thể kết luận ||| không chính xác + id: da368473-3a66-4222-9de1-05d66037a708 + jinja: '{{premise}} Chỉ sử dụng mô tả ở trên và những gì bạn biết về thế giới, + "{{hypothesis}}" chắc chắn là chính xác, không chính xác, hay không thể kết + luận? ||| {{ answer_choices[label] }}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: MNLI crowdsource_viht + reference: Adapted from Williams et al. 2018's instructions to crowdsourcing workers. + fd41837b-4f78-4fb6-8edf-273df9a6b55a: !Template + answer_choices: ĐÚNG VẬY ||| Cũng không ||| Sai + id: fd41837b-4f78-4fb6-8edf-273df9a6b55a + jinja: '{{premise}} Câu hỏi: {{hypothesis}} Đúng, Sai hay Không? ||| {{ answer_choices[label] + }}' + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: GPT-3 style_vimt + reference: '' diff --git a/promptsource/templates/xnli/zh/templates.yaml b/promptsource/templates/xnli/zh/templates.yaml new file mode 100644 index 000000000..44c198f5d --- /dev/null +++ b/promptsource/templates/xnli/zh/templates.yaml @@ -0,0 +1,138 @@ +dataset: xnli +subset: zh +templates: + 39a53af0-df1e-4489-b5b9-c734338a840f: !Template + answer_choices: 是的 ||| 也许 ||| 不 + id: 39a53af0-df1e-4489-b5b9-c734338a840f + jinja: '{{premise}}我们有理由说“{{hypothesis}}”吗？是的，不是的，或者也许？ ||| {{ answer_choices[label] + }}' + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: justified in saying_zhmt + reference: '' + 3d2950c2-93da-499b-9e84-7708027e8666: !Template + answer_choices: 正确的 ||| 无定论 ||| 不正确 + id: 3d2950c2-93da-499b-9e84-7708027e8666 + jinja: '{{premise}} 仅使用上述描述和您对世界的了解，“{{hypothesis}}”肯定是正确的、不正确的还是不确定的？ ||| {{ + answer_choices[label] }}' + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: MNLI crowdsource_zhmt + reference: '' + 47d2f061-06b0-4aa3-af53-871a2b06748f: !Template + answer_choices: 对 ||| 无关 ||| 不对 + id: 47d2f061-06b0-4aa3-af53-871a2b06748f + jinja: '{{premise}} + + 问题: {{hypothesis}} 对，不对还是无关？ ||| {{ answer_choices[label] }}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: GPT-3 style_zhht + reference: 'Same as reported in Figure G7 of the GPT-3 paper, except that there + is no task identifying tokens like "anli R1: ".' + 504b04ed-9d85-47ef-bf5d-b28d5552aae7: !Template + answer_choices: 真的 ||| 两者都不 ||| 错误的 + id: 504b04ed-9d85-47ef-bf5d-b28d5552aae7 + jinja: '{{premise}} 问题：{{hypothesis}} 对、错或两者都不是？ ||| {{ answer_choices[label] + }}' + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: GPT-3 style_zhmt + reference: '' + 9c75ff67-bb66-413b-a33d-325707b035d7: !Template + answer_choices: 能 ||| 有可能 ||| 不能 + id: 9c75ff67-bb66-413b-a33d-325707b035d7 + jinja: '如果{{premise}}能不能推断“{{hypothesis}}”？能，不能，还是有可能？ ||| {{ answer_choices[label] + }} ' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: can we infer_zhht + reference: Webson & Pavlick 2021 + 9cda8e36-c881-4c9a-a3a9-eec68388a6f6: !Template + answer_choices: 可以 ||| 有可能 ||| 不可以 + id: 9cda8e36-c881-4c9a-a3a9-eec68388a6f6 + jinja: '{{premise}}我们就可不可以说“{{hypothesis}}”？可以，不可以，还是有可能？||| {{ answer_choices[label] + }} ' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: justified in saying_zhht + reference: Webson & Pavlick 2021 + 9dc1e5f4-75f4-44ed-b564-888b08b3d27a: !Template + answer_choices: 是的 ||| 也许 ||| 不 + id: 9dc1e5f4-75f4-44ed-b564-888b08b3d27a + jinja: 假设{{premise}}我们可以推断出“{{hypothesis}}”吗？是的，不是的，或者也许？ ||| {{ answer_choices[label] + }} + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: can we infer_zhmt + reference: '' + e27bbe1f-7f86-4cb6-a2c1-955e020e3d72: !Template + answer_choices: 保证 ||| 可能的 ||| 不可能的 + id: e27bbe1f-7f86-4cb6-a2c1-955e020e3d72 + jinja: 假设 {{premise}} 为真，因此，“{{hypothesis}}”是 {{"guaranteed"}}、{{"possible"}} + 还是 {{"impossible"}}？ ||| {{ answer_choices[label] }} + metadata: !TemplateMetadata + choices_in_prompt: null + languages: null + metrics: null + original_task: null + name: guaranteed/possible/impossible_zhmt + reference: '' + e44c2947-f8e0-49ea-9770-e59f0581a931: !Template + answer_choices: 肯定的 ||| 有可能的 ||| 不可能的 + id: e44c2947-f8e0-49ea-9770-e59f0581a931 + jinja: '假如{{premise}}是对的。 + + + 那么，”{{hypothesis}}“是 {{"肯定的"}}，{{"有可能的"}}还是{{"不可能的"}}？||| {{ answer_choices[label] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: guaranteed/possible/impossible_zhht + reference: Sanh et al. 2021 + ea368462-3a66-4222-9de1-05d66037a718: !Template + answer_choices: 正确 ||| 不确定 ||| 不正确 + id: ea368462-3a66-4222-9de1-05d66037a718 + jinja: '{{premise}} 仅仅利用上述描述和你对世界的了解，”{{hypothesis}}“肯定是正确，不正确，还是不确定？||| {{ answer_choices[label] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: MNLI crowdsource_zhht + reference: Adapted from Williams et al. 2018's instructions to crowdsourcing workers. diff --git a/promptsource/xnli_machine_translate.py b/promptsource/xnli_machine_translate.py new file mode 100644 index 000000000..8652443c2 --- /dev/null +++ b/promptsource/xnli_machine_translate.py @@ -0,0 +1,94 @@ +import os +import re + +from promptsource.templates import Template, TemplateCollection + + +PROMPTS = [ + "GPT-3 style", + "can we infer", + "justified in saying", + "guaranteed/possible/impossible", + "MNLI crowdsource", +] + +LANGS = [ + "ar", + "es", + "fr", + "hi", + "sw", + "ur", + "vi", + "zh", +] +# Path to key +os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "/Users/niklasmuennighoff/Desktop/gcp_translate_key.json" + + +def translate(target, text): + """Translates text into the target language. + Target must be an ISO 639-1 language code. + See https://g.co/cloud/translate/v2/translate-reference#supported_languages + (pip install --upgrade google-api-python-client) + pip install google-cloud-translate + """ + import six + from google.cloud import translate_v2 as translate + + translate_client = translate.Client() + if isinstance(text, six.binary_type): + text = text.decode("utf-8") + # Text can also be a sequence of strings, in which case this method + # will return a sequence of results for each text. + result = translate_client.translate(text, target_language=target) + print("Text: {}".format(result["input"])) + print("Translation: {}".format(result["translatedText"])) + print("Detected source language: {}".format(result["detectedSourceLanguage"])) + return result["translatedText"] + + +def normalize_string(zh_string, en_string): + """ + This is not specific to zh just to given an example. + Replaces the content in brackets in zh_string with the content in brackets from en_string. + All else is left the same in zh_string. + Args: + zh_string: {{前提}} 问题：{{假设}} 对、错或两者都不是？ ||| {{ answer_choices[标签] }} + en_string: {{premise}} Question: {{hypothesis}} True, False, or Neither? ||| {{ answer_choices[label] }} + Returns: + zh_string_normalized: {{premise}} 问题：{{hypothesis}} 对、错或两者都不是？ ||| {{ answer_choices[label] }} + """ + zh_string_normalized = zh_string + en_string_normalized = en_string + # Find all the content in brackets in zh_string + zh_bracket_content = re.findall(r"{{(.*?)}}", zh_string) + # Find all the content in brackets in en_string + en_bracket_content = re.findall(r"{{(.*?)}}", en_string) + # Replace the content in brackets in zh_string with the content in brackets from en_string + for i in range(len(zh_bracket_content)): + zh_string_normalized = zh_string_normalized.replace(zh_bracket_content[i], en_bracket_content[i]) + return zh_string_normalized + + +template_collection = TemplateCollection() +source_templates = template_collection.get_dataset("xnli", "en") + +for lang in LANGS: + target_templates = template_collection.get_dataset("xnli", lang) + for uid, template in source_templates.templates.items(): + if template.name.strip() not in PROMPTS: + continue + print(f"Translating {template.name.strip()} to {lang}") + answer_choices = [] + if template.answer_choices is not None: + choices = template.answer_choices.split("|||") + for c in choices: + answer_choices.append(translate(lang, c.strip())) + or_jinja = template.jinja.strip() + jinja = normalize_string(translate(lang, or_jinja), or_jinja) + template_name = template.name.strip() + f"_{lang}mt" + target_template = Template( + template_name, jinja=jinja, reference="", answer_choices=" ||| ".join(answer_choices) + ) + target_templates.add_template(target_template)