clue prompt templates

promptsource/templates/clue/afqmc/templates.yaml

@@ -0,0 +1,69 @@
+dataset: clue
+subset: afqmc
+templates:
+  997437fd-6888-482d-95e9-ffd867b497ee: !Template
+    answer_choices: no ||| yes
+    id: 997437fd-6888-482d-95e9-ffd867b497ee
+    jinja: 'Do "{{ sentence1 }}" and "{{ sentence2 }}" express the same thing?
+
+      |||
+
+      {{ answer_choices[label] }}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      languages:
+      - zh
+      metrics:
+      - Accuracy
+      original_task: true
+    name: express_same_yes_no
+    reference: ''
+  a28370c0-d43b-405c-a9b1-4d77b3a27244: !Template
+    answer_choices: no ||| yes
+    id: a28370c0-d43b-405c-a9b1-4d77b3a27244
+    jinja: "\"{{ sentence1 }}\" and \"{{ sentence2 }}\" have the same meaning. Would\
+      \ you agree? Answer yes or no. \n|||\n{{ answer_choices[label] }}"
+    metadata: !TemplateMetadata
+      choices_in_prompt: true
+      languages:
+      - zh
+      metrics:
+      - Accuracy
+      original_task: true
+    name: same_meaning_agree
+    reference: ''
+  d8c303a6-61a4-47f9-8623-cc72cc3294eb: !Template
+    answer_choices: null
+    id: d8c303a6-61a4-47f9-8623-cc72cc3294eb
+    jinja: 'Generate another sentence that has the same meaning as "{{ sentence1 }}".
+
+      |||
+
+      {% if label == 1 %}
+
+      {{ sentence2}}
+
+      {% endif %}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      languages:
+      - zh
+      metrics:
+      - ROUGE
+      original_task: false
+    name: generate_similar_sentence
+    reference: ''
+  e3fcaefd-4e8e-4491-aab7-8efeb67a2909: !Template
+    answer_choices: no ||| yes
+    id: e3fcaefd-4e8e-4491-aab7-8efeb67a2909
+    jinja: "Sentence 1: {{ sentence1 }}\nSentence 2: {{ sentence2 }}\nAre the two\
+      \ sentences similar? Yes or no? \n|||\n{{ answer_choices[label] }}"
+    metadata: !TemplateMetadata
+      choices_in_prompt: true
+      languages:
+      - zh
+      metrics:
+      - Accuracy
+      original_task: true
+    name: is_similar_yes_no
+    reference: ''

promptsource/templates/clue/c3/templates.yaml

@@ -0,0 +1,65 @@
+dataset: clue
+subset: c3
+templates:
+  51b3c3fe-2fa2-474a-81f9-5b421c884109: !Template
+    answer_choices: '{{ choice | join(" ||| ") }}'
+    id: 51b3c3fe-2fa2-474a-81f9-5b421c884109
+    jinja: "{% for statement in context %} \n{{ statement }}\n{% endfor %}\nGiven\
+      \ the dialogue / passage above, use the following options to answer the question\
+      \ \"{{question}}\".\nOptions: \n- {{ answer_choices | join('\\n- ') }}\n|||\n\
+      {{ answer }}"
+    metadata: !TemplateMetadata
+      choices_in_prompt: true
+      languages:
+      - zh
+      metrics:
+      - Accuracy
+      original_task: true
+    name: answer-question-affirmative
+    reference: ''
+  5e06f05f-d7dd-4329-b6d8-3a62dcdba838: !Template
+    answer_choices: '{{ choice | join(" ||| ") }}'
+    id: 5e06f05f-d7dd-4329-b6d8-3a62dcdba838
+    jinja: "Question: \"{{question}}\"\nAnswer choices: {{ answer_choices[:-1] | join(',\
+      \ ') }}, or  {{ answer_choices[-1] }}?\nPassage: {% for statement in context\
+      \ %} \n{{ statement }}\n{% endfor %}\n|||\n{{ answer }}"
+    metadata: !TemplateMetadata
+      choices_in_prompt: true
+      languages:
+      - zh
+      metrics:
+      - Accuracy
+      original_task: true
+    name: question_choices_context
+    reference: ''
+  63b5e5df-40d3-47ee-b77e-bf385c042fa9: !Template
+    answer_choices: null
+    id: 63b5e5df-40d3-47ee-b77e-bf385c042fa9
+    jinja: "Passage: {% for statement in context %} \n{{ statement }}\n{% endfor %}\n\
+      What kind of question would elicit an answer response of {{ answer }}?\n|||\n\
+      {{ question }}"
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      languages:
+      - zh
+      metrics:
+      - ROUGE
+      original_task: false
+    name: generate_question
+    reference: ''
+  a5820d05-a8df-4e31-a284-6969e478174b: !Template
+    answer_choices: '{{ choice | join('' ||| '') }}'
+    id: a5820d05-a8df-4e31-a284-6969e478174b
+    jinja: "Given the dialogue / passage below, what is the answer for the question\
+      \ \"{{question}}\"\nAnswer choices: {{ answer_choices[:-1] | join(', ') }},\
+      \ or  {{ answer_choices[-1] }}?\n{% for statement in context %} \n{{ statement\
+      \ }}\n{% endfor %}\n|||\n{{ answer }}"
+    metadata: !TemplateMetadata
+      choices_in_prompt: true
+      languages:
+      - zh
+      metrics:
+      - Accuracy
+      original_task: true
+    name: answer-question-interrogative
+    reference: ''

promptsource/templates/clue/cluewsc2020/templates.yaml

@@ -0,0 +1,78 @@
+dataset: clue
+subset: cluewsc2020
+templates:
+  321f55bb-c725-4fbf-bb7e-d46ea2f510b8: !Template
+    answer_choices: correct ||| wrong
+    id: 321f55bb-c725-4fbf-bb7e-d46ea2f510b8
+    jinja: 'In the class, a teacher asks what  the word "{{ target[''span2_text'']
+      }}" refers to in the text of "{{ text }}". The student answers "{{ target[''span1_text'']
+      }}". The teacher would say what? {{ answer_choices[0] | capitalize }} or {{answer_choices[1]
+      }}?
+
+      |||
+
+      {{ answer_choices[label] }}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: true
+      languages:
+      - zh
+      metrics:
+      - Accuracy
+      original_task: true
+    name: teacher_asking_student
+    reference: ''
+  7282b4b5-f854-42af-8e75-d509608d97bb: !Template
+    answer_choices: null
+    id: 7282b4b5-f854-42af-8e75-d509608d97bb
+    jinja: 'What does the word "{{ target[''span2_text''] }}" refers to in the text
+      of "{{ text }}"?
+
+      |||
+
+      {% if label == 0 %}
+
+      {{ target[''span1_text''] }}
+
+      {% endif %}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      languages:
+      - zh
+      metrics:
+      - ROUGE
+      original_task: false
+    name: generate_correct_response
+    reference: ''
+  e649a609-f7b2-43da-800d-a32090e92221: !Template
+    answer_choices: yes ||| no
+    id: e649a609-f7b2-43da-800d-a32090e92221
+    jinja: "In the sentence \"{{ text }}\", does \"{{ target['span2_text'] }}\" refer\
+      \ to \"{{ target['span1_text'] }}\"? \n|||\n{{ answer_choices[label] }}"
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      languages:
+      - zh
+      metrics:
+      - Accuracy
+      original_task: true
+    name: are_they_same
+    reference: ''
+  fc436a38-d9f5-4d17-bcf8-1e506bba5681: !Template
+    answer_choices: yes ||| no
+    id: fc436a38-d9f5-4d17-bcf8-1e506bba5681
+    jinja: 'In the sentence "{{ text }}", the word "{{ target[''span2_text''] }}"
+      refers to "{{ target[''span1_text''] }}". Answer {{ answer_choices[0] }} if
+      you agree; otherwise, answer {{ answer_choices[1] }}.
+
+      |||
+
+      {{ answer_choices[label] }}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: true
+      languages:
+      - zh
+      metrics:
+      - Accuracy
+      original_task: true
+    name: affirmative_are_they_same
+    reference: ''

promptsource/templates/clue/cmrc2018/templates.yaml

@@ -0,0 +1,76 @@
+dataset: clue
+subset: cmrc2018
+templates:
+  3bba02e6-9266-418b-9ba0-4f71755cf3b6: !Template
+    answer_choices: null
+    id: 3bba02e6-9266-418b-9ba0-4f71755cf3b6
+    jinja: 'Given this context "{{ context }}", generate a question that would return
+      the answer of "{{ answers[''text''][0] }}".
+
+      |||
+
+      {{ question }} '
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      languages:
+      - zh
+      metrics:
+      - ROUGE
+      original_task: false
+    name: generate_question
+    reference: ''
+  8fe02215-7881-4a61-a6e7-579680e40b9b: !Template
+    answer_choices: null
+    id: 8fe02215-7881-4a61-a6e7-579680e40b9b
+    jinja: "In an exam, you are asked {{ question }}, and you are tasked to find the\
+      \ answer from the following passage. \n{{ context }}\nWhat's the answer?\n|||\n\
+      {{ answers['text'][0] }}"
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      languages:
+      - zh
+      metrics:
+      - Squad
+      original_task: true
+    name: in_an_exam
+    reference: ''
+  9e82f5da-b206-4758-94e6-085cf2608378: !Template
+    answer_choices: null
+    id: 9e82f5da-b206-4758-94e6-085cf2608378
+    jinja: '{{ context }}
+
+      The answer to {{ question }} is in the passage above. What is it?
+
+      |||
+
+      {{ answers[''text''][0] }}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      languages:
+      - zh
+      metrics:
+      - Squad
+      original_task: true
+    name: answer_in_the_passage
+    reference: ''
+  9fb15385-814e-419a-b862-2d4e06a58ef6: !Template
+    answer_choices: null
+    id: 9fb15385-814e-419a-b862-2d4e06a58ef6
+    jinja: 'Answer the question using the given context.
+
+      Question: {{ question }}
+
+      Context: {{ context }}
+
+      |||
+
+      {{ answers[''text''][0] }}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      languages:
+      - zh
+      metrics:
+      - Squad
+      original_task: true
+    name: answer_following_question
+    reference: ''

promptsource/templates/clue/csl/templates.yaml

@@ -0,0 +1,90 @@
+dataset: clue
+subset: csl
+templates:
+  219679f8-a02f-4ee3-91c7-9ed4726dd828: !Template
+    answer_choices: no ||| yes
+    id: 219679f8-a02f-4ee3-91c7-9ed4726dd828
+    jinja: 'After John wrote the abstract "{{abst}}", he wrote these keywords "{{
+      keyword | join('', '') }}". Do you think his choice of keywords was correct?
+      Answer {{ answer_choices[1]}} or {{ answer_choices[0]}}.
+
+      |||
+
+      {{ answer_choices[label] }}
+
+
+      '
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      languages:
+      - zh
+      metrics:
+      - Accuracy
+      original_task: true
+    name: write_keywords_after_abstract
+    reference: ''
+  2e851dd2-2677-415a-ad90-5d885aa91fdc: !Template
+    answer_choices: no ||| yes
+    id: 2e851dd2-2677-415a-ad90-5d885aa91fdc
+    jinja: 'Do these keywords "{{ keyword | join('', '') }}" represent key concepts
+      in the abstract "{{ abst }}"?
+
+      |||
+
+      {{ answer_choices[label] }}
+
+
+      '
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      languages:
+      - zh
+      metrics:
+      - Accuracy
+      original_task: true
+    name: do_represent
+    reference: ''
+  2fa6151b-a296-4cd5-83ca-dcd434b831be: !Template
+    answer_choices: null
+    id: 2fa6151b-a296-4cd5-83ca-dcd434b831be
+    jinja: 'Given the abstract {{abst}}, list out {{ keyword | length }} keywords
+      for it.
+
+      |||
+
+      {% if label == 1 %}
+
+      {{ keyword | join('', '') }}
+
+      {% endif %}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      languages:
+      - zh
+      metrics:
+      - ROUGE
+      original_task: false
+    name: generate_keywords
+    reference: ''
+  aaf47f6f-fd8f-4180-8d85-e4c7df088ac6: !Template
+    answer_choices: no ||| yes
+    id: aaf47f6f-fd8f-4180-8d85-e4c7df088ac6
+    jinja: 'A scholar used "{{ keyword | join('', '') }}" as search terms. Do you
+      think the search engine would return the abstract "{{abst}}"? Answer {{ answer_choices[1]
+      }} or {{ answer_choices[0] }}.
+
+      |||
+
+      {{ answer_choices[label] }}
+
+
+      '
+    metadata: !TemplateMetadata
+      choices_in_prompt: true
+      languages:
+      - zh
+      metrics:
+      - Accuracy
+      original_task: true
+    name: search_terms
+    reference: ''