From 62db1a9e27967083a32bd944b0ec9e45db03ea4e Mon Sep 17 00:00:00 2001 From: YizhenJia Date: Thu, 19 Dec 2024 23:24:40 +0800 Subject: [PATCH] [usability] add qwen2.5 (qwq) template --- .../supported_conversation_template.md | 90 +++++++++++++++++++ .../utils/conversation_template/__init__.py | 3 +- .../utils/conversation_template/qwen.py | 52 +++++++++++ 3 files changed, 144 insertions(+), 1 deletion(-) diff --git a/docs/source/examples/supported_conversation_template.md b/docs/source/examples/supported_conversation_template.md index 1e565fe0b..4d5555a00 100644 --- a/docs/source/examples/supported_conversation_template.md +++ b/docs/source/examples/supported_conversation_template.md @@ -13,6 +13,7 @@ - [Mixtral 8x7B](#mixtral-8x7b) - [Phi-3](#phi-3) - [Qwen-2](#qwen-2) + - [Qwen-2.5](#qwen-25) - [Yi](#yi) - [Yi-1.5](#yi-15) - [Zephyr](#zephyr) @@ -405,6 +406,95 @@ The conversation template for Mixtral 8x7B is slightly different from the templa ``` +## Qwen-2.5 + +(Also QwQ) + +**With a system message** +``` +<|im_start|>system\n{{system_message}}<|im_end|>\n<|im_start|>user\n{{user_message_0}}<|im_end|>\n +``` + +**Without a system message** +``` +<|im_start|>user\n{{user_message_0}}<|im_end|>\n +``` + +**A complete conversation** +``` +<|im_start|>system\n{{system_message}}<|im_end|>\n<|im_start|>user\n{{user_message_0}}<|im_end|>\n<|im_start|>assistant\n{{assistant_reply_0}}<|im_end|>\n +``` + +**Multiple rounds** +``` +<|im_start|>system\n{{system_message}}<|im_end|>\n<|im_start|>user\n{{user_message_0}}<|im_end|>\n<|im_start|>assistant\n{{assistant_reply_0}}<|im_end|>\n<|im_start|>user\n{{user_message_1}}<|im_end|>\n<|im_start|>assistant\n{{assistant_reply_1}}<|im_end|>\n +``` + +**jinja template** +[[Reference]()] +``` +{%- if tools %} + {{- '<|im_start|>system\\n' }} + {%- if messages[0]['role'] == 'system' %} + {{- messages[0]['content'] }} + {%- else %} + {{- 'You are a helpful and harmless assistant. You are Qwen developed by Alibaba. You should think step-by-step.' }} + {%- endif %} + {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within XML tags:\\n\" }} + {%- for tool in tools %} + {{- \"\\n\" }} + {{- tool | tojson }} + {%- endfor %} + {{- \"\\n\\n\\nFor each function call, return a json object with function name and arguments within XML tags:\\n\\n{\\\"name\\\": , \\\"arguments\\\": }\\n<|im_end|>\\n\" }} +{%- else %} + {%- if messages[0]['role'] == 'system' %} + {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }} + {%- else %} + {{- '<|im_start|>system\\nYou are a helpful and harmless assistant. You are Qwen developed by Alibaba. You should think step-by-step.<|im_end|>\\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %} + {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }} + {%- elif message.role == \"assistant\" %} + {{- '<|im_start|>' + message.role }} + {%- if message.content %} + {{- '\\n' + message.content }} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\\n\\n{\"name\": \"' }} + {{- tool_call.name }} + {{- '\", \"arguments\": ' }} + {{- tool_call.arguments | tojson }} + {{- '}\\n' }} + {%- endfor %} + {{- '<|im_end|>\\n' }} + {%- elif message.role == \"tool\" %} + {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\\n\\n' }} + {{- message.content }} + {{- '\\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %} + {{- '<|im_end|>\\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\\n' }} +{%- endif %} +``` + +**Filled Example** +``` +<|im_start|>system\nYou are a chatbot developed by LMFlow team.<|im_end|>\n<|im_start|>user\nWho are you?<|im_end|>\n<|im_start|>assistant\nI am a chatbot developed by LMFlow team.<|im_end|>\n<|im_start|>user\nHow old are you?<|im_end|>\n<|im_start|>assistant\nI don't age like humans do. I exist as a piece of software, so I don't have a concept of age in the traditional sense.<|im_end|>\n +``` + + ## Yi **With a system message** ``` diff --git a/src/lmflow/utils/conversation_template/__init__.py b/src/lmflow/utils/conversation_template/__init__.py index 1c0d58aeb..ba6339175 100644 --- a/src/lmflow/utils/conversation_template/__init__.py +++ b/src/lmflow/utils/conversation_template/__init__.py @@ -10,7 +10,7 @@ from .internlm import INTERNLM2_TEMPLATE from .llama import LLAMA2_TEMPLATE, LLAMA3_TEMPLATE, LLAMA3_TEMPLATE_FOR_TOOL from .phi import PHI3_TEMPLATE -from .qwen import QWEN2_TEMPLATE, QWEN2_TEMPLATE_FOR_TOOL +from .qwen import QWEN2_TEMPLATE, QWEN2_TEMPLATE_FOR_TOOL, QWEN2_5_TEMPLATE from .yi import YI1_5_TEMPLATE from .zephyr import ZEPHYR_TEMPLATE @@ -31,6 +31,7 @@ 'phi3': PHI3_TEMPLATE, 'qwen2': QWEN2_TEMPLATE, 'qwen2_for_tool': QWEN2_TEMPLATE_FOR_TOOL, + 'qwen2_5': QWEN2_5_TEMPLATE, 'yi': CHATML_TEMPLATE, 'yi1_5': YI1_5_TEMPLATE, 'zephyr': ZEPHYR_TEMPLATE diff --git a/src/lmflow/utils/conversation_template/qwen.py b/src/lmflow/utils/conversation_template/qwen.py index 88c8fb581..c2efdd7d1 100644 --- a/src/lmflow/utils/conversation_template/qwen.py +++ b/src/lmflow/utils/conversation_template/qwen.py @@ -1,6 +1,10 @@ #!/usr/bin/env python # coding=utf-8 # Copyright 2024 Statistics and Machine Learning Research Group. All rights reserved. +from typing import Dict, Set, Sequence, Literal, Union, List, Optional, Tuple + +from transformers import PreTrainedTokenizer + from .base import StringFormatter, TemplateComponent, ConversationTemplate, ConversationTemplateForTool @@ -24,6 +28,7 @@ separator=TemplateComponent(type='string', content='\n') ) + QWEN2_TEMPLATE_FOR_TOOL = ConversationTemplateForTool( template_name='qwen2_for_tool', user_formatter=StringFormatter( @@ -52,4 +57,51 @@ ] ), separator=TemplateComponent(type='string', content='\n') +) + + +class Qwen2_5ConversationTemplate(ConversationTemplateForTool): + def _handle_tools(self, tools: Optional[List[str]]) -> str: + prompt1 = "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" + prompt2 = '\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{"name": , "arguments": }\n' + tools_out = '' + + if tools is not None: + tools_out = prompt1 + for tool in tools: + tools_out += "\n" + tool + tools_out += prompt2 + + return tools_out + + +QWEN2_5_TEMPLATE = Qwen2_5ConversationTemplate( + template_name='qwen2_5', + user_formatter=StringFormatter( + template=[ + TemplateComponent(type='string', content='<|im_start|>user\n{{content}}<|im_end|>\n') + ] + ), + function_formatter=StringFormatter( + template=[ + TemplateComponent(type='string', content='<|im_start|>assistant\n{{content}}<|im_end|>\n') + ] + ), + observation_formatter=StringFormatter( + template=[ + TemplateComponent(type='string', content='<|im_start|>tool\n{{content}}<|im_end|>\n') + ] + ), + assistant_formatter=StringFormatter( + template=[ + TemplateComponent(type='string', content='<|im_start|>assistant\n{{content}}<|im_end|>\n') + ] + ), + system_formatter=StringFormatter( + template=[ + TemplateComponent(type='string', content='<|im_start|>system\n{{content}}<|im_end|>\n') + ] + ), + separator=TemplateComponent(type='string', content='\n'), + system_default='You are a helpful and harmless assistant. You are Qwen developed by Alibaba. You should think step-by-step.' ) \ No newline at end of file