diff --git a/src/fastapi_poe/base.py b/src/fastapi_poe/base.py index b9a17a5..96ec80c 100644 --- a/src/fastapi_poe/base.py +++ b/src/fastapi_poe/base.py @@ -44,6 +44,12 @@ SettingsResponse, ) +from fastapi_poe.templates import ( + IMAGE_VISION_ATTACHMENT_TEMPLATE, + TEXT_ATTACHMENT_TEMPLATE, + URL_ATTACHMENT_TEMPLATE +) + logger = logging.getLogger("uvicorn.default") @@ -102,6 +108,7 @@ async def http_exception_handler(request: Request, ex: Exception) -> Response: class PoeBot: path: str = "/" # Path where this bot will be exposed access_key: Optional[str] = None # Access key for this bot + concat_attachments_to_message: bool = True # attachment content will be concatenated to message # Override these for your bot @@ -302,6 +309,38 @@ async def _process_pending_attachment_requests( logger.error("Error processing pending attachment requests") raise + def concat_attachment_content_to_message_body( + self, query_request: QueryRequest + ) -> QueryRequest: + """Concat received attachment file content into message content body.""" + last_message = query_request.query[-1] + concatenated_content = last_message.content + for attachment in last_message.attachments: + if attachment.parsed_content: + if attachment.content_type == "text/html": + url_attachment_content = URL_ATTACHMENT_TEMPLATE.format( + attachment_name=attachment.name, + content=attachment.parsed_content, + ) + concatenated_content = f"{concatenated_content}\n\n{url_attachment_content}" + elif "text" in attachment.content_type: + text_attachment_content = TEXT_ATTACHMENT_TEMPLATE.format( + attachment_name=attachment.name, + attachment_parsed_content=attachment.parsed_content, + ) + concatenated_content = f"{concatenated_content}\n\n{text_attachment_content}" + elif "image" in attachment.content_type: + parsed_content_filename = attachment.parsed_content.split("***")[0] + parsed_content_text = attachment.parsed_content.split("***")[1] + image_attachment_content = IMAGE_VISION_ATTACHMENT_TEMPLATE.format( + filename=parsed_content_filename, + parsed_image_description=parsed_content_text, + ) + concatenated_content = f"{concatenated_content}\n\n{image_attachment_content}" + modified_last_message = last_message.model_copy(update={"content": concatenated_content}) + modified_query = query_request.model_copy(update={"query": query_request.query[:-1] + [modified_last_message]}) + return modified_query + @staticmethod def text_event(text: str) -> ServerSentEvent: return ServerSentEvent(data=json.dumps({"text": text}), event="text") @@ -378,6 +417,8 @@ async def handle_query( self, request: QueryRequest, context: RequestContext ) -> AsyncIterable[ServerSentEvent]: try: + if self.concat_attachments_to_message: + request = self.concat_attachment_content_to_message_body(query_request=request) async for event in self.get_response_with_context(request, context): if isinstance(event, ServerSentEvent): yield event diff --git a/src/fastapi_poe/templates.py b/src/fastapi_poe/templates.py new file mode 100644 index 0000000..6855875 --- /dev/null +++ b/src/fastapi_poe/templates.py @@ -0,0 +1,25 @@ +""" + +This module contains a collection of string templates designed to streamline the integration +of features like attachments into the LLM request. + +""" + +TEXT_ATTACHMENT_TEMPLATE = ( + """Your response must be in the language of the relevant queries related to the document.\n""" + """Below is the content of {attachment_name}:\n\n{attachment_parsed_content}""" +) +URL_ATTACHMENT_TEMPLATE = ( + """Assume you can access the external URL {attachment_name}. """ + """Your response must be in the language of the relevant queries related to the URL.\n""" + """Use the URL's content below to respond to the queries:\n\n{content}""" +) +IMAGE_VISION_ATTACHMENT_TEMPLATE = ( + """I have uploaded an image ({filename}). """ + """Assume that you can see the attached image. """ + """First, read the image analysis:\n\n""" + """{parsed_image_description}\n\n""" + """Use any relevant parts to inform your response. """ + """Do NOT reference the image analysis in your response. """ + """Respond in the same language as my next message. """ +) \ No newline at end of file diff --git a/src/fastapi_poe/types.py b/src/fastapi_poe/types.py index 88a819e..c36652c 100644 --- a/src/fastapi_poe/types.py +++ b/src/fastapi_poe/types.py @@ -21,6 +21,7 @@ class Attachment(BaseModel): url: str content_type: str name: str + parsed_content: Optional[str] = None class ProtocolMessage(BaseModel): @@ -93,6 +94,10 @@ class SettingsResponse(BaseModel): server_bot_dependencies: Dict[str, int] = Field(default_factory=dict) allow_attachments: bool = False introduction_message: str = "" + expand_text_attachments: bool = True + enable_image_comprehension: bool = False + enforce_author_role_alternation: bool = True + class AttachmentUploadResponse(BaseModel):