diff --git a/PacletInfo.wl b/PacletInfo.wl index 830c9f3e..96c4912b 100644 --- a/PacletInfo.wl +++ b/PacletInfo.wl @@ -1,7 +1,7 @@ PacletObject[ <| "Name" -> "Wolfram/Chatbook", "PublisherID" -> "Wolfram", - "Version" -> "1.3.1", + "Version" -> "1.3.2", "WolframVersion" -> "13.3+", "Description" -> "Wolfram Notebooks + LLMs", "License" -> "MIT", @@ -11,7 +11,8 @@ PacletObject[ <| "ReleaseDate" -> "$RELEASE_DATE$", "ReleaseURL" -> "$RELEASE_URL$", "ActionURL" -> "$ACTION_URL$", - "Loading" -> "Startup", + "CommitURL" -> "$COMMIT_URL$", + "Loading" -> "Startup", "PrimaryContext" -> "Wolfram`Chatbook`", "Extensions" -> { (* NOTE: The BeginStartup and EndStartup contexts are special, and need to diff --git a/Scripts/Common.wl b/Scripts/Common.wl index d5c95301..b2ad9cc8 100644 --- a/Scripts/Common.wl +++ b/Scripts/Common.wl @@ -107,6 +107,13 @@ messageString[ ___ ] := "-- Message text not found --"; (* ::Section::Closed:: *) (*Definitions*) +$envSHA = SelectFirst[ + { Environment[ "GITHUB_SHA" ], Environment[ "BUILD_VCS_NUMBER_WolframLanguage_Paclets_Chatbook_PacChatbook" ] }, + StringQ +]; + +$inCICD = StringQ @ $envSHA; + (* ::**************************************************************************************************************:: *) (* ::Subsection::Closed:: *) (*gitCommand*) @@ -124,24 +131,24 @@ gitCommand[ cmd_ ] := gitCommand[ cmd, Directory[ ] ]; (* ::**************************************************************************************************************:: *) (* ::Subsection::Closed:: *) (*releaseID*) -releaseID[ dir_ ] := - With[ { sha = Environment[ "GITHUB_SHA" ] }, - If[ StringQ @ sha, - sha, - gitCommand[ { "rev-parse", "HEAD" }, dir ] - ] - ]; +releaseID[ dir_ ] := FirstCase[ + Unevaluated @ { $envSHA, gitCommand[ { "rev-parse", "HEAD" }, dir ] }, + expr_ :> With[ { id = expr }, id /; StringQ @ id ], + "None" +]; (* ::**************************************************************************************************************:: *) (* ::Subsection::Closed:: *) (*releaseURL*) -releaseURL[ file_ ] := +releaseURL[ file_ ] := Enclose[ Enclose @ Module[ { pac, repo, ver }, pac = PacletObject @ Flatten @ File @ file; repo = ConfirmBy[ Environment[ "GITHUB_REPOSITORY" ], StringQ ]; ver = ConfirmBy[ pac[ "Version" ], StringQ ]; TemplateApply[ "https://github.com/`1`/releases/tag/v`2`", { repo, ver } ] - ]; + ], + "None" & +]; (* ::**************************************************************************************************************:: *) (* ::Subsection::Closed:: *) @@ -154,35 +161,39 @@ actionURL[ ] := Enclose[ runID = cs @ Environment[ "GITHUB_RUN_ID" ]; cs @ URLBuild @ { domain, repo, "actions", "runs", runID } ], - "$ACTION_URL$" & + "None" & ]; (* ::**************************************************************************************************************:: *) (* ::Subsection::Closed:: *) (*updatePacletInfo*) -updatePacletInfo[ dir_ ] /; StringQ @ Environment[ "GITHUB_ACTION" ] := Enclose[ +updatePacletInfo[ dir_ ] /; $inCICD := Enclose[ Module[ { cs, file, string, id, date, url, run, cmt, new }, - cs = ConfirmBy[ #, StringQ ] &; - file = cs @ FileNameJoin @ { dir, "PacletInfo.wl" }; - string = cs @ ReadString @ file; - id = cs @ releaseID @ dir; - date = cs @ DateString[ "ISODateTime", TimeZone -> 0 ]; + cs = ConfirmBy[ Echo[ #1, "Update PacletInfo [" <> ToString @ #2 <> "]: " ], StringQ, #2 ] &; + file = cs[ FileNameJoin @ { dir, "PacletInfo.wl" }, "Original PacletInfo" ]; + string = cs[ ReadString @ file, "ReadString" ]; + id = cs[ releaseID @ dir, "ReleaseID" ]; + date = cs[ DateString[ "ISODateTime", TimeZone -> 0 ], "Timestamp" ]; date = StringTrim[ date, "Z" ] <> "Z"; - url = cs @ releaseURL @ file; - run = cs @ actionURL[ ]; - cmt = cs @ commitURL @ id; - - new = cs @ StringReplace[ - string, - { - "\r\n" -> "\n", - "$RELEASE_ID$" -> id, - "$RELEASE_DATE$" -> date, - "$RELEASE_URL$" -> url, - "$ACTION_URL$" -> run - } + url = cs[ releaseURL @ file, "ReleaseURL" ]; + run = cs[ actionURL[ ], "ActionURL" ]; + cmt = cs[ commitURL @ id, "CommitURL" ]; + + new = cs[ + StringReplace[ + string, + { + "\r\n" -> "\n", + "$RELEASE_ID$" -> id, + "$RELEASE_DATE$" -> date, + "$RELEASE_URL$" -> url, + "$ACTION_URL$" -> run, + "$COMMIT_URL$" -> cmt + } + ], + "Updated PacletInfo" ]; Print[ "Updating PacletInfo" ]; @@ -190,6 +201,7 @@ updatePacletInfo[ dir_ ] /; StringQ @ Environment[ "GITHUB_ACTION" ] := Enclose[ Print[ " ReleaseDate: ", date ]; Print[ " ReleaseURL: ", url ]; Print[ " ActionURL: ", run ]; + Print[ " CommitURL: ", cmt ]; Confirm @ WithCleanup[ BinaryWrite[ file, new ], Close @ file @@ -200,7 +212,8 @@ updatePacletInfo[ dir_ ] /; StringQ @ Environment[ "GITHUB_ACTION" ] := Enclose[ ], Function[ Print[ "::error::Failed to update PacletInfo template parameters." ]; - Exit[ 1 ] + Print[ " ", ToString[ #, InputForm ] ]; + If[ StringQ @ Environment[ "GITHUB_ACTION" ], Exit[ 1 ] ] ] ]; @@ -224,12 +237,7 @@ updateReleaseInfoCell[ dir_, url_, cmt_, run_ ] /; ]; -commitURL[ sha_String ] := Enclose @ URLBuild @ { - "https://github.com", - ConfirmBy[ Environment[ "GITHUB_REPOSITORY" ], StringQ ], - "commit", - sha -}; +commitURL[ sha_String ] := URLBuild @ { "https://github.com/WolframResearch/Chatbook/commit", sha }; releaseInfoCell[ release_, commit_, run_ ] := Enclose[ diff --git a/Source/Chatbook/Actions.wl b/Source/Chatbook/Actions.wl index 8fee9240..0eef2b55 100644 --- a/Source/Chatbook/Actions.wl +++ b/Source/Chatbook/Actions.wl @@ -666,8 +666,13 @@ revertMultimodalContent // beginDefinition; revertMultimodalContent[ messages_List ] := revertMultimodalContent /@ messages; -revertMultimodalContent[ as: KeyValuePattern[ "Content" -> content_List ] ] := - <| as, "Content" -> StringJoin @ Select[ content, StringQ ] |>; +revertMultimodalContent[ as: KeyValuePattern[ "Content" -> content_List ] ] := <| + as, + "Content" -> StringJoin @ Cases[ + content, + s_String | KeyValuePattern @ { "Type" -> "Text", "Data" -> s_String } :> s + ] +|>; revertMultimodalContent[ as: KeyValuePattern[ "Content" -> _String ] ] := as; diff --git a/Source/Chatbook/ChatMessages.wl b/Source/Chatbook/ChatMessages.wl index 7e1ef2e5..0ec16383 100644 --- a/Source/Chatbook/ChatMessages.wl +++ b/Source/Chatbook/ChatMessages.wl @@ -386,7 +386,7 @@ tokenCount // endDefinition; applyTokenizer // beginDefinition; applyTokenizer[ tokenizer_, content_String ] := tokenizer @ content; applyTokenizer[ tokenizer_, content_? graphicsQ ] := tokenizer @ content; -applyTokenizer[ tokenizer_, content_List ] := Flatten[ tokenizer /@ content ]; +applyTokenizer[ tokenizer_, content_List ] := Flatten[ applyTokenizer[ tokenizer, # ] & /@ content ]; applyTokenizer[ tokenizer_, KeyValuePattern[ "Data" -> data_ ] ] := tokenizer @ data; applyTokenizer // endDefinition; @@ -658,7 +658,7 @@ makeMessageContent // endDefinition; expandMultimodalString // beginDefinition; expandMultimodalString[ string_String ] /; $multimodalMessages := Enclose[ - Module[ { split, joined }, + Module[ { split, joined, typed }, split = Flatten @ StringSplit[ string, @@ -675,8 +675,9 @@ expandMultimodalString[ string_String ] /; $multimodalMessages := Enclose[ } ]; - joined = FixedPoint[ Replace[ { a___, b_String, c_String, d___ } :> { a, b<>c, d } ], split ]; - Replace[ joined, { msg_String } :> msg ] + joined = Flatten @ Replace[ SplitBy[ split, StringQ ], s: { _String, ___ } :> StringJoin @ s, { 1 } ]; + typed = ConfirmMatch[ inferMultimodalTypes @ joined, { ___? AssociationQ } | { ___? StringQ }, "Typed" ]; + Replace[ typed, { msg_String } :> msg ] ], throwInternalFailure[ expandMultimodalString @ string, ## ] & ]; @@ -686,6 +687,44 @@ expandMultimodalString[ string_String ] := expandMultimodalString // endDefinition; +(* ::**************************************************************************************************************:: *) +(* ::Subsubsubsection::Closed:: *) +(*inferMultimodalTypes*) +inferMultimodalTypes // beginDefinition; + +inferMultimodalTypes[ content_List ] := Enclose[ + Module[ { typed }, + typed = ConfirmMatch[ inferMultimodalTypes0 @ content, { ___? AssociationQ }, "Typed" ]; + If[ MatchQ[ typed, { KeyValuePattern[ "Type" -> "Text" ] .. } ], + ConfirmMatch[ Lookup[ typed, "Data" ], { __String }, "TextData" ], + typed + ] + ], + throwInternalFailure +]; + +inferMultimodalTypes // endDefinition; + +inferMultimodalTypes0 // beginDefinition; +inferMultimodalTypes0[ content_List ] := inferMultimodalTypes0 /@ content; +inferMultimodalTypes0[ content_String ] := <| "Type" -> "Text" , "Data" -> content |>; +inferMultimodalTypes0[ content_? graphicsQ ] := <| "Type" -> "Image", "Data" -> ensureCompatibleImage @ content |>; +inferMultimodalTypes0 // endDefinition; + +(* ::**************************************************************************************************************:: *) +(* ::Subsubsubsection::Closed:: *) +(*ensureCompatibleImage*) +ensureCompatibleImage // beginDefinition; +ensureCompatibleImage[ img_ ] /; $useRasterizationCompatibility && ! Image`PossibleImageQ @ img := Rasterize @ img; +ensureCompatibleImage[ img_ ] := img; +ensureCompatibleImage // endDefinition; + + +$useRasterizationCompatibility := Enclose[ + $useRasterizationCompatibility = + ! PacletNewerQ[ ConfirmBy[ PacletObject[ "ServiceConnection_OpenAI" ], PacletObjectQ ], "13.3.18" ] +]; + (* ::**************************************************************************************************************:: *) (* ::Subsubsection::Closed:: *) (*expressionURIQ*) diff --git a/Source/Chatbook/SendChat.wl b/Source/Chatbook/SendChat.wl index b3fc9368..3eef60e9 100644 --- a/Source/Chatbook/SendChat.wl +++ b/Source/Chatbook/SendChat.wl @@ -673,12 +673,12 @@ withFETasks // endDefinition; (*writeChunk*) writeChunk // beginDefinition; -writeChunk[ container_, cell_, KeyValuePattern[ "BodyChunkProcessed" -> chunk_String ] ] := - writeChunk0[ container, cell, chunk, chunk ]; - -writeChunk[ container_, cell_, KeyValuePattern[ "BodyChunkProcessed" -> { chunks___String } ] ] := - With[ { chunk = StringJoin @ chunks }, - writeChunk0[ container, cell, chunk, chunk ] +writeChunk[ container_, cell_, KeyValuePattern[ "BodyChunkProcessed" -> chunks_ ] ] := + With[ { chunk = StringJoin @ Select[ Flatten @ { chunks }, StringQ ] }, + If[ chunk === "", + Null, + writeChunk0[ container, cell, chunk, chunk ] + ] ]; (* TODO: this definition is obsolete once LLMServices is widely available: *) @@ -968,7 +968,7 @@ toolFreeQ // endDefinition; toolEvaluation // beginDefinition; toolEvaluation[ settings_, container_Symbol, cell_, as_Association ] := Enclose[ - Module[ { string, callPos, toolCall, toolResponse, output, messages, newMessages, req, toolID }, + Module[ { string, callPos, toolCall, toolResponse, output, messages, newMessages, req, toolID, task }, string = ConfirmBy[ container[ "FullContent" ], StringQ, "FullContent" ]; @@ -1010,7 +1010,18 @@ toolEvaluation[ settings_, container_Symbol, cell_, as_Association ] := Enclose[ appendToolResult[ container, output, toolID ]; - $lastTask = chatSubmit[ container, req, cell, settings ] + task = $lastTask = chatSubmit[ container, req, cell, settings ]; + + addHandlerArguments[ "Task" -> task ]; + + CurrentValue[ cell, { TaggingRules, "ChatNotebookSettings", "CellObject" } ] = cell; + CurrentValue[ cell, { TaggingRules, "ChatNotebookSettings", "Task" } ] = task; + + If[ FailureQ @ task, throwTop @ writeErrorCell[ cell, task ] ]; + + If[ task === $Canceled, StopChat @ cell ]; + + task ], throwInternalFailure[ toolEvaluation[ settings, container, cell, as ], ## ] & ]; @@ -1356,10 +1367,39 @@ multimodalPacletsAvailable[ ] := multimodalPacletsAvailable[ ] = ( ); multimodalPacletsAvailable[ llmFunctions_PacletObject? PacletObjectQ, openAI_PacletObject? PacletObjectQ ] := - TrueQ @ And[ PacletNewerQ[ llmFunctions, "1.2.4" ], PacletNewerQ[ openAI, "13.3.18" ] ]; + TrueQ @ And[ + PacletNewerQ[ llmFunctions, "1.2.4" ], + Or[ PacletNewerQ[ openAI, "13.3.18" ], + openAI[ "Version" ] === "13.3.18" && multimodalOpenAIQ @ openAI + ] + ]; multimodalPacletsAvailable // endDefinition; +(* ::**************************************************************************************************************:: *) +(* ::Subsubsubsection::Closed:: *) +(*multimodalOpenAIQ*) +multimodalOpenAIQ // beginDefinition; + +multimodalOpenAIQ[ openAI_PacletObject ] := Enclose[ + Catch @ Module[ { dir, file, multimodal }, + + dir = ConfirmBy[ openAI[ "Location" ], DirectoryQ, "Location" ]; + file = ConfirmBy[ FileNameJoin @ { dir, "Kernel", "OpenAI.m" }, FileExistsQ, "File" ]; + + multimodal = WithCleanup[ + Quiet @ Close @ file, + ConfirmMatch[ Find[ file, "data:image/jpeg;base64," ], _String? StringQ | EndOfFile, "Find" ], + Quiet @ Close @ file + ]; + + StringQ @ multimodal + ], + throwInternalFailure +]; + +multimodalOpenAIQ // endDefinition; + (* ::**************************************************************************************************************:: *) (* ::Subsubsection::Closed:: *) (*getLLMEvaluator*) diff --git a/Source/Chatbook/Tools.wl b/Source/Chatbook/Tools.wl index 73fd63a0..9549e080 100644 --- a/Source/Chatbook/Tools.wl +++ b/Source/Chatbook/Tools.wl @@ -387,7 +387,7 @@ getCachedToolName // endDefinition; (*getToolSelections*) getToolSelections // beginDefinition; getToolSelections[ as_Association ] := getToolSelections[ as, Lookup[ as, "ToolSelections", <| |> ] ]; -getToolSelections[ as_, selections_Association ] := selections; +getToolSelections[ as_, selections_Association ] := KeyTake[ selections, Keys @ $AvailableTools ]; getToolSelections[ as_, Except[ _Association ] ] := <| |>; getToolSelections // endDefinition; @@ -396,7 +396,7 @@ getToolSelections // endDefinition; (*getToolSelectionTypes*) getToolSelectionTypes // beginDefinition; getToolSelectionTypes[ as_Association ] := getToolSelectionTypes[ as, Lookup[ as, "ToolSelectionType", <| |> ] ]; -getToolSelectionTypes[ as_, selections_Association ] := selections; +getToolSelectionTypes[ as_, selections_Association ] := KeyTake[ selections, Keys @ $AvailableTools ]; getToolSelectionTypes[ as_, Except[ _Association ] ] := <| |>; getToolSelectionTypes // endDefinition; @@ -542,8 +542,20 @@ makeToolPrompt[ settings_Association ] := $lastToolPrompt = TemplateObject[ DeleteMissing @ { $toolPre, TemplateSequence[ - TemplateExpression @ StringTemplate[ - "Tool Name: `Name`\nDescription: `Description`\nSchema:\n`Schema`\n\n" + TemplateExpression @ TemplateObject[ + { + "Tool Name: ", + TemplateSlot[ "Name" ], + "\nDisplay Name: ", + TemplateSlot[ "DisplayName", DefaultValue :> toDisplayToolName @ TemplateSlot[ "Name" ] ], + "\nDescription: ", + TemplateSlot[ "Description" ], + "\nSchema:\n", + TemplateSlot[ "Schema" ], + "\n\n" + }, + CombinerFunction -> StringJoin, + InsertionFunction -> TextString ], TemplateExpression @ Map[ Append[ #[ "Data" ], "Schema" -> ExportString[ #[ "JSONSchema" ], "JSON" ] ] &, @@ -596,7 +608,11 @@ If a user asks you to use a specific tool, you MUST attempt to use that tool as even if you think it will not work. \ If the tool fails, use any error message to correct the issue or explain why it failed. \ NEVER state that a tool cannot be used for a particular task without trying it first. \ -You did not create these tools, so you do not know what they can and cannot do."; +You did not create these tools, so you do not know what they can and cannot do. + +You should try to avoid mentioning tools by name in your response and instead speak generally about their function. \ +For example, if there were a number_adder tool, you would instead talk about \"adding numbers\". If you must mention \ +a tool by name, you should use the DisplayName property instead of the tool name."; (* ::**************************************************************************************************************:: *) (* ::Subsubsubsection::Closed:: *) @@ -1250,11 +1266,18 @@ webSearch0 // endDefinition; $webSearchResultTemplate := StringTemplate @ StringJoin[ "Results\n-------\n\n`1`\n\n-------", If[ KeyExistsQ[ $selectedTools, "WebFetcher" ], - "\n\nUse the web_fetcher tool to get the content of a URL.", + $webSearchFetchPrompt, "" ] ]; +$webSearchFetchPrompt = " + +Important: The snippet text is not enough information to write an informed response! If there are any relevant \ +results, you should now immediately use the web_fetcher tool to retrieve them before responding. Do not ask the user \ +for permission first. If it made sense to use the web_searcher tool, it's also implied that you should use the \ +web_fetcher tool."; + (* ::**************************************************************************************************************:: *) (* ::Subsection::Closed:: *) (*WebFetch*) diff --git a/Source/Chatbook/Utils.wl b/Source/Chatbook/Utils.wl index c82d9a9c..3ca350ca 100644 --- a/Source/Chatbook/Utils.wl +++ b/Source/Chatbook/Utils.wl @@ -336,15 +336,19 @@ uriData // endDefinition; exportDataURI // beginDefinition; exportDataURI[ data_ ] := - exportDataURI[ data, guessExpressionMimeType @ data ]; + With[ { mime = guessExpressionMimeType @ data }, + exportDataURI[ data, mimeTypeToFormat @ mime, mime ] + ]; + +exportDataURI[ data_, fmt_String ] := + exportDataURI[ data, fmt, formatToMIMEType @ fmt ]; -exportDataURI[ data_, fmt_String ] := Enclose[ - Module[ { mime, base64 }, - mime = ConfirmBy[ formatToMIMEType @ fmt, StringQ, "MIMEType" ]; +exportDataURI[ data_, fmt_String, mime_String ] := Enclose[ + Module[ { base64 }, base64 = ConfirmBy[ ExportString[ data, { "Base64", fmt } ], StringQ, "Base64" ]; "data:" <> mime <> ";base64," <> StringDelete[ base64, "\n" ] ], - throwInternalFailure[ exportDataURI[ data, fmt ], ## ] & + throwInternalFailure ]; exportDataURI // endDefinition; @@ -353,10 +357,11 @@ exportDataURI // endDefinition; (* ::Subsubsection::Closed:: *) (*guessExpressionMimeType*) guessExpressionMimeType // beginDefinition; -guessExpressionMimeType[ _? image2DQ ] := "image/jpeg"; -guessExpressionMimeType[ _? graphicsQ ] := "image/png"; -guessExpressionMimeType[ _String? StringQ ] := "text/plain"; -guessExpressionMimeType[ ___ ] := "application/octet-stream"; +guessExpressionMimeType[ _? image2DQ ] := "image/jpeg"; +guessExpressionMimeType[ _? graphicsQ ] := "image/png"; +guessExpressionMimeType[ _String? StringQ ] := "text/plain"; +guessExpressionMimeType[ _ByteArray? ByteArrayQ ] := "application/octet-stream"; +guessExpressionMimeType[ ___ ] := "application/vnd.wolfram.wl"; guessExpressionMimeType // endDefinition; (* ::**************************************************************************************************************:: *)