From 47e64c63a068202293a0f67d2258b113b704aa5a Mon Sep 17 00:00:00 2001 From: Rick Hennigan Date: Fri, 22 Nov 2024 17:11:35 -0500 Subject: [PATCH] Fixed several issues with the LLM being unable to properly reference audio/video expressions --- Source/Chatbook/Prompting.wl | 10 ++++++++++ Source/Chatbook/Sandbox.wl | 13 ++++++++++++- Source/Chatbook/SendChat.wl | 11 +++++++---- Source/Chatbook/Serialization.wl | 9 ++++++--- 4 files changed, 35 insertions(+), 8 deletions(-) diff --git a/Source/Chatbook/Prompting.wl b/Source/Chatbook/Prompting.wl index 72d9fd1b..218a4488 100644 --- a/Source/Chatbook/Prompting.wl +++ b/Source/Chatbook/Prompting.wl @@ -29,6 +29,8 @@ $basePromptOrder = { "ConversionGraphics", "MarkdownImageBox", "MarkdownImageBoxImporting", + "AudioBoxImporting", + "VideoBoxImporting", "Checkboxes", "CheckboxesIndeterminate", "ConversionFormatting", @@ -83,6 +85,8 @@ $basePromptDependencies = Append[ "GeneralInstructionsHeader" ] /@ <| "ConversionGraphics" -> { "MessageConversionHeader" }, "MarkdownImageBox" -> { "MessageConversionHeader" }, "MarkdownImageBoxImporting" -> { "MarkdownImageBox" }, + "AudioBoxImporting" -> { "MarkdownImageBoxImporting" }, + "VideoBoxImporting" -> { "MarkdownImageBoxImporting" }, "ConversionFormatting" -> { "MessageConversionHeader" }, "ExternalLanguageCells" -> { "MessageConversionHeader" }, "SpecialURI" -> { }, @@ -202,6 +206,12 @@ $basePromptComponents[ "MarkdownImageBoxImporting" ] = "\ tool. For example, ``ColorNegate[]``. The expression will be inserted in place. Do not \ include the MarkdownImageBox wrapper. You can also use this syntax to inline images into WL code blocks."; +$basePromptComponents[ "AudioBoxImporting" ] = "\ + * You can also use this syntax for audio that similarly appears in AudioBox[...]."; + +$basePromptComponents[ "VideoBoxImporting" ] = "\ + * You can also use this syntax for video that similarly appears in VideoBox[...]."; + $basePromptComponents[ "Checkboxes" ] = "\ * Checkboxes in the UI will be replaced with one of the following text representations: * A Checkbox that's selected becomes ``[\[Checkmark]]`` diff --git a/Source/Chatbook/Sandbox.wl b/Source/Chatbook/Sandbox.wl index b6a522aa..7960c100 100644 --- a/Source/Chatbook/Sandbox.wl +++ b/Source/Chatbook/Sandbox.wl @@ -2070,7 +2070,7 @@ inlineExpressionURI[ uri_String ] := inlineExpressionURI[ uri_, failed_? FailureQ ] := makeCachedBoxes @ failed; -inlineExpressionURI[ uri_, HoldComplete[ expr_ ] ] /; graphicsQ @ Unevaluated @ expr := +inlineExpressionURI[ uri_, HoldComplete[ expr_? makeCachedBoxesQ ] ] := makeCachedBoxes @ expr; inlineExpressionURI[ uri_, HoldComplete[ expr_ ] ] := @@ -2083,6 +2083,17 @@ inlineExpressionURI[ uri_, HoldComplete[ expr_ ] ] := inlineExpressionURI // endDefinition; +(* ::**************************************************************************************************************:: *) +(* ::Subsubsection::Closed:: *) +(*makeCachedBoxesQ*) +makeCachedBoxesQ // beginDefinition; +makeCachedBoxesQ // Attributes = { HoldAllComplete }; +makeCachedBoxesQ[ audio_Audio ] := AudioQ @ Unevaluated @ audio; +makeCachedBoxesQ[ video_Video ] := VideoQ @ Unevaluated @ video; +makeCachedBoxesQ[ graph_Graph ] := GraphQ @ Unevaluated @ graph; +makeCachedBoxesQ[ expr_ ] := graphicsQ @ Unevaluated @ expr; +makeCachedBoxesQ // endDefinition; + (* ::**************************************************************************************************************:: *) (* ::Subsubsection::Closed:: *) (*makeCachedBoxes*) diff --git a/Source/Chatbook/SendChat.wl b/Source/Chatbook/SendChat.wl index 66767f0f..9a81ee7b 100644 --- a/Source/Chatbook/SendChat.wl +++ b/Source/Chatbook/SendChat.wl @@ -894,14 +894,17 @@ autoCorrect // beginDefinition; autoCorrect[ string_String ] := StringReplace[ string, $llmAutoCorrectRules ]; autoCorrect // endDefinition; + +$$specialBoxName = "AudioBox"|"MarkdownImageBox"|"VideoBox"; + $llmAutoCorrectRules := $llmAutoCorrectRules = Flatten @ { StartOfLine ~~ WhitespaceCharacter... ~~ "/command\ncode:" :> "/wl\ncode:", "```" ~~ code: Except[ "\n" ].. ~~ "```" :> "``"<>code<>"``", "wolfram_language_evaliator" -> "wolfram_language_evaluator", - "\\!\\(\\*MarkdownImageBox[\"" ~~ Shortest[ uri__ ] ~~ "\"]\\)" :> uri, - "\\!\\(MarkdownImageBox[\"" ~~ Shortest[ uri__ ] ~~ "\"]\\)" :> uri, - "\"\\\\!\\\\(\\\\*MarkdownImageBox[\\\"" ~~ Shortest[ uri__ ] ~~ "\\\"]\\\\)\"" :> uri, - "\"\\\\!\\\\(MarkdownImageBox[\\\"" ~~ Shortest[ uri__ ] ~~ "\\\"]\\\\)\"" :> uri, + "\\!\\(\\*"~~$$specialBoxName~~"[\"" ~~ Shortest[ uri__ ] ~~ "\"]\\)" :> uri, + "\\!\\("~~$$specialBoxName~~"[\"" ~~ Shortest[ uri__ ] ~~ "\"]\\)" :> uri, + "\"\\\\!\\\\(\\\\*"~~$$specialBoxName~~"[\\\"" ~~ Shortest[ uri__ ] ~~ "\\\"]\\\\)\"" :> uri, + "\"\\\\!\\\\("~~$$specialBoxName~~"[\\\"" ~~ Shortest[ uri__ ] ~~ "\\\"]\\\\)\"" :> uri, "<" ~~ Shortest[ scheme: LetterCharacter.. ~~ "://" ~~ id: Except[ "!" ].. ] ~~ ">" :> " scheme <> "://" <> id <> "!>", "\\uf351" -> "\[FreeformPrompt]", diff --git a/Source/Chatbook/Serialization.wl b/Source/Chatbook/Serialization.wl index 1035d6f7..c237303d 100644 --- a/Source/Chatbook/Serialization.wl +++ b/Source/Chatbook/Serialization.wl @@ -1202,6 +1202,7 @@ serializeVideo[ box: TemplateBox[ _, "VideoBox1"|"VideoBox2", ___ ] ] := seriali serializeVideo[ box_, video_ ] := Enclose[ If[ VideoQ @ video, + If[ toolSelectedQ[ "WolframLanguageEvaluator" ], needsBasePrompt[ "VideoBoxImporting" ] ]; "\\!\\(\\*VideoBox[\"" <> ConfirmBy[ MakeExpressionURI @ video, StringQ, "URI" ] <> "\"]\\)", "\\!\\(\\*VideoBox[...]\\)" ], @@ -1214,17 +1215,19 @@ serializeVideo // endDefinition; (* ::Subsubsubsection::Closed:: *) (*Audio*) fasterCellToString0[ box: TagBox[ _, _Audio`AudioBox, ___ ] ] := serializeAudio @ box; +fasterCellToString0[ box: TemplateBox[ _, "AudioBox1", ___ ] ] := serializeAudio @ box; (* ::**************************************************************************************************************:: *) (* ::Subsubsubsubsection::Closed:: *) (*serializeAudio*) serializeAudio // beginDefinition; -serializeAudio[ box: TagBox[ content_, _Audio`AudioBox, ___ ] ] := serializeAudio[ box ] = - serializeAudio[ content, Quiet @ ToExpression[ box, StandardForm ] ]; +serializeAudio[ box_ ] := serializeAudio[ box ] = + serializeAudio[ box, Quiet @ ToExpression[ box, StandardForm ] ]; -serializeAudio[ content_, audio_ ] := Enclose[ +serializeAudio[ box_, audio_ ] := Enclose[ If[ AudioQ @ audio, + If[ toolSelectedQ[ "WolframLanguageEvaluator" ], needsBasePrompt[ "AudioBoxImporting" ] ]; "\\!\\(\\*AudioBox[\"" <> ConfirmBy[ MakeExpressionURI @ audio, StringQ, "URI" ] <> "\"]\\)", "\\!\\(\\*AudioBox[...]\\)" ],