Merge pull request #949 from WolframResearch/bugfix/special-box-parsing

Fixed several issues with the LLM being unable to properly reference audio/video expressions
WolframResearch · Nov 22, 2024 · b7ef24f · b7ef24f
2 parents 52a868c + 47e64c6
commit b7ef24f
Show file tree

Hide file tree

Showing 4 changed files with 35 additions and 8 deletions.
diff --git a/Source/Chatbook/Prompting.wl b/Source/Chatbook/Prompting.wl
@@ -29,6 +29,8 @@ $basePromptOrder = {
     "ConversionGraphics",
     "MarkdownImageBox",
     "MarkdownImageBoxImporting",
+    "AudioBoxImporting",
+    "VideoBoxImporting",
     "Checkboxes",
     "CheckboxesIndeterminate",
     "ConversionFormatting",
@@ -83,6 +85,8 @@ $basePromptDependencies = Append[ "GeneralInstructionsHeader" ] /@ <|
     "ConversionGraphics"                   -> { "MessageConversionHeader" },
     "MarkdownImageBox"                     -> { "MessageConversionHeader" },
     "MarkdownImageBoxImporting"            -> { "MarkdownImageBox" },
+    "AudioBoxImporting"                    -> { "MarkdownImageBoxImporting" },
+    "VideoBoxImporting"                    -> { "MarkdownImageBoxImporting" },
     "ConversionFormatting"                 -> { "MessageConversionHeader" },
     "ExternalLanguageCells"                -> { "MessageConversionHeader" },
     "SpecialURI"                           -> { },
@@ -202,6 +206,12 @@ $basePromptComponents[ "MarkdownImageBoxImporting" ] = "\
 tool. For example, ``ColorNegate[<!attachment://content-id!>]``. The expression will be inserted in place. Do not \
 include the MarkdownImageBox wrapper. You can also use this syntax to inline images into WL code blocks.";
 
+$basePromptComponents[ "AudioBoxImporting" ] = "\
+		* You can also use this syntax for audio that similarly appears in AudioBox[...].";
+
+$basePromptComponents[ "VideoBoxImporting" ] = "\
+		* You can also use this syntax for video that similarly appears in VideoBox[...].";
+
 $basePromptComponents[ "Checkboxes" ] = "\
 	* Checkboxes in the UI will be replaced with one of the following text representations:
 		* A Checkbox that's selected becomes ``[\[Checkmark]]``

diff --git a/Source/Chatbook/Sandbox.wl b/Source/Chatbook/Sandbox.wl
@@ -2070,7 +2070,7 @@ inlineExpressionURI[ uri_String ] :=
 inlineExpressionURI[ uri_, failed_? FailureQ ] :=
     makeCachedBoxes @ failed;
 
-inlineExpressionURI[ uri_, HoldComplete[ expr_ ] ] /; graphicsQ @ Unevaluated @ expr :=
+inlineExpressionURI[ uri_, HoldComplete[ expr_? makeCachedBoxesQ ] ] :=
     makeCachedBoxes @ expr;
 
 inlineExpressionURI[ uri_, HoldComplete[ expr_ ] ] :=
@@ -2083,6 +2083,17 @@ inlineExpressionURI[ uri_, HoldComplete[ expr_ ] ] :=
 
 inlineExpressionURI // endDefinition;
 
+(* ::**************************************************************************************************************:: *)
+(* ::Subsubsection::Closed:: *)
+(*makeCachedBoxesQ*)
+makeCachedBoxesQ // beginDefinition;
+makeCachedBoxesQ // Attributes = { HoldAllComplete };
+makeCachedBoxesQ[ audio_Audio ] := AudioQ @ Unevaluated @ audio;
+makeCachedBoxesQ[ video_Video ] := VideoQ @ Unevaluated @ video;
+makeCachedBoxesQ[ graph_Graph ] := GraphQ @ Unevaluated @ graph;
+makeCachedBoxesQ[ expr_ ] := graphicsQ @ Unevaluated @ expr;
+makeCachedBoxesQ // endDefinition;
+
 (* ::**************************************************************************************************************:: *)
 (* ::Subsubsection::Closed:: *)
 (*makeCachedBoxes*)

diff --git a/Source/Chatbook/SendChat.wl b/Source/Chatbook/SendChat.wl
@@ -894,14 +894,17 @@ autoCorrect // beginDefinition;
 autoCorrect[ string_String ] := StringReplace[ string, $llmAutoCorrectRules ];
 autoCorrect // endDefinition;
 
+
+$$specialBoxName = "AudioBox"|"MarkdownImageBox"|"VideoBox";
+
 $llmAutoCorrectRules := $llmAutoCorrectRules = Flatten @ {
     StartOfLine ~~ WhitespaceCharacter... ~~ "/command\ncode:" :> "/wl\ncode:",
     "```" ~~ code: Except[ "\n" ].. ~~ "```" :> "``"<>code<>"``",
     "wolfram_language_evaliator" -> "wolfram_language_evaluator",
-    "\\!\\(\\*MarkdownImageBox[\"" ~~ Shortest[ uri__ ] ~~ "\"]\\)" :> uri,
-    "\\!\\(MarkdownImageBox[\"" ~~ Shortest[ uri__ ] ~~ "\"]\\)" :> uri,
-    "\"\\\\!\\\\(\\\\*MarkdownImageBox[\\\"" ~~ Shortest[ uri__ ] ~~ "\\\"]\\\\)\"" :> uri,
-    "\"\\\\!\\\\(MarkdownImageBox[\\\"" ~~ Shortest[ uri__ ] ~~ "\\\"]\\\\)\"" :> uri,
+    "\\!\\(\\*"~~$$specialBoxName~~"[\"" ~~ Shortest[ uri__ ] ~~ "\"]\\)" :> uri,
+    "\\!\\("~~$$specialBoxName~~"[\"" ~~ Shortest[ uri__ ] ~~ "\"]\\)" :> uri,
+    "\"\\\\!\\\\(\\\\*"~~$$specialBoxName~~"[\\\"" ~~ Shortest[ uri__ ] ~~ "\\\"]\\\\)\"" :> uri,
+    "\"\\\\!\\\\("~~$$specialBoxName~~"[\\\"" ~~ Shortest[ uri__ ] ~~ "\\\"]\\\\)\"" :> uri,
     "<" ~~ Shortest[ scheme: LetterCharacter.. ~~ "://" ~~ id: Except[ "!" ].. ] ~~ ">" :>
         "<!" <> scheme <> "://" <> id <> "!>",
     "\\uf351" -> "\[FreeformPrompt]",

diff --git a/Source/Chatbook/Serialization.wl b/Source/Chatbook/Serialization.wl
@@ -1202,6 +1202,7 @@ serializeVideo[ box: TemplateBox[ _, "VideoBox1"|"VideoBox2", ___ ] ] := seriali
 
 serializeVideo[ box_, video_ ] := Enclose[
     If[ VideoQ @ video,
+        If[ toolSelectedQ[ "WolframLanguageEvaluator" ], needsBasePrompt[ "VideoBoxImporting" ] ];
         "\\!\\(\\*VideoBox[\"" <> ConfirmBy[ MakeExpressionURI @ video, StringQ, "URI" ] <> "\"]\\)",
         "\\!\\(\\*VideoBox[...]\\)"
     ],
@@ -1214,17 +1215,19 @@ serializeVideo // endDefinition;
 (* ::Subsubsubsection::Closed:: *)
 (*Audio*)
 fasterCellToString0[ box: TagBox[ _, _Audio`AudioBox, ___ ] ] := serializeAudio @ box;
+fasterCellToString0[ box: TemplateBox[ _, "AudioBox1", ___ ] ] := serializeAudio @ box;
 
 (* ::**************************************************************************************************************:: *)
 (* ::Subsubsubsubsection::Closed:: *)
 (*serializeAudio*)
 serializeAudio // beginDefinition;
 
-serializeAudio[ box: TagBox[ content_, _Audio`AudioBox, ___ ] ] := serializeAudio[ box ] =
-    serializeAudio[ content, Quiet @ ToExpression[ box, StandardForm ] ];
+serializeAudio[ box_ ] := serializeAudio[ box ] =
+    serializeAudio[ box, Quiet @ ToExpression[ box, StandardForm ] ];
 
-serializeAudio[ content_, audio_ ] := Enclose[
+serializeAudio[ box_, audio_ ] := Enclose[
     If[ AudioQ @ audio,
+        If[ toolSelectedQ[ "WolframLanguageEvaluator" ], needsBasePrompt[ "AudioBoxImporting" ] ];
         "\\!\\(\\*AudioBox[\"" <> ConfirmBy[ MakeExpressionURI @ audio, StringQ, "URI" ] <> "\"]\\)",
         "\\!\\(\\*AudioBox[...]\\)"
     ],