Merge pull request #439 from WolframResearch/feature/add-prompts-option

Support "Prompts" property in LLMEvaluator association specs
WolframResearch · Nov 1, 2023 · 251e0cd · 251e0cd
2 parents 70492d0 + 6709e8d
commit 251e0cd
Show file tree

Hide file tree

Showing 2 changed files with 13 additions and 3 deletions.
diff --git a/Source/Chatbook/ChatMessages.wl b/Source/Chatbook/ChatMessages.wl
@@ -112,6 +112,7 @@ constructMessages[ settings_Association? AssociationQ, messages0: { __Associatio
             messagePrint[ "InvalidMessages", getProcessingFunction[ settings, "ChatMessages" ], processed ];
             processed = messages
         ];
+        processed //= DeleteCases @ KeyValuePattern[ "Content" -> "" ];
         Sow[ <| "Messages" -> processed |>, $chatDataTag ];
 
         $lastSettings = settings;
@@ -340,11 +341,14 @@ getPrePrompt[ as_Association ] := toPromptString @ FirstCase[
         as[ "LLMEvaluator", "ChatContextPreprompt" ],
         as[ "LLMEvaluator", "Pre" ],
         as[ "LLMEvaluator", "PromptTemplate" ],
+        as[ "LLMEvaluator", "Prompts" ],
         as[ "ChatContextPreprompt" ],
         as[ "Pre" ],
         as[ "PromptTemplate" ]
     },
-    expr_ :> With[ { e = expr }, e /; MatchQ[ e, _String | _TemplateObject ] ]
+    expr_ :> With[ { e = expr },
+        e /; MatchQ[ e, _String | _TemplateObject | { (_String|_TemplateObject) ... } ]
+    ]
 ];
 
 getPrePrompt // endDefinition;
@@ -412,6 +416,12 @@ toPromptString // beginDefinition;
 toPromptString[ string_String ] := string;
 toPromptString[ template_TemplateObject ] := With[ { string = TemplateApply @ template }, string /; StringQ @ string ];
 toPromptString[ _Missing | Automatic | Inherited | None ] := Missing[ ];
+
+toPromptString[ prompts_List ] :=
+    With[ { strings = DeleteMissing[ toPromptString /@ prompts ] },
+        StringRiffle[ prompts, "\n\n" ] /; MatchQ[ strings, { ___String } ]
+    ];
+
 toPromptString // endDefinition;
 
 (* ::**************************************************************************************************************:: *)

diff --git a/Source/Chatbook/SendChat.wl b/Source/Chatbook/SendChat.wl
@@ -854,8 +854,8 @@ toolEvaluation[ settings_, container_Symbol, cell_, as_Association ] := Enclose[
         newMessages = Join[
             messages,
             {
-                <| "role" -> "assistant", "content" -> StringTrim @ string <> "\nENDTOOLCALL" |>,
-                <| "role" -> "system"   , "content" -> ToString @ output |>
+                <| "Role" -> "assistant", "Content" -> StringTrim @ string <> "\nENDTOOLCALL" |>,
+                <| "Role" -> "system"   , "Content" -> ToString @ output |>
             }
         ];