Skip to content

Commit

Permalink
Merge pull request #460 from WolframResearch/feature/tool-call-frequency
Browse files Browse the repository at this point in the history 
Customizable tool call frequency
  • Loading branch information
@rhennigan
rhennigan authored Nov 22, 2023
2 parents 8f97757 + d68439e commit 3f983d0
Show file tree
Hide file tree
Showing 4 changed files with 199 additions and 25 deletions.
20 changes: 11 additions & 9 deletions Source/Chatbook/SendChat.wl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -576,7 +576,7 @@ ServiceConnectionUtilities`ConnectionInformation["Anthropic", "ProcessedRequests

makeLLMConfiguration[ as_Association ] :=
$lastLLMConfiguration = LLMConfiguration @ Association[
KeyTake[ as, { "Model", "MaxTokens" } ],
KeyTake[ as, { "Model", "MaxTokens", "Temperature" } ],
"StopTokens" -> { "ENDTOOLCALL" }
];

Expand Down Expand Up @@ -1228,21 +1228,22 @@ resolveAutoSetting[ settings_, key_ -> value_ ] := <| settings, key -> resolveAu
resolveAutoSetting // endDefinition;

resolveAutoSetting0 // beginDefinition;
resolveAutoSetting0[ as_, "ToolsEnabled" ] := toolsEnabledQ @ as;
resolveAutoSetting0[ as_, "DynamicAutoFormat" ] := dynamicAutoFormatQ @ as;
resolveAutoSetting0[ as_, "EnableLLMServices" ] := $useLLMServices;
resolveAutoSetting0[ as_, "HandlerFunctionsKeys" ] := chatHandlerFunctionsKeys @ as;
resolveAutoSetting0[ as_, "IncludeHistory" ] := Automatic;
resolveAutoSetting0[ as_, "MaxCellStringLength" ] := chooseMaxCellStringLength @ as;
resolveAutoSetting0[ as_, "MaxContextTokens" ] := autoMaxContextTokens @ as;
resolveAutoSetting0[ as_, "MaxOutputCellStringLength" ] := chooseMaxOutputCellStringLength @ as;
resolveAutoSetting0[ as_, "MaxTokens" ] := autoMaxTokens @ as;
resolveAutoSetting0[ as_, "Multimodal" ] := multimodalQ @ as;
resolveAutoSetting0[ as_, "NotebookWriteMethod" ] := "PreemptiveLink";
resolveAutoSetting0[ as_, "ShowMinimized" ] := Automatic;
resolveAutoSetting0[ as_, "StreamingOutputMethod" ] := "PartialDynamic";
resolveAutoSetting0[ as_, "TrackScrollingWhenPlaced" ] := scrollOutputQ @ as;
resolveAutoSetting0[ as_, "Multimodal" ] := multimodalQ @ as;
resolveAutoSetting0[ as_, "MaxContextTokens" ] := autoMaxContextTokens @ as;
resolveAutoSetting0[ as_, "MaxTokens" ] := autoMaxTokens @ as;
resolveAutoSetting0[ as_, "MaxCellStringLength" ] := chooseMaxCellStringLength @ as;
resolveAutoSetting0[ as_, "MaxOutputCellStringLength" ] := chooseMaxOutputCellStringLength @ as;
resolveAutoSetting0[ as_, "Tokenizer" ] := getTokenizer @ as;
resolveAutoSetting0[ as_, "ToolCallFrequency" ] := Automatic;
resolveAutoSetting0[ as_, "ToolsEnabled" ] := toolsEnabledQ @ as;
resolveAutoSetting0[ as_, "TrackScrollingWhenPlaced" ] := scrollOutputQ @ as;
resolveAutoSetting0[ as_, key_String ] := Automatic;
resolveAutoSetting0 // endDefinition;

Expand All @@ -1256,7 +1257,7 @@ $autoSettingKeyDependencies = <|
"Multimodal" -> { "EnableLLMServices", "Model" },
"Tokenizer" -> "Model",
"Tools" -> { "LLMEvaluator", "ToolsEnabled" },
"ToolsEnabled" -> "Model"
"ToolsEnabled" -> { "Model", "ToolCallFrequency" }
|>;

(* Sort topologically so dependencies will be satisfied in order: *)
Expand Down Expand Up @@ -1392,6 +1393,7 @@ getNamedLLMEvaluator // endDefinition;
(*toolsEnabledQ*)
(*FIXME: move to Tools.wl *)
toolsEnabledQ[ KeyValuePattern[ "ToolsEnabled" -> enabled: True|False ] ] := enabled;
toolsEnabledQ[ KeyValuePattern[ "ToolCallFrequency" -> freq: (_Integer|_Real)? NonPositive ] ] := False;
toolsEnabledQ[ KeyValuePattern[ "Model" -> model_ ] ] := toolsEnabledQ @ toModelName @ model;
toolsEnabledQ[ "chat-bison-001" ] := False;
toolsEnabledQ[ model_String ] := ! TrueQ @ StringContainsQ[ model, "gpt-3", IgnoreCase -> True ];
Expand Down
3 changes: 2 additions & 1 deletion Source/Chatbook/Settings.wl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ $defaultChatSettings = <|
"StreamingOutputMethod" -> Automatic,
"Temperature" -> 0.7,
"Tokenizer" -> Automatic,
"ToolCallFrequency" -> Automatic,
"ToolOptions" :> $DefaultToolOptions,
"Tools" -> Automatic,
"ToolsEnabled" -> Automatic,
Expand Down Expand Up @@ -433,7 +434,7 @@ $currentEvaluationObject := $FrontEndSession;
verifyInheritance // beginDefinition;
verifyInheritance[ obj: $$feObj ] /; $cloudNotebooks := True;
verifyInheritance[ obj: $$feObj? inheritingQ ] := True;
verifyInheritance[ obj: $$feObj ] := With[ { verified = verifyInheritance0 @ obj }, True /; inheritingQ @ obj ];
verifyInheritance[ obj: $$feObj ] := With[ { verified = verifyInheritance0 @ obj }, inheritingQ @ obj ];
verifyInheritance // endDefinition;


Expand Down
86 changes: 73 additions & 13 deletions Source/Chatbook/Tools.wl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -509,7 +509,7 @@ makeToolConfiguration // beginDefinition;
makeToolConfiguration[ settings_Association ] := Enclose[
Module[ { tools },
tools = ConfirmMatch[ DeleteDuplicates @ Flatten @ Values @ $selectedTools, { ___LLMTool }, "SelectedTools" ];
$toolConfiguration = LLMConfiguration @ <| "Tools" -> tools, "ToolPrompt" -> $toolPrompt |>
$toolConfiguration = LLMConfiguration @ <| "Tools" -> tools, "ToolPrompt" -> makeToolPrompt @ settings |>
],
throwInternalFailure[ makeToolConfiguration @ settings, ## ] &
];
Expand All @@ -532,6 +532,35 @@ toolRequestParser := toolRequestParser =
LLMConfiguration::invprop
];

(* ::**************************************************************************************************************:: *)
(* ::Subsubsection::Closed:: *)
(*makeToolPrompt*)
makeToolPrompt // beginDefinition;

makeToolPrompt[ settings_Association ] := $lastToolPrompt = TemplateObject[
Riffle[
DeleteMissing @ {
$toolPre,
TemplateSequence[
TemplateExpression @ StringTemplate[
"Tool Name: `Name`\nDescription: `Description`\nSchema:\n`Schema`\n\n"
],
TemplateExpression @ Map[
Append[ #[ "Data" ], "Schema" -> ExportString[ #[ "JSONSchema" ], "JSON" ] ] &,
TemplateSlot[ "Tools" ]
]
],
$toolPost,
makeToolPreferencePrompt @ settings
},
"\n\n"
],
CombinerFunction -> StringJoin,
InsertionFunction -> TextString
];

makeToolPrompt // endDefinition;

(* ::**************************************************************************************************************:: *)
(* ::Subsubsection::Closed:: *)
(*$toolPrompt*)
Expand All @@ -547,7 +576,8 @@ $toolPrompt := TemplateObject[
TemplateSlot[ "Tools" ]
]
],
$toolPost
$toolPost,
$fullExamples
},
CombinerFunction -> StringJoin,
InsertionFunction -> TextString
Expand All @@ -558,13 +588,10 @@ $toolPre = "\
# Tool Instructions
You have access to tools which can be used to do things, fetch data, compute, etc. while you create your response. \
Each tool takes input as JSON following a JSON schema. Here are the available tools and their associated schemas:
";
Each tool takes input as JSON following a JSON schema. Here are the available tools and their associated schemas:";


$toolPost := "
$toolPost := "\
To call a tool, write the following at any time during your response:
TOOLCALL: <tool name>
Expand All @@ -590,14 +617,47 @@ If a user asks you to use a specific tool, you MUST attempt to use that tool as
even if you think it will not work. \
If the tool fails, use any error message to correct the issue or explain why it failed. \
NEVER state that a tool cannot be used for a particular task without trying it first. \
You did not create these tools, so you do not know what they can and cannot do.
You did not create these tools, so you do not know what they can and cannot do.";

" <> $fullExamples;
(* ::**************************************************************************************************************:: *)
(* ::Subsubsubsection::Closed:: *)
(*makeToolPreferencePrompt*)
makeToolPreferencePrompt // beginDefinition;

(* TODO:
Create a preference setting called "ToolCallFrequency" that inserts a prompt here telling the LLM how often it
should be making tool calls.
*)
makeToolPreferencePrompt[ settings_ ] :=
makeToolPreferencePrompt[ settings, settings[ "ToolCallFrequency" ] ];

makeToolPreferencePrompt[ settings_, Automatic ] :=
Missing[ "NotAvailable" ];

makeToolPreferencePrompt[ settings_, freq_? NumberQ ] :=
With[ { key = Round @ Clip[ 5 * freq, { 0, 5 } ] },
TemplateApply[
$toolPreferencePrompt,
<| "Number" -> Round[ freq * 100 ], "Explanation" -> Lookup[ $toolFrequencyExplanations, key, "" ] |>
]
];

makeToolPreferencePrompt // endDefinition;


$toolPreferencePrompt = "\
## User Tool Call Preferences
The user has specified their desired tool calling frequency to be `Number`% with the following instructions:
IMPORTANT
`Explanation`";


$toolFrequencyExplanations = <|
0 -> "Only use a tool if explicitly instructed to use tools. Never use tools unless specifically asked to.",
1 -> "Avoid using tools unless you think it is necessary.",
2 -> "Only use tools if you think it will significantly improve the quality of your response.",
3 -> "Use tools whenever it is appropriate to do so.",
4 -> "Use tools whenever there's even a slight chance that it could improve the quality of your response (e.g. fact checking).",
5 -> "ALWAYS make a tool call in EVERY response, no matter what."
|>;

(* ::**************************************************************************************************************:: *)
(* ::Section::Closed:: *)
Expand Down
115 changes: 113 additions & 2 deletions Source/Chatbook/UI.wl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ Needs[ "Wolfram`Chatbook`Models`" ];
Needs[ "Wolfram`Chatbook`Personas`" ];
Needs[ "Wolfram`Chatbook`PreferencesUtils`" ];
Needs[ "Wolfram`Chatbook`Serialization`" ];
Needs[ "Wolfram`Chatbook`Services`" ];
Needs[ "Wolfram`Chatbook`Settings`" ];
Needs[ "Wolfram`Chatbook`Utils`" ];

Expand Down Expand Up @@ -471,6 +472,98 @@ labeledCheckbox[value_, label_, enabled_ : Automatic] :=

(*====================================*)

makeToolCallFrequencySlider[ obj_ ] := Pane[
Grid[
{
{
labeledCheckbox[
Dynamic[
currentChatSettings[ obj, "ToolCallFrequency" ] === Automatic,
Function[
If[ TrueQ[ # ],
CurrentValue[ obj, { TaggingRules, "ChatNotebookSettings", "ToolCallFrequency" } ] = Inherited,
CurrentValue[ obj, { TaggingRules, "ChatNotebookSettings", "ToolCallFrequency" } ] = 0.5
]
]
],
Style[ "Choose automatically", "ChatMenuLabel" ]
]
},
{
Pane[
Slider[
Dynamic[
Replace[ currentChatSettings[ obj, "ToolCallFrequency" ], Automatic -> 0.5 ],
(CurrentValue[ obj, { TaggingRules, "ChatNotebookSettings", "ToolCallFrequency" } ] = #) &
],
{ 0, 1, 0.01 },
(* Enabled -> Dynamic[ currentChatSettings[ obj, "ToolCallFrequency" ] =!= Automatic ], *)
ImageSize -> { 150, Automatic },
ImageMargins -> { { 5, 0 }, { 5, 5 } }
],
ImageSize -> { 180, Automatic },
BaseStyle -> { FontSize -> 12 }
],
SpanFromLeft
}
},
Alignment -> Left,
Spacings -> { Automatic, 0 }
],
ImageMargins -> { { 5, 0 }, { 5, 5 } }
];

makeToolCallFrequencySlider[ obj_ ] :=
Module[ { checkbox, slider },
checkbox = labeledCheckbox[
Dynamic[
currentChatSettings[ obj, "ToolCallFrequency" ] === Automatic,
Function[
If[ TrueQ[ # ],
CurrentValue[ obj, { TaggingRules, "ChatNotebookSettings", "ToolCallFrequency" } ] = Inherited,
CurrentValue[ obj, { TaggingRules, "ChatNotebookSettings", "ToolCallFrequency" } ] = 0.5
]
]
],
Style[ "Choose automatically", "ChatMenuLabel" ]
];
slider = Pane[
Grid[
{
{
Style[ "Rare", "ChatMenuLabel", FontSize -> 12 ],
Slider[
Dynamic[
Replace[ currentChatSettings[ obj, "ToolCallFrequency" ], Automatic -> 0.5 ],
(CurrentValue[ obj, { TaggingRules, "ChatNotebookSettings", "ToolCallFrequency" } ] = #) &
],
{ 0, 1, 0.01 },
ImageSize -> { 100, Automatic },
ImageMargins -> { { 0, 0 }, { 5, 5 } }
],
Style[ "Often", "ChatMenuLabel", FontSize -> 12 ]
}
},
Spacings -> { { 0, { 0.5 }, 0 }, 0 },
Alignment -> { { Left, Center, Right }, Baseline }
],
ImageMargins -> 0,
ImageSize -> { 170, Automatic }
];
Pane[
PaneSelector[
{
True -> Column[ { checkbox }, Alignment -> Left ],
False -> Column[ { slider, checkbox }, Alignment -> Left ]
},
Dynamic[ currentChatSettings[ obj, "ToolCallFrequency" ] === Automatic ],
ImageSize -> Automatic
],
ImageMargins -> { { 5, 0 }, { 5, 5 } }
]
];


makeTemperatureSlider[
value_
] :=
Expand All @@ -486,6 +579,7 @@ makeTemperatureSlider[
BaseStyle -> { FontSize -> 12 }
]

(* cSpell: ignore AIAPI *)
makeOpenAIAPICompletionURLForm[value_]:= Pane[
InputField[value,
String,
Expand Down Expand Up @@ -599,6 +693,10 @@ makeFrontEndAndNotebookSettingsContent[
]
]
}, Spacer[3]]},
{Row[{
tr["Default Tool Call Frequency:"],
makeToolCallFrequencySlider[ targetObj ]
}, Spacer[3]]},
{Row[{
tr["Default Temperature:"],
makeTemperatureSlider[
Expand All @@ -614,7 +712,9 @@ makeFrontEndAndNotebookSettingsContent[
]
}, Spacer[3]]},

{Row[{
If[ TrueQ @ $useLLMServices,
Nothing,
{Row[{
tr["Chat Completion URL:"],
makeOpenAIAPICompletionURLForm[
Dynamic[
Expand All @@ -627,7 +727,7 @@ makeFrontEndAndNotebookSettingsContent[
)
]
]
}, Spacer[3]]},
}, Spacer[3]]}],
{
labeledCheckbox[
Dynamic[
Expand Down Expand Up @@ -1007,6 +1107,7 @@ makeChatActionMenu[
targetObj,
{TaggingRules, "ChatNotebookSettings", "Role"}
],
"ToolCallFrequency" -> targetObj,
"TemperatureValue" -> Dynamic[
currentChatSettings[ targetObj, "Temperature" ],
newValue |-> (
Expand All @@ -1027,6 +1128,7 @@ Options[makeChatActionMenuContent] = {
"PersonaValue" -> Automatic,
"ModelValue" -> Automatic,
"RoleValue" -> Automatic,
"ToolCallFrequency" -> Automatic,
"TemperatureValue" -> Automatic,
"ActionCallback" -> (Null &)
}
Expand All @@ -1042,6 +1144,7 @@ makeChatActionMenuContent[
personaValue = OptionValue["PersonaValue"],
modelValue = OptionValue["ModelValue"],
roleValue = OptionValue["RoleValue"],
toolValue = OptionValue["ToolCallFrequency"],
tempValue = OptionValue["TemperatureValue"],
advancedSettingsMenu,
menuLabel,
Expand All @@ -1061,6 +1164,14 @@ makeChatActionMenuContent[
None
}
},
{
"Tool Call Frequency",
{
None,
makeToolCallFrequencySlider[toolValue],
None
}
},
{"Roles"},
Map[
entry |-> ConfirmReplace[entry, {
Expand Down

0 comments on commit 3f983d0

Please sign in to comment.