From 1157a50cdcd274ee4fb52411ea12f1ac4cc62d8d Mon Sep 17 00:00:00 2001 From: "Loic A. Royer" Date: Fri, 30 Aug 2024 17:24:07 -0400 Subject: [PATCH] upgraded langchain dependency versions and therefore fixed a lot of issues caused by dependencies changes... --- setup.cfg | 15 +-- .../callbacks/callbacks_handle_chat.py | 14 ++- src/napari_chatgpt/omega/memory/memory.py | 3 + src/napari_chatgpt/omega/napari_bridge.py | 13 ++- .../OmegaOpenAIFunctionsAgentOutputParser.py | 92 +++++++++++++++++++ .../omega_agent/OpenAIFunctionsOmegaAgent.py | 34 +++---- .../omega/omega_agent/prompts.py | 2 + src/napari_chatgpt/omega/omega_init.py | 5 +- .../omega/tools/async_base_tool.py | 23 +++-- .../omega/tools/instructions.py | 3 +- .../delegated_code/test/classic_test.py | 4 +- .../omega/tools/napari/napari_base_tool.py | 8 +- .../tools/special/exception_catcher_tool.py | 10 +- .../omega/tools/special/file_download_tool.py | 11 ++- .../tools/special/functions_info_tool.py | 10 +- .../omega/tools/special/human_input_tool.py | 10 +- .../omega/tools/special/package_info_tool.py | 60 ++++++++++++ .../omega/tools/special/pip_install_tool.py | 10 +- .../omega/tools/special/python_repl.py | 60 ++++++------ .../napari/test/napari_viewer_info_test.py | 9 ++ .../utils/network/demo/port_available_demo.py | 9 ++ src/napari_chatgpt/utils/openai/gpt_vision.py | 2 +- src/napari_chatgpt/utils/openai/model_list.py | 4 +- .../utils/python/relevant_libraries.py | 15 +++ .../utils/web/test/duckduckgo_test.py | 8 +- .../utils/web/test/google_test.py | 5 +- .../utils/web/test/metasearch_test.py | 12 +-- .../utils/web/test/wikipedia_test.py | 26 ++++-- 28 files changed, 378 insertions(+), 99 deletions(-) create mode 100644 src/napari_chatgpt/omega/omega_agent/OmegaOpenAIFunctionsAgentOutputParser.py create mode 100644 src/napari_chatgpt/omega/tools/special/package_info_tool.py diff --git a/setup.cfg b/setup.cfg index 723cf4f..50b8107 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = napari-chatgpt -version = v2024.5.15 +version = v2024.8.27 description = A napari plugin to process and analyse images with chatGPT. long_description = file: README.md long_description_content_type = text/markdown @@ -36,12 +36,13 @@ install_requires = scikit-image qtpy QtAwesome - langchain==0.2.0rc2 - langchain-community==0.2.0rc1 - langchain-openai==0.1.6 - langchain-anthropic==0.1.11 - openai==1.29.0 - anthropic + langchain==0.2.15 + langchain-community==0.2.14 + langchain-openai==0.1.23 + langchain-anthropic==0.1.23 +# langchain-google-genai==1.0.10 + openai==1.42.0 + anthropic==0.34.1 fastapi uvicorn websockets diff --git a/src/napari_chatgpt/chat_server/callbacks/callbacks_handle_chat.py b/src/napari_chatgpt/chat_server/callbacks/callbacks_handle_chat.py index 5f1eb2a..1481308 100644 --- a/src/napari_chatgpt/chat_server/callbacks/callbacks_handle_chat.py +++ b/src/napari_chatgpt/chat_server/callbacks/callbacks_handle_chat.py @@ -121,7 +121,19 @@ async def on_agent_action(self, action: AgentAction, **kwargs: Any) -> Any: if self.verbose: aprint(f"CHAT on_agent_action: {action}") tool = camel_case_to_lower_case(action.tool) - message = f"I am using the {tool} to tackle your request: '{action.tool_input}'" + + # extract value for args key after checking if action.tool_input is a dict: + if isinstance(action.tool_input, dict): + argument = action.tool_input.get('args', '') + + # if argument is a singleton list, unpop that single element: + if isinstance(argument, list): + argument = argument[0] + + else: + argument = action.tool_input + + message = f"I am using the {tool} to tackle your request: '{argument}'" self.last_tool_used = tool self.last_tool_input = action.tool_input diff --git a/src/napari_chatgpt/omega/memory/memory.py b/src/napari_chatgpt/omega/memory/memory.py index bf70ca8..b7ab562 100644 --- a/src/napari_chatgpt/omega/memory/memory.py +++ b/src/napari_chatgpt/omega/memory/memory.py @@ -2,6 +2,7 @@ from typing import Type from langchain.chains import LLMChain +from langchain.memory import ConversationSummaryMemory from langchain.memory.chat_memory import BaseChatMemory from langchain.memory.prompt import SUMMARY_PROMPT from langchain_core.language_models import BaseLanguageModel @@ -16,6 +17,8 @@ ### LangChain's license is the MIT License ### +ConversationSummaryMemory + class SummarizerMixin(BaseModel): human_prefix: str = "Human" ai_prefix: str = "AI" diff --git a/src/napari_chatgpt/omega/napari_bridge.py b/src/napari_chatgpt/omega/napari_bridge.py index 306f55b..e2881a6 100644 --- a/src/napari_chatgpt/omega/napari_bridge.py +++ b/src/napari_chatgpt/omega/napari_bridge.py @@ -71,7 +71,18 @@ def get_viewer_info(self) -> str: # Setting up delegated function: delegated_function = lambda v: get_viewer_info(v) - return self._execute_in_napari_context(delegated_function) + try: + # execute delegated function in napari context: + info = self._execute_in_napari_context(delegated_function) + + return info + + except Exception as e: + # print exception stack trace: + import traceback + traceback.print_exc() + + return 'Could not get information about the viewer because of an error.' def take_snapshot(self): diff --git a/src/napari_chatgpt/omega/omega_agent/OmegaOpenAIFunctionsAgentOutputParser.py b/src/napari_chatgpt/omega/omega_agent/OmegaOpenAIFunctionsAgentOutputParser.py new file mode 100644 index 0000000..484ec0f --- /dev/null +++ b/src/napari_chatgpt/omega/omega_agent/OmegaOpenAIFunctionsAgentOutputParser.py @@ -0,0 +1,92 @@ +import json +from json import JSONDecodeError +from typing import List, Union + +from langchain_core.agents import AgentAction, AgentActionMessageLog, AgentFinish +from langchain_core.exceptions import OutputParserException +from langchain_core.messages import ( + AIMessage, + BaseMessage, +) +from langchain_core.outputs import ChatGeneration, Generation + +from langchain.agents.agent import AgentOutputParser + + +class OpenAIFunctionsAgentOutputParser(AgentOutputParser): + """Parses a message into agent action/finish. + + Is meant to be used with OpenAI models, as it relies on the specific + function_call parameter from OpenAI to convey what tools to use. + + If a function_call parameter is passed, then that is used to get + the tool and tool input. + + If one is not passed, then the AIMessage is assumed to be the final output. + """ + + @property + def _type(self) -> str: + return "openai-functions-agent" + + @staticmethod + def _parse_ai_message(message: BaseMessage) -> Union[AgentAction, AgentFinish]: + """Parse an AI message.""" + if not isinstance(message, AIMessage): + raise TypeError(f"Expected an AI message got {type(message)}") + + function_call = message.additional_kwargs.get("function_call", {}) + + if function_call: + function_name = function_call["name"] + try: + if len(function_call["arguments"].strip()) == 0: + # OpenAI returns an empty string for functions containing no args + _tool_input = {} + else: + # otherwise it returns a json object + _tool_input = json.loads(function_call["arguments"], strict=False) + except JSONDecodeError: + + # let's chill, no idea why this is a problem, my tools are just fine with this: + _tool_input = function_call["arguments"] + + # raise OutputParserException( + # f"Could not parse tool input: {function_call} because " + # f"the `arguments` is not valid JSON." + # ) + + # HACK HACK HACK: + # The code that encodes tool input into Open AI uses a special variable + # name called `__arg1` to handle old style tools that do not expose a + # schema and expect a single string argument as an input. + # We unpack the argument here if it exists. + # Open AI does not support passing in a JSON array as an argument. + if "__arg1" in _tool_input: + tool_input = _tool_input["__arg1"] + else: + tool_input = _tool_input + + content_msg = f"responded: {message.content}\n" if message.content else "\n" + log = f"\nInvoking: `{function_name}` with `{tool_input}`\n{content_msg}\n" + return AgentActionMessageLog( + tool=function_name, + tool_input=tool_input, + log=log, + message_log=[message], + ) + + return AgentFinish( + return_values={"output": message.content}, log=str(message.content) + ) + + def parse_result( + self, result: List[Generation], *, partial: bool = False + ) -> Union[AgentAction, AgentFinish]: + if not isinstance(result[0], ChatGeneration): + raise ValueError("This output parser only works on ChatGeneration output") + message = result[0].message + return self._parse_ai_message(message) + + def parse(self, text: str) -> Union[AgentAction, AgentFinish]: + raise ValueError("Can only parse messages") diff --git a/src/napari_chatgpt/omega/omega_agent/OpenAIFunctionsOmegaAgent.py b/src/napari_chatgpt/omega/omega_agent/OpenAIFunctionsOmegaAgent.py index ff1dabe..922ad97 100644 --- a/src/napari_chatgpt/omega/omega_agent/OpenAIFunctionsOmegaAgent.py +++ b/src/napari_chatgpt/omega/omega_agent/OpenAIFunctionsOmegaAgent.py @@ -5,9 +5,6 @@ from langchain.agents.format_scratchpad.openai_functions import ( format_to_openai_function_messages, ) -from langchain.agents.output_parsers.openai_functions import ( - OpenAIFunctionsAgentOutputParser, -) from langchain_core.agents import AgentAction, AgentFinish from langchain_core.callbacks import Callbacks from langchain_core.messages import ( @@ -15,6 +12,8 @@ ) from napari_chatgpt.omega.napari_bridge import _get_viewer_info +from napari_chatgpt.omega.omega_agent.OmegaOpenAIFunctionsAgentOutputParser import \ + OpenAIFunctionsAgentOutputParser from napari_chatgpt.omega.omega_agent.prompts import DIDACTICS @@ -25,26 +24,27 @@ class OpenAIFunctionsOmegaAgent(OpenAIFunctionsAgent): be_didactic: bool = False async def aplan( - self, - intermediate_steps: List[Tuple[AgentAction, str]], - callbacks: Callbacks = None, - **kwargs: Any, + self, + intermediate_steps: List[Tuple[AgentAction, str]], + callbacks: Callbacks = None, + **kwargs: Any, ) -> Union[AgentAction, AgentFinish]: - """Given input, decided what to do. + """Async given input, decided what to do. Args: intermediate_steps: Steps the LLM has taken to date, - along with observations + along with observations. + callbacks: Callbacks to use. Defaults to None. **kwargs: User inputs. Returns: Action specifying what tool to use. + If the agent is finished, returns an AgentFinish. + If the agent is not finished, returns an AgentAction. """ - agent_scratchpad = format_to_openai_function_messages( - intermediate_steps) + agent_scratchpad = format_to_openai_function_messages(intermediate_steps) selected_inputs = { - k: kwargs[k] for k in self.prompt.input_variables if - k != "agent_scratchpad" + k: kwargs[k] for k in self.prompt.input_variables if k != "agent_scratchpad" } full_inputs = dict(**selected_inputs, agent_scratchpad=agent_scratchpad) prompt = self.prompt.format_prompt(**full_inputs) @@ -60,6 +60,7 @@ async def aplan( ) )) + # Add didactics to the messages: if self.be_didactic: messages.insert(-1, SystemMessage( content=DIDACTICS, @@ -68,10 +69,11 @@ async def aplan( ) )) + # predict the message: predicted_message = await self.llm.apredict_messages( messages, functions=self.functions, callbacks=callbacks ) - agent_decision = OpenAIFunctionsAgentOutputParser._parse_ai_message( - predicted_message - ) + + # parse the AI message: + agent_decision = OpenAIFunctionsAgentOutputParser._parse_ai_message(predicted_message) return agent_decision \ No newline at end of file diff --git a/src/napari_chatgpt/omega/omega_agent/prompts.py b/src/napari_chatgpt/omega/omega_agent/prompts.py index 2e067f3..622abc0 100644 --- a/src/napari_chatgpt/omega/omega_agent/prompts.py +++ b/src/napari_chatgpt/omega/omega_agent/prompts.py @@ -7,6 +7,8 @@ You can use all the tools and functions at your disposal (see below) to assist the user with image processing and image analysis. Since you are an helpful expert, you are polite and answer in the same language as the user's question. You have been created by Loic A. Royer, a Senior Group Leader and Director of Imaging AI at the Chan Zuckerberg Biohub San Francisco. + +You are provided with a series of tools/functions that give you the possibility to execute code in the context of an existing napari viewer instance. """ PERSONALITY = {} diff --git a/src/napari_chatgpt/omega/omega_init.py b/src/napari_chatgpt/omega/omega_init.py index b976487..8dee402 100644 --- a/src/napari_chatgpt/omega/omega_init.py +++ b/src/napari_chatgpt/omega/omega_init.py @@ -2,7 +2,7 @@ import langchain from arbol import aprint -from langchain.agents import AgentExecutor +from langchain.agents import AgentExecutor, create_openai_functions_agent from langchain.agents.conversational_chat.prompt import SUFFIX from langchain.base_language import BaseLanguageModel from langchain.callbacks.base import BaseCallbackHandler @@ -39,6 +39,8 @@ from napari_chatgpt.omega.tools.special.functions_info_tool import \ PythonFunctionsInfoTool from napari_chatgpt.omega.tools.special.human_input_tool import HumanInputTool +from napari_chatgpt.omega.tools.special.package_info_tool import \ + PythonPackageInfoTool from napari_chatgpt.omega.tools.special.pip_install_tool import PipInstallTool from napari_chatgpt.omega.tools.special.python_repl import \ PythonCodeExecutionTool @@ -90,6 +92,7 @@ def initialize_omega_agent(to_napari_queue: Queue = None, ExceptionCatcherTool(callbacks=tool_callbacks), # FileDownloadTool(), PythonCodeExecutionTool(callbacks=tool_callbacks), + PythonPackageInfoTool(callbacks=tool_callbacks), PipInstallTool(callbacks=tool_callbacks)] # Adding the human input tool if required: diff --git a/src/napari_chatgpt/omega/tools/async_base_tool.py b/src/napari_chatgpt/omega/tools/async_base_tool.py index 616af7b..d3189f9 100644 --- a/src/napari_chatgpt/omega/tools/async_base_tool.py +++ b/src/napari_chatgpt/omega/tools/async_base_tool.py @@ -14,11 +14,18 @@ class AsyncBaseTool(BaseTool): notebook: JupyterNotebookFile = None - async def _arun(self, query: str) -> str: - """Use the tool asynchronously.""" - aprint(f"Starting async call to {type(self).__name__}({query}) ") - result = await asyncio.get_running_loop().run_in_executor( - _aysync_tool_thread_pool, - self._run, - query) - return result + def normalise_to_string(self, kwargs): + + # extract the value for args key in kwargs: + if isinstance(kwargs, dict): + query = kwargs.get('args', '') + else: + query = kwargs + + # If query is a singleton list, extract the value: + if isinstance(query, list) and len(query) == 1: + query = query[0] + + # convert the query to string: + query = str(query) + return query diff --git a/src/napari_chatgpt/omega/tools/instructions.py b/src/napari_chatgpt/omega/tools/instructions.py index e3efb63..838c536 100644 --- a/src/napari_chatgpt/omega/tools/instructions.py +++ b/src/napari_chatgpt/omega/tools/instructions.py @@ -20,7 +20,8 @@ - When and if you use PyTorch functions make sure to pass tensors with the right dtype and number of dimensions in order to match PyTorch's functions parameter requirements. For instance, add and remove batch dimensions and convert to a compatible dtype before and after a series of calls to PyTorch functions. - The only data types supported by PyTorch are: float32, float64, float16, bfloat16, uint8, int8, int16, int32, int64, and bool. Make sure to convert the input to one of these types before passing it to a PyTorch function. - When using Numba to write image processing code make sure to avoid high-level numpy functions and instead implement the algorithms with loops and low-level numpy functions. Also, make sure to use the right data types for the input and output arrays. -- If you need to get the selected layer in the napari viewer, use the following code: `viewer.layers.selection.active` . +- If you need to get the selected layer in the napari viewer, use the following code: `viewer.layers.selection.active`. +- napari layers do not have a 'type' field, if you need to check the type of a layer, use for example the following code: `isinstance(layer, napari.layers.Shapes)`. - If you need to rotate the viewer camera to a specific set of angles, use the following code: `viewer.camera.angles = (angle_z, angle_y, angle_x)` . """ diff --git a/src/napari_chatgpt/omega/tools/napari/delegated_code/test/classic_test.py b/src/napari_chatgpt/omega/tools/napari/delegated_code/test/classic_test.py index fc263b4..476151b 100644 --- a/src/napari_chatgpt/omega/tools/napari/delegated_code/test/classic_test.py +++ b/src/napari_chatgpt/omega/tools/napari/delegated_code/test/classic_test.py @@ -47,7 +47,7 @@ def test_classsic_3d(show_viewer: bool = False): aprint('') # Load the 'cells' example dataset - cells = skimage.data.cells3d()[:, 1] + cells = skimage.data.cells3d()[0:100, 0:100, 1].copy() # Segment the cells: labels = classic_segmentation(cells) @@ -58,7 +58,7 @@ def test_classsic_3d(show_viewer: bool = False): aprint(nb_unique_labels) # Check that the number of unique labels is correct: - assert nb_unique_labels == 25 + assert nb_unique_labels == 6 # If the viewer is not requested, return: if not show_viewer: diff --git a/src/napari_chatgpt/omega/tools/napari/napari_base_tool.py b/src/napari_chatgpt/omega/tools/napari/napari_base_tool.py index 6a99f58..dff4e17 100644 --- a/src/napari_chatgpt/omega/tools/napari/napari_base_tool.py +++ b/src/napari_chatgpt/omega/tools/napari/napari_base_tool.py @@ -3,7 +3,7 @@ import traceback from pathlib import Path from queue import Queue -from typing import Union, Optional +from typing import Union, Optional, Any from arbol import aprint, asection from langchain.chains import LLMChain @@ -63,9 +63,12 @@ class NapariBaseTool(AsyncBaseTool): last_generated_code: Optional[str] = None - def _run(self, query: str) -> str: + def _run(self, *args: Any, **kwargs: Any) -> Any: """Use the tool.""" + # Get query: + query = self.normalise_to_string(kwargs) + if self.prompt: # Instantiate chain: chain = LLMChain( @@ -137,6 +140,7 @@ def _run(self, query: str) -> str: return response + def _run_code(self, query: str, code: str, viewer: Viewer) -> str: """ This is the code that is executed, see implementations for details, diff --git a/src/napari_chatgpt/omega/tools/special/exception_catcher_tool.py b/src/napari_chatgpt/omega/tools/special/exception_catcher_tool.py index 33c7621..9790fea 100644 --- a/src/napari_chatgpt/omega/tools/special/exception_catcher_tool.py +++ b/src/napari_chatgpt/omega/tools/special/exception_catcher_tool.py @@ -2,6 +2,7 @@ import queue import sys import traceback +from typing import Any from arbol import aprint, asection @@ -50,14 +51,19 @@ class ExceptionCatcherTool(AsyncBaseTool): ) prompt: str = None - def _run(self, query: str) -> str: - """Use the tool.""" + def _run(self, + *args: Any, + **kwargs: Any + ) -> Any: with asection('ExceptionCatcherTool: List of caught exceptions:'): text = "Here is the list of exceptions that occurred:\n\n" text += "```\n" try: + # Get query: + query = self.normalise_to_string(kwargs) + # We try to convert the input to an integer: number_of_exceptions = int(query.strip()) except Exception as e: diff --git a/src/napari_chatgpt/omega/tools/special/file_download_tool.py b/src/napari_chatgpt/omega/tools/special/file_download_tool.py index 19b6950..9009d48 100644 --- a/src/napari_chatgpt/omega/tools/special/file_download_tool.py +++ b/src/napari_chatgpt/omega/tools/special/file_download_tool.py @@ -1,3 +1,5 @@ +from typing import Any + from arbol import asection, aprint from napari_chatgpt.omega.tools.async_base_tool import AsyncBaseTool @@ -13,9 +15,14 @@ class FileDownloadTool(AsyncBaseTool): "and thus is(are) directly accessible using its(their) filename. " "Use this tool to download files before any subsequent operations on these files.") - def _run(self, query: str) -> str: - """Use the tool.""" + def _run(self, + *args: Any, + **kwargs: Any + ) -> Any: + try: + # Get query: + query = self.normalise_to_string(kwargs) with asection(f"FileDownloadTool: query= {query} "): # extract urls from query diff --git a/src/napari_chatgpt/omega/tools/special/functions_info_tool.py b/src/napari_chatgpt/omega/tools/special/functions_info_tool.py index 95b22f0..a25224a 100644 --- a/src/napari_chatgpt/omega/tools/special/functions_info_tool.py +++ b/src/napari_chatgpt/omega/tools/special/functions_info_tool.py @@ -1,5 +1,6 @@ """A tool for running python code in a REPL.""" import traceback +from typing import Any from arbol import asection, aprint @@ -22,8 +23,13 @@ class PythonFunctionsInfoTool(AsyncBaseTool): "and example usages, please prefix your request with the single star character '*'." ) - def _run(self, query: str) -> str: - """Use the tool.""" + def _run(self, + *args: Any, + **kwargs: Any + ) -> Any: + + # Get query: + query = self.normalise_to_string(kwargs) with asection(f"PythonFunctionsInfoTool: query= {query} "): diff --git a/src/napari_chatgpt/omega/tools/special/human_input_tool.py b/src/napari_chatgpt/omega/tools/special/human_input_tool.py index 7b70e2a..b72496e 100644 --- a/src/napari_chatgpt/omega/tools/special/human_input_tool.py +++ b/src/napari_chatgpt/omega/tools/special/human_input_tool.py @@ -1,6 +1,6 @@ """Tool for asking human input.""" -from typing import Callable +from typing import Callable, Any from pydantic import Field @@ -25,7 +25,13 @@ class HumanInputTool(AsyncBaseTool): default_factory=lambda: _print_func) input_func: Callable = Field(default_factory=lambda: input) - def _run(self, query: str) -> str: + def _run(self, + *args: Any, + **kwargs: Any + ) -> Any: + # Get query: + query = self.normalise_to_string(kwargs) + """Use the Human input tool.""" self.prompt_func(query) return self.input_func() diff --git a/src/napari_chatgpt/omega/tools/special/package_info_tool.py b/src/napari_chatgpt/omega/tools/special/package_info_tool.py new file mode 100644 index 0000000..7e35a78 --- /dev/null +++ b/src/napari_chatgpt/omega/tools/special/package_info_tool.py @@ -0,0 +1,60 @@ +"""A tool for running python code in a REPL.""" +import traceback +from typing import Any + +from arbol import asection, aprint + +from napari_chatgpt.omega.tools.async_base_tool import AsyncBaseTool +from napari_chatgpt.utils.python.installed_packages import \ + installed_package_list +from napari_chatgpt.utils.python.relevant_libraries import \ + get_all_relevant_packages + + +class PythonPackageInfoTool(AsyncBaseTool): + """A tool for querying and searching the list of installed packages.""" + + name = "PackageInfoTool" + description = ( + "Use this tool for querying and searching the list of installed package sin the system. " + "You can provide a substring to search for a specific package or list of packages. " + "For example, send and empty string to get the full list of installed packages. " + "For example, send: `numpy` to get the information about the numpy package. " + ) + + def _run(self, + *args: Any, + **kwargs: Any + ) -> Any: + + # Get query: + query = self.normalise_to_string(kwargs) + + with asection(f"PythonPackageInfoTool: query= {query} "): + + try: + # remove white spaces and other non alphanumeric characters from the query: + query = query.strip() + + # Get list of all python packages installed + packages = installed_package_list(filter=None) + + # If query is not empty, filter the list of packages: + if query: + packages = [p for p in packages if query.lower() in p.lower()] + + # If the list of packages is too long, restrict to signal processing related packages, + # then take the intersection of packages and get_all_relevant_packages(): + if len(packages) > 50: + packages = [p for p in packages if p.lower() in get_all_relevant_packages()] + + # convert the list of packages to a string: + result = "\n".join(packages) + + aprint(result) + return result + + except Exception as e: + error_info = f"Error: {type(e).__name__} with message: '{str(e)}' occurred while trying to get information about packages containing: '{query}'." + traceback.print_exc() + return error_info diff --git a/src/napari_chatgpt/omega/tools/special/pip_install_tool.py b/src/napari_chatgpt/omega/tools/special/pip_install_tool.py index 667fd93..cd69b75 100644 --- a/src/napari_chatgpt/omega/tools/special/pip_install_tool.py +++ b/src/napari_chatgpt/omega/tools/special/pip_install_tool.py @@ -1,5 +1,6 @@ """A tool for running python code in a REPL.""" import traceback +from typing import Any from arbol import asection, aprint @@ -21,8 +22,13 @@ class PipInstallTool(AsyncBaseTool): "This tool is useful for installing packages that are not installed by default in the napari environment. " ) - def _run(self, query: str) -> str: - """Use the tool.""" + def _run(self, + *args: Any, + **kwargs: Any + ) -> Any: + + # Get query: + query = self.normalise_to_string(kwargs) with asection(f"PipInstallTool: query= {query} "): diff --git a/src/napari_chatgpt/omega/tools/special/python_repl.py b/src/napari_chatgpt/omega/tools/special/python_repl.py index cbb5e77..3f3888b 100644 --- a/src/napari_chatgpt/omega/tools/special/python_repl.py +++ b/src/napari_chatgpt/omega/tools/special/python_repl.py @@ -2,72 +2,74 @@ import re from contextlib import redirect_stdout from io import StringIO -from typing import Dict, Optional - -from langchain.callbacks.manager import ( - CallbackManagerForToolRun, -) -from pydantic import Field +from typing import Dict, Optional, Any from napari_chatgpt.omega.tools.async_base_tool import AsyncBaseTool -def sanitize_input(query: str) -> str: - # Remove whitespace, backtick & python (if llm mistakes python console as terminal) - - # Removes `, whitespace & python from start - query = re.sub(r"^(\s|`)*(?i:python)?\s*", "", query) - # Removes whitespace & ` from end - query = re.sub(r"(\s|`)*$", "", query) - return query - - class PythonCodeExecutionTool(AsyncBaseTool): """A tool for running non-napari-related python code in a REPL.""" name = "PythonCodeExecutionTool" description = ( - "Use this tool to execute short snippets of python code unrelated to images. " - "Do not use this tool if you need access to the napari viewer or its layers: instead use the napari viewer query, control or execution tools. " - "This tool is absolutely *not* suitable for generating, processing, analysing or visualising images, videos, large nD arrays, or other large datasets. " - "Input should be a short and valid python command. " + "Use this tool *sparingly* to execute very short snippets of python code. " + "Do *not* use this tool to access to the napari viewer or its layers. " + "Do *not* use this tool to work on images, videos, large nD arrays, or other large datasets. " + "Input should be a *very short* and valid python command, ideally a print statement." "For example, send: `print(3**3+1)` to get the result of this calculation which is 28. " "If you want to see the output, you should print it out with `print(...)`." ) - globals: Optional[Dict] = Field(default_factory=dict) - locals: Optional[Dict] = Field(default_factory=dict) sanitize_input: bool = True def _run( self, - query: str, - run_manager: Optional[CallbackManagerForToolRun] = None, - ) -> str: - """Use the tool.""" + *args: Any, + **kwargs: Any + ) -> Any: + try: + _globals = globals() + _locals = locals() + + # Get query: + query = self.normalise_to_string(kwargs) + + # Sanitize input: if self.sanitize_input: query = sanitize_input(query) + # add code cell to notebook if available: if self.notebook: self.notebook.add_code_cell(query) + # Parse and execute the code: tree = ast.parse(query) module = ast.Module(tree.body[:-1], type_ignores=[]) - exec(ast.unparse(module), self.globals, self.locals) # type: ignore + exec(ast.unparse(module), _globals, _locals) # type: ignore module_end = ast.Module(tree.body[-1:], type_ignores=[]) module_end_str = ast.unparse(module_end) # type: ignore io_buffer = StringIO() try: with redirect_stdout(io_buffer): - ret = eval(module_end_str, self.globals, self.locals) + ret = eval(module_end_str, _globals, _locals) if ret is None: return io_buffer.getvalue() else: return ret except Exception: with redirect_stdout(io_buffer): - exec(module_end_str, self.globals, self.locals) + exec(module_end_str, _globals, _locals) return io_buffer.getvalue() except Exception as e: return "{}: {}".format(type(e).__name__, str(e)) + + +def sanitize_input(query: str) -> str: + # Remove whitespace, backtick & python (if llm mistakes python console as terminal) + + # Removes `, whitespace & python from start + query = re.sub(r"^(\s|`)*(?i:python)?\s*", "", query) + # Removes whitespace & ` from end + query = re.sub(r"(\s|`)*$", "", query) + return query \ No newline at end of file diff --git a/src/napari_chatgpt/utils/napari/test/napari_viewer_info_test.py b/src/napari_chatgpt/utils/napari/test/napari_viewer_info_test.py index 4162aca..910e409 100644 --- a/src/napari_chatgpt/utils/napari/test/napari_viewer_info_test.py +++ b/src/napari_chatgpt/utils/napari/test/napari_viewer_info_test.py @@ -89,17 +89,26 @@ def test_napari_viewer_info(): vectors = numpy.zeros((n, 2, 2), dtype=numpy.float32) phi_space = numpy.linspace(0, 4 * numpy.pi, n) radius_space = numpy.linspace(0, 100, n) + # assign x-y projection vectors[:, 1, 0] = radius_space * numpy.cos(phi_space) vectors[:, 1, 1] = radius_space * numpy.sin(phi_space) + # assign x-y position vectors[:, 0] = vectors[:, 1] + 256 + # add the vectors vectors_layer = viewer.add_vectors(vectors, edge_width=3) # GET LAYER INFO FROM VIEWER: layers_info = get_viewer_info(viewer) + # Print the layers_info: aprint(layers_info) + # Check that the layers_info is not empty: assert len(layers_info) > 0 + + # Close the viewer: + viewer.close() + diff --git a/src/napari_chatgpt/utils/network/demo/port_available_demo.py b/src/napari_chatgpt/utils/network/demo/port_available_demo.py index 2f231f9..508a08e 100644 --- a/src/napari_chatgpt/utils/network/demo/port_available_demo.py +++ b/src/napari_chatgpt/utils/network/demo/port_available_demo.py @@ -1,11 +1,16 @@ # main to test automatic port increment in the omega server: if __name__ == '__main__': + # now start a simple server asynchronously on that port to occupy it: import asyncio from aiohttp import web + + # Define a simple handler that returns a simple response: async def handle(request): return web.Response(text="Hello, world") + + # Start the server: app = web.Application() app.router.add_get('/', handle) runner = web.AppRunner(app) @@ -14,7 +19,11 @@ async def handle(request): try: loop.run_until_complete(runner.setup()) site = web.TCPSite(runner, 'localhost', 9000) + + # Start the server: loop.run_until_complete(site.start()) + + # wait until key pressed on terminal: input("Press Enter to continue...") except Exception as e: print(f"Error occurred: {e}") diff --git a/src/napari_chatgpt/utils/openai/gpt_vision.py b/src/napari_chatgpt/utils/openai/gpt_vision.py index 2ba1502..2e5f0ab 100644 --- a/src/napari_chatgpt/utils/openai/gpt_vision.py +++ b/src/napari_chatgpt/utils/openai/gpt_vision.py @@ -130,7 +130,7 @@ def describe_image(image_path: str, # if the response contains these words: "sorry" and ("I cannot" or "I can't") then try again: if ("sorry" in response_lc and ("i cannot" in response_lc or "i can't" in response_lc or 'i am unable' in response_lc)) \ - or "i cannot assist" in response_lc: + or "i cannot assist" in response_lc or "i can't assist" in response_lc or 'i am unable to assist' in response_lc or "I'm sorry" in response_lc: aprint(f"Vision model refuses to assist (response: {response}). Trying again...") continue else: diff --git a/src/napari_chatgpt/utils/openai/model_list.py b/src/napari_chatgpt/utils/openai/model_list.py index 98865f8..e4f00c5 100644 --- a/src/napari_chatgpt/utils/openai/model_list.py +++ b/src/napari_chatgpt/utils/openai/model_list.py @@ -79,6 +79,8 @@ def postprocess_openai_model_list(model_list: list) -> list: """ try: + # First, sort the list of models: + model_list = sorted(model_list) # get list of bad models for main LLM: bad_models_filters = ['0613', 'vision', @@ -109,7 +111,7 @@ def postprocess_openai_model_list(model_list: list) -> list: # Ensure that the very best models are at the top of the list: very_best_models = [m for m in model_list if - ('gpt-4o' in m)] + ('gpt-4o' in m and not 'mini' in m)] model_list = very_best_models + [m for m in model_list if m not in very_best_models] diff --git a/src/napari_chatgpt/utils/python/relevant_libraries.py b/src/napari_chatgpt/utils/python/relevant_libraries.py index 1ba0990..cad30e1 100644 --- a/src/napari_chatgpt/utils/python/relevant_libraries.py +++ b/src/napari_chatgpt/utils/python/relevant_libraries.py @@ -6,6 +6,21 @@ def get_all_signal_processing_related_packages(): return list_of_signal_processing_related_packages +def get_all_essential_packages(): + + # Since the list was generated by ChatGPT 4, we first remove duplicates from the list: + list_of_essential_packages = list(set(_essential_packages)) + + return list_of_essential_packages + + +def get_all_relevant_packages(): + + # Since the list was generated by ChatGPT 4, we first remove duplicates from the list: + list_of_relevant_packages = list(set(_essential_packages + _signal_processing_related_packages)) + + return list_of_relevant_packages + _essential_packages = \ [ 'numpy', # Fundamental package for numerical computations diff --git a/src/napari_chatgpt/utils/web/test/duckduckgo_test.py b/src/napari_chatgpt/utils/web/test/duckduckgo_test.py index e4b736f..e72c790 100644 --- a/src/napari_chatgpt/utils/web/test/duckduckgo_test.py +++ b/src/napari_chatgpt/utils/web/test/duckduckgo_test.py @@ -19,8 +19,8 @@ def test_duckduckgo_search_overview_summary(): except RatelimitException as e: aprint(f"RatelimitException: {e}") - aprint(f"RatelimitException: {e.response}") - aprint(f"RatelimitException: {e.response.text}") + import traceback + traceback.print_exc() @@ -36,5 +36,5 @@ def test_duckduckgo_search_overview(): except RatelimitException as e: aprint(f"RatelimitException: {e}") - aprint(f"RatelimitException: {e.response}") - aprint(f"RatelimitException: {e.response.text}") + import traceback + traceback.print_exc() diff --git a/src/napari_chatgpt/utils/web/test/google_test.py b/src/napari_chatgpt/utils/web/test/google_test.py index ffcfb94..ce26ce6 100644 --- a/src/napari_chatgpt/utils/web/test/google_test.py +++ b/src/napari_chatgpt/utils/web/test/google_test.py @@ -14,7 +14,8 @@ def test_google_search_overview(): except RatelimitException as e: aprint(f"RatelimitException: {e}") - aprint(f"RatelimitException: {e.response}") - aprint(f"RatelimitException: {e.response.text}") + import traceback + traceback.print_exc() + diff --git a/src/napari_chatgpt/utils/web/test/metasearch_test.py b/src/napari_chatgpt/utils/web/test/metasearch_test.py index 44a2806..447d67e 100644 --- a/src/napari_chatgpt/utils/web/test/metasearch_test.py +++ b/src/napari_chatgpt/utils/web/test/metasearch_test.py @@ -19,11 +19,8 @@ def test_metasearch_summary(): except RatelimitException as e: aprint(f"RatelimitException: {e}") - aprint(f"RatelimitException: {e.response}") - aprint(f"RatelimitException: {e.response.text}") - - - + import traceback + traceback.print_exc() def test_metasearch(): @@ -37,8 +34,9 @@ def test_metasearch(): except RatelimitException as e: aprint(f"RatelimitException: {e}") - aprint(f"RatelimitException: {e.response}") - aprint(f"RatelimitException: {e.response.text}") + import traceback + traceback.print_exc() + diff --git a/src/napari_chatgpt/utils/web/test/wikipedia_test.py b/src/napari_chatgpt/utils/web/test/wikipedia_test.py index 89e6cc9..9c9fcac 100644 --- a/src/napari_chatgpt/utils/web/test/wikipedia_test.py +++ b/src/napari_chatgpt/utils/web/test/wikipedia_test.py @@ -17,10 +17,13 @@ def test_wikipedia_search_MM(): aprint(text) + assert 'Mickey Mouse' in text + except RatelimitException as e: aprint(f"RatelimitException: {e}") - aprint(f"RatelimitException: {e.response}") - aprint(f"RatelimitException: {e.response.text}") + import traceback + traceback.print_exc() + @@ -36,10 +39,15 @@ def test_wikipedia_search_AE(): do_summarize=True) aprint(text) + + assert 'Albert Einstein' in text + except RatelimitException as e: aprint(f"RatelimitException: {e}") - aprint(f"RatelimitException: {e.response}") - aprint(f"RatelimitException: {e.response.text}") + import traceback + traceback.print_exc() + + @pytest.mark.skipif(not is_api_key_available('OpenAI'), @@ -55,10 +63,16 @@ def test_wikipedia_search_CZB(): aprint(text) + assert 'CZ Biohub' in text + except RatelimitException as e: aprint(f"RatelimitException: {e}") - aprint(f"RatelimitException: {e.response}") - aprint(f"RatelimitException: {e.response.text}") + + + + + +