share/search-agent.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "2d24defe-66ea-4478-9ad4-f485c7f2daa0",
   "metadata": {},
   "source": [
    "# Motivating work\n",
    "\n",
    "I retrieved that notebook to document what I was thinking about when I started thinking about that..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "id": "cea7dbfd-0a9a-410e-bb9a-99c9a582f816",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import io, os, sys, json, time, string\n",
    "\n",
    "import openai\n",
    "openai.api_key=\"sk-XXXXX\"\n",
    "\n",
    "os.environ[\"LD_LIBRARY_PATH\"] = \"/usr/lib/x86_64-linux-gnu\"\n",
    "os.environ[\"OPENAI_API_KEY\"] = openai.api_key\n",
    "os.environ[\"SERPAPI_API_KEY\"] = \"XXXX\"\n",
    "\n",
    "from langchain.cache import InMemoryCache\n",
    "import langchain\n",
    "langchain.llm_cache = InMemoryCache()\n",
    "\n",
    "from langchain import LLMChain\n",
    "from langchain.prompts import PromptTemplate\n",
    "from langchain import OpenAI\n",
    "\n",
    "def make_prompt_template(prompt, **kwargs):\n",
    "    variables = [v[1] for v in string.Formatter().parse(prompt)]\n",
    "    input_variables = []\n",
    "    for var in variables:\n",
    "        if var is not None and not var in kwargs:\n",
    "            kwargs.update({ var : '{'+var+'}' })\n",
    "            input_variables.append(var)\n",
    "    return PromptTemplate(template=prompt.format(**kwargs), input_variables=input_variables)\n",
    "\n",
    "from ipywebrtc import AudioRecorder, CameraStream\n",
    "import torchaudio\n",
    "from IPython.display import Audio\n",
    "import tqdm\n",
    "\n",
    "base_prompt = '\\n\\n'.join([\n",
    "    \"You are the {ai_agent} automaton in the AI system {ai_name}.\",\n",
    "    \"# Topic\", \"{topic}\", \"# {ai_name} Mind\",\n",
    "    \"Text summary of {ai_name}' concepts and ideas on the Topic.\",\n",
    "    \"{ai_mind}\", \"# Scratch Pad\"\n",
    "])\n",
    "\n",
    "cold_llm = OpenAI(temperature=0.05)\n",
    "hot_llm = OpenAI(temperature=0.7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "befa62c7-c48a-4f58-8d9c-249d0ec8f337",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from serpapi import GoogleSearch\n",
    "\n",
    "def run_search_api(query, start=0):\n",
    "    start = 0\n",
    "    search = GoogleSearch({\n",
    "      \"start\": str(start), \"num\": \"10\",\n",
    "      \"q\": query,\n",
    "      \"hl\": \"en\", \"gl\": \"us\", \"engine\": \"google_scholar\",\n",
    "      \"google_domain\": \"google.com\",\n",
    "      \"api_key\": os.environ[\"SERPAPI_API_KEY\"]\n",
    "    })\n",
    "    results = search.get_dict()\n",
    "    results = results[\"organic_results\"]\n",
    "    results = [ [ r[k] for k in [ 'link', 'title', 'snippet' ] ] for r in results ]\n",
    "    return ( results, start+len(results) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "aa646a73-87e0-4ed9-8f51-5ac2b1687965",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "query_prompt = base_prompt + \"\"\"\\n\n",
    "You need to formulate an appropriate search query.\n",
    "The query should be simple: one or two nominal groups with a single verb or conjunction.\n",
    "It should be specific: it should be closely related to the \"Topic being research\".\n",
    "\n",
    "Use the following format:\n",
    "\n",
    "Reflection: identify the knowledge missing from \"{ai_name} Mind\" with respect to the \"Topic being research\"\n",
    "Goal: what information do you need to obtain? \n",
    "Query: the simple and specfic query that will best fulfill your Goal\n",
    "\n",
    "Begin!\n",
    "\n",
    "Reflection: \"\"\"\n",
    "query_chain = LLMChain(llm=cold_llm, prompt=make_prompt_template(\n",
    "    prompt=query_prompt,\n",
    "    ai_name='Helios',\n",
    "    ai_agent='search',\n",
    "    input_title=\"Topic being researched\",\n",
    "    task_title=\"Search\"\n",
    "))\n",
    "\n",
    "def run_query_chain(**kwargs):\n",
    "    reflection = query_chain(kwargs)['text']\n",
    "    (reflection, goal) = map(lambda x: x.strip(), reflection.split('Goal:'))\n",
    "    (goal, query) = map(lambda x: x.strip(), goal.split('Query:'))\n",
    "    return { 'reflection' : reflection, 'goal' : goal, 'query' : query }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "d6e12c47-f313-4098-80c5-e2b884c98e00",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "select_prompt = base_prompt + \"\"\"\\n\n",
    "You have reflected on the \"Topic being researched\", established a goal for the search, and provided a query.\n",
    "You need to decide whether the item shown below is relevant for your goals.\n",
    "\n",
    "Use the following format:\n",
    "\n",
    "Reflection: identify the knowledge missing from \"{ai_name} Mind\" with respect to the \"Topic being researched\"\n",
    "Goal: what information do you need to obtain? \n",
    "Query: the simple and specfic query that will best fulfill your Goal\n",
    "Title: the title provided for the item\n",
    "Identifier: a unique identifier such as a URL, file-path, physical addresse\n",
    "Description: a description or snippet of the results\n",
    "Observation: infer from the title, identifier, and description the information that could be found in the item\n",
    "Relevant: yes/no \n",
    "\n",
    "Begin!\n",
    "\n",
    "Reflection: {reflection}\n",
    "Goal: {goal}\n",
    "Query: {query}\n",
    "Title: {item_title}\n",
    "Identifier: {item_uid}\n",
    "Description: {item_desc}\n",
    "Observation: \"\"\"\n",
    "select_chain = LLMChain(llm=cold_llm, prompt=make_prompt_template(\n",
    "    prompt=select_prompt,\n",
    "    ai_name='Helios',\n",
    "    ai_agent='search',\n",
    "    input_title=\"Topic being researched\",\n",
    "    task_title=\"Search\"\n",
    "))\n",
    "\n",
    "def run_select_chain(**kwargs):\n",
    "    observation = select_chain(kwargs)['text']\n",
    "    (observation,relevant) = list(map(lambda x: x.strip(), observation.split('Relevant:')))\n",
    "    return { 'observation' : observation, 'relevant' : relevant }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "14533304-006f-4df9-8369-d1b3703a5571",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "summary_prompt = base_prompt + \"\"\"\\n\n",
    "You have reflected on the \"Topic being researched\", established a goal for the search, and provided a query.\n",
    "You have filtered the search results and selected the ones that seem relevant.\n",
    "Write a summary of these finding, cross reference the item in your summary.\n",
    "Use number to reference items, for example: references to the 3rd and 5th items would be \"[3,5]\".\n",
    "\n",
    "Use the following format:\n",
    "\n",
    "Reflection: identify the knowledge missing from \"{ai_name} Mind\" with respect to the \"Topic being researched\"\n",
    "Goal: what information do you need to obtain? \n",
    "Query: the simple and specfic query that will best fulfill your Goal\n",
    "Title 1: the title provided for the 1st item\n",
    "Identifier 1: a unique identifier such as a URL, file-path, physical addresse for the 1st item\n",
    "Description 1: a description or snippet of the results for the 1st item\n",
    "Observation 1: your observation about this item\n",
    "... Title, Identifier, Description, and Observation are repeated as many time as there are items\n",
    "Categories: Identify common theme, keyword, or sources accross these items, items can be in more than one category\n",
    "Category 1: Description of the category\n",
    "Items 1:    list items that fit the description\n",
    "... identify ut to 5 categories\n",
    "Summary: Write your summary, one or two sentences per categories with citation at the end of each sentence.\n",
    "\n",
    "Begin!\n",
    "\n",
    "Reflection: {reflection}\n",
    "Goal: {goal}\n",
    "Query: {query}\n",
    "{selections}\n",
    "Categories: \"\"\"\n",
    "summary_chain = LLMChain(llm=hot_llm, prompt=make_prompt_template(\n",
    "    prompt=summary_prompt,\n",
    "    ai_name='Helios',\n",
    "    ai_agent='search',\n",
    "    input_title=\"Topic being researched\",\n",
    "    task_title=\"Search\"\n",
    "))\n",
    "\n",
    "def run_summary_chain(**kwargs):\n",
    "    categories = summary_chain(kwargs)['text']\n",
    "    (categories,summary) = list(map(lambda x: x.strip(), categories.split('Summary:')))\n",
    "    return { 'categories' : categories.split('\\n'), 'summary' : summary }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "45fbf36d-e1cf-46f4-a1fd-5b36868e1662",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def run_search(topic, ai_mind=\"I am a blank slate\", select=3, max_pull=3):\n",
    "    data = { 'topic' : topic, \"ai_mind\" : ai_mind }\n",
    "#    print(json.dumps(data, indent=4))\n",
    "    data.update(run_query_chain(**data))\n",
    "#    print(json.dumps(data, indent=4))\n",
    "    start = 0\n",
    "    selections = []\n",
    "    for i in range(max_pull):\n",
    "#        print(i, start)\n",
    "        (items, start) = run_search_api(data['query'], start=start)\n",
    "#        print(items)\n",
    "        for (item_uid, item_title, item_desc) in items:\n",
    "#            print(item_uid)\n",
    "            response = run_select_chain(item_uid=item_uid, item_title=item_title, item_desc=item_desc, **data)\n",
    "#            print(json.dumps(response, indent=4))\n",
    "            if response['relevant'].split(' ')[0].split('\\n')[0].lower() == 'yes':\n",
    "                selections.append({\n",
    "                    'uid'   : item_uid,\n",
    "                    'title' : item_title,\n",
    "                    'desc'  : item_desc\n",
    "                })\n",
    "                selections[-1].update(**response)\n",
    "        if len(selections) >= select:\n",
    "            break\n",
    "    selections_str = '\\n'.join([\n",
    "        f\"\"\"\\\n",
    "Title {i}: {s['title']}\n",
    "Identifier {i}: {s['uid']}\n",
    "Description {i}: {s['desc']}\n",
    "Observation {i}: {s['observation']}\"\"\"\n",
    "        for (i,s) in enumerate(selections)\n",
    "    ])\n",
    "    response = run_summary_chain(selections=selections_str, **data)\n",
    "    data.update({ 'selections' : selections })\n",
    "    data.update(response)\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "bedf0489-df11-4cea-9096-4b5629c29b72",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "    \"topic\": \"Testing an AI agent\",\n",
      "    \"ai_mind\": \"I am a blank slate\",\n",
      "    \"reflection\": \"Helios Mind does not have any information on testing an AI agent.\",\n",
      "    \"goal\": \"To find information on how to test an AI agent.\",\n",
      "    \"query\": \"How to test AI agents?\",\n",
      "    \"selections\": [\n",
      "        {\n",
      "            \"uid\": \"https://www.functionize.com/page/ai-testing-with-intelligent-test-agent\",\n",
      "            \"title\": \"AI testing with Functionize's intelligent test agent\",\n",
      "            \"desc\": \"Functionize uses artificial intelligence to simplify the whole process of automated testing, from writing new tests to analyzing the test results. AI testing ...\",\n",
      "            \"observation\": \"This item provides information on how to use Functionize's intelligent test agent to test AI agents.\",\n",
      "            \"relevant\": \"Yes\"\n",
      "        },\n",
      "        {\n",
      "            \"uid\": \"https://techbeacon.com/app-dev-testing/ai-testing-13-essential-resources-qa-pros\",\n",
      "            \"title\": \"AI in testing: 13 essential resources for QA pros\",\n",
      "            \"desc\": \"AGENT is an acronym for \\\"AI Generation and Exploration in Test.\\\" Find this bot generator for testing your sites in Tariq King's repos on GitHub. AGENT, using ...\",\n",
      "            \"observation\": \"This item provides resources for QA pros on how to use AI in testing.\",\n",
      "            \"relevant\": \"Yes\"\n",
      "        },\n",
      "        {\n",
      "            \"uid\": \"https://www.openaccessgovernment.org/article/artificial-intelligence-test-agents-for-automated-testing-of-extended-reality-xr-ai/149203/\",\n",
      "            \"title\": \"Artificial Intelligence test agents for automated testing of ...\",\n",
      "            \"desc\": \"Artificial Intelligence agents can actively pursue testing goals (e.g., find the fastest way to the door), adapt to change (e.g., if the path is ...\",\n",
      "            \"observation\": \"This article provides information on how to use AI agents for automated testing of extended reality (XR) and AI.\",\n",
      "            \"relevant\": \"Yes\"\n",
      "        },\n",
      "        {\n",
      "            \"uid\": \"https://blog.emumba.com/ai-driven-testing-intelligent-way-of-testing-a-software-8a9194cf420c\",\n",
      "            \"title\": \"AI-Driven Testing\\u2014 Intelligent Way of Testing a Software\",\n",
      "            \"desc\": \"AI-driven testing means delegating some of the human tasks in the manual as well as automation testing to artificial intelligence. The two main ...\",\n",
      "            \"observation\": \"This article provides information on how to use AI to test software.\",\n",
      "            \"relevant\": \"Yes\"\n",
      "        },\n",
      "        {\n",
      "            \"uid\": \"https://study.com/academy/exam/topic/intelligent-agents.html\",\n",
      "            \"title\": \"Intelligent Agents - Practice Test Questions & Chapter Exam\",\n",
      "            \"desc\": \"Intelligent Agents Chapter Exam. Exam Instructions: Choose your answers to the questions and click 'Next' to see the next set of questions.\",\n",
      "            \"observation\": \"This item could provide practice questions and exams related to intelligent agents.\",\n",
      "            \"relevant\": \"Yes\"\n",
      "        },\n",
      "        {\n",
      "            \"uid\": \"https://servian.dev/why-testing-a-dialogflow-agent-is-different-to-traditional-software-testing-9e81a71de016\",\n",
      "            \"title\": \"Why AI Testing is Different to Traditional Software Testing\",\n",
      "            \"desc\": \"Once you have trained your agent to respond to user utterances (in this case \\u201cPlay Lord of the Rings\\u201d or \\u201cSearch for Leonardo DiCaprio movies\\u201d), you will ...\",\n",
      "            \"observation\": \"This article provides information on the differences between traditional software testing and AI testing.\",\n",
      "            \"relevant\": \"Yes\"\n",
      "        },\n",
      "        {\n",
      "            \"uid\": \"https://www.techtarget.com/searchenterpriseai/definition/Turing-test\",\n",
      "            \"title\": \"What is the Turing Test?\",\n",
      "            \"desc\": \"The Turing Test is used to determine if a computer program or artificial intelligence agent is capable of thinking like a human.\",\n",
      "            \"observation\": \"The item provides information on the Turing Test, which is a test used to determine if an AI agent is capable of thinking like a human.\",\n",
      "            \"relevant\": \"Yes\"\n",
      "        },\n",
      "        {\n",
      "            \"uid\": \"https://testguild.com/ai-testing-bots/\",\n",
      "            \"title\": \"How Can We Trust AI Testing Bots?\",\n",
      "            \"desc\": \"The first one is AGENT (AI Generation and Exploration in Tests). AGENT, using training data from AGENT-X, autonomously learns to explore a ...\",\n",
      "            \"observation\": \"This article provides information on how to trust AI testing bots.\",\n",
      "            \"relevant\": \"Yes\"\n",
      "        },\n",
      "        {\n",
      "            \"uid\": \"https://www.javatpoint.com/turing-test-in-ai\",\n",
      "            \"title\": \"Turing Test in AI\",\n",
      "            \"desc\": \"Turing Test in AI \\u00b7 Interrogator: Are you a computer? \\u00b7 PlayerA (Computer): No \\u00b7 Interrogator: Multiply two large numbers such as (256896489*456725896) \\u00b7 Player A: ...\",\n",
      "            \"observation\": \"This item provides information on the Turing Test, which is a method of testing AI agents.\",\n",
      "            \"relevant\": \"Yes\"\n",
      "        },\n",
      "        {\n",
      "            \"uid\": \"https://www.geeksforgeeks.org/turing-test-artificial-intelligence/\",\n",
      "            \"title\": \"Turing Test in Artificial Intelligence\",\n",
      "            \"desc\": \"Imagine a game of three players having two humans and one computer, an interrogator(as a human) is isolated from the other two players. The ...\",\n",
      "            \"observation\": \"This article provides information on the Turing Test, which is a method of testing AI agents.\",\n",
      "            \"relevant\": \"Yes\"\n",
      "        }\n",
      "    ],\n",
      "    \"categories\": [\n",
      "        \"Category 1: AI Testing Agents\",\n",
      "        \"Items 1: Title 0, Title 1, Title 2, Title 3, Title 5, Title 7\",\n",
      "        \"Category 2: Turing Test\",\n",
      "        \"Items 2: Title 6, Title 8, Title 9\"\n",
      "    ],\n",
      "    \"summary\": \"AI testing agents can simplify the process of automated testing, from writing new tests to analyzing the results. AGENT is an acronym for \\\"AI Generation and Exploration in Test\\\" which can be used for testing websites (ref. [1]). AI-driven testing means delegating some of the human tasks in the manual as well as automation testing to artificial intelligence (ref. [3]). The Turing Test is used to determine if a computer program or artificial intelligence agent is capable of thinking like a human (ref. [6,8,9]). Additionally, AGENT-X can be used to learn to explore and trust AI testing bots (ref. [7]).\"\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "data = run_search(topic=\"Testing an AI agent\", ai_mind=\"I am a blank slate\", select=3, max_pull=3)\n",
    "print(json.dumps(data, indent=4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fda8e0da-9e8e-4792-86b4-1ac8d3392f87",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Reflection(text): XXX                   % Gaps in {ai_name}'s knowledge of the Topic\n",
    "Goal(text): XXX                         % Information needed\n",
    "Queries[{Q}](text): XXX                        % query for the search engine\n",
    "Item[{N}](record):                      % a list of item records\n",
    "> Title(short): XXX                     % \n",
    "> Identifier(short): XXX                % Unique ID, can be a URL, filepath, name, ...\n",
    "> Description(markdow): ```XXX```       % Description provided by the search engine\n",
    "> Observation(text):```XXX```           % Observation you made about this item\n",
    "Categories[{C}](record):                % You categorized the items based on topic\n",
    "> Topic(short): XXX                     % Description of the category\n",
    "> Items[{N}](short): XXX                % item's identifier, one per line\n",
    "Sections[{S}](record):                  %\n",
    "> Topic(short): XXX                     % topic of the section\n",
    "> Facts[{F}](short): XXX                % list of facts from the\n",
    "> Paragraphs[{P}](short): XXX           % concepts for each paragraph in the section\n",
    "Report(record):                         % \n",
    "> Title(short): XXX                     % title of the search report\n",
    "> Introduction(markdown):```XXX```      % Explain your goal and query\n",
    "> Sections[{S}](record):                % \n",
    "> > Subtitle(short): XXX                %   \n",
    "> > Paragraphs[{P}](markdown):```       %\n",
    "XXX XXX XXX                             % \n",
    "```                                     % \n",
    "> Conclusion(markdown):```XXX```        % \n",
    "\"\"\""
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}