notebooks/1.1-sr-dspy-RAG.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Retrieval Augmented Generation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Configure LLM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/suriya/projects/dspy/env-dspy/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import dspy\n",
    "\n",
    "# Set up the LM.\n",
    "OPENAI_API_KEY = open(\"../.secrets/openai-api_key.txt\").read()\n",
    "turbo = dspy.OpenAI(model='gpt-3.5-turbo-instruct', max_tokens=250, api_key=OPENAI_API_KEY)\n",
    "# configure language model\n",
    "dspy.settings.configure(lm=turbo)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setup ColBERTv2 Retriever\n",
    "\n",
    "A free server hosting a Wikipedia 2017 \"abstracts\" search index containing the first paragraph of each article from this 2017 dump."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "colbertv2 = dspy.ColBERTv2(url='http://20.102.90.50:2017/wiki17_abstracts')\n",
    "# configure retrieval model\n",
    "dspy.settings.configure(rm=colbertv2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Downloading builder script: 100%|██████████| 6.42k/6.42k [00:00<00:00, 10.6kB/s]\n",
      "Downloading readme: 100%|██████████| 9.19k/9.19k [00:00<00:00, 15.0kB/s]\n",
      "Downloading data: 100%|██████████| 566M/566M [10:48<00:00, 873kB/s]    \n",
      "Downloading data: 100%|██████████| 47.5M/47.5M [00:29<00:00, 1.58MB/s]\n",
      "Downloading data: 100%|██████████| 46.2M/46.2M [00:55<00:00, 826kB/s] \n",
      "Generating train split: 100%|██████████| 90447/90447 [00:13<00:00, 6899.10 examples/s]\n",
      "Generating validation split: 100%|██████████| 7405/7405 [00:01<00:00, 7146.72 examples/s]\n",
      "Generating test split: 100%|██████████| 7405/7405 [00:00<00:00, 7589.25 examples/s]\n"
     ]
    }
   ],
   "source": [
    "from dspy.datasets import HotPotQA\n",
    "\n",
    "# Load the dataset.\n",
    "dataset = HotPotQA(train_seed=1, train_size=20, eval_seed=69, dev_size=50, test_size=0)\n",
    "trainset = [x.with_inputs('question') for x in dataset.train]\n",
    "devset = [x.with_inputs('question') for x in dataset.dev]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('The Victorians - Their Story In Pictures is a documentary series written by an author born in what year?',\n",
       " '1950')"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainset[3].question, trainset[3].answer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define Signature\n",
    "\n",
    "For some advanced tasks, you need more verbose signatures defined using Class-based DSPy signatures.\n",
    "\n",
    "- Clarify something about the nature of the task\n",
    "- Supply hints on the nature of an input field, expressed as a desc keyword argument for `dspy.InputField`\n",
    "- Supply constraints on an output field, expressed as a desc keyword argument for `dspy.OutputField`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class GenerateAnswer(dspy.Signature):\n",
    "    \"\"\"Answer questions with short factoid answers.\"\"\"\n",
    "\n",
    "    context = dspy.InputField(desc=\"may contain relevant facts\")\n",
    "    question = dspy.InputField()\n",
    "    answer = dspy.OutputField(desc=\"often between 1 and 5 words\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Build the Pipeline\n",
    "\n",
    "1. Declare sub-modules in `__init__`\n",
    "2. Use sub-modules to define control flow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "class RAG(dspy.Module):\n",
    "    def __init__(self, num_passages=3):\n",
    "        super().__init__()\n",
    "        self.retrieve = dspy.Retrieve(k=num_passages)  # this should work since we've configured our retrieval model early on\n",
    "        self.generate_answer = dspy.ChainOfThought(GenerateAnswer)  # use the signature we defined in the previous step\n",
    "    \n",
    "    def forward(self, question):\n",
    "        # retrieve context = 3 passages from ColBERTv2\n",
    "        context = self.retrieve(question).passages\n",
    "        # Use CoT module to generate answer given a question\n",
    "        prediction = self.generate_answer(context=context, question=question)\n",
    "        return dspy.Prediction(context=context, answer=prediction.answer)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Optimize the Pipeline\n",
    "\n",
    "Optimization, in this case, is collecting and selecting good demonstrations for inclusion within the prompt(s).\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 55%|█████▌    | 11/20 [00:22<00:18,  2.06s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bootstrapped 4 full traces after 12 examples in round 0.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "from dspy.teleprompt import BootstrapFewShot\n",
    "\n",
    "# Validation logic: check that the predicted answer is correct.\n",
    "# Also check that the retrieved context does actually contain that answer.\n",
    "def validate_context_and_answer(example, pred, trace=None):\n",
    "    answer_EM = dspy.evaluate.answer_exact_match(example, pred)\n",
    "    answer_PM = dspy.evaluate.answer_passage_match(example, pred)\n",
    "    return answer_EM and answer_PM\n",
    "\n",
    "# Set up a basic teleprompter, which will compile our RAG program.\n",
    "teleprompter = BootstrapFewShot(metric=validate_context_and_answer)\n",
    "\n",
    "# Compile!\n",
    "compiled_rag = teleprompter.compile(RAG(), trainset=trainset)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Question: What castle did David Gregory inherit?\n",
      "Predicted Answer: Kinnairdy Castle\n",
      "Retrieved Contexts (truncated): ['David Gregory (physician) | David Gregory (20 December 1625 – 1720) was a Scottish physician and inventor. His surname is sometimes spelt as Gregorie, the original Scottish spelling. He inherited Kinn...', 'Gregory Tarchaneiotes | Gregory Tarchaneiotes (Greek: Γρηγόριος Ταρχανειώτης , Italian: \"Gregorio Tracanioto\" or \"Tracamoto\" ) was a \"protospatharius\" and the long-reigning catepan of Italy from 998 t...', 'David Gregory (mathematician) | David Gregory (originally spelt Gregorie) FRS (? 1659 – 10 October 1708) was a Scottish mathematician and astronomer. He was professor of mathematics at the University ...']\n"
     ]
    }
   ],
   "source": [
    "# Ask any question you like to this simple RAG program.\n",
    "my_question = \"What castle did David Gregory inherit?\"\n",
    "\n",
    "# Get the prediction. This contains `pred.context` and `pred.answer`.\n",
    "pred = compiled_rag(my_question)\n",
    "\n",
    "# Print the contexts and the answer.\n",
    "print(f\"Question: {my_question}\")\n",
    "print(f\"Predicted Answer: {pred.answer}\")\n",
    "print(f\"Retrieved Contexts (truncated): {[c[:200] + '...' for c in pred.context]}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "Answer questions with short factoid answers.\n",
      "\n",
      "---\n",
      "\n",
      "Question: At My Window was released by which American singer-songwriter?\n",
      "Answer: John Townes Van Zandt\n",
      "\n",
      "Question: \"Everything Has Changed\" is a song from an album released under which record label ?\n",
      "Answer: Big Machine Records\n",
      "\n",
      "Question: The Victorians - Their Story In Pictures is a documentary series written by an author born in what year?\n",
      "Answer: 1950\n",
      "\n",
      "Question: Which Pakistani cricket umpire who won 3 consecutive ICC umpire of the year awards in 2009, 2010, and 2011 will be in the ICC World Twenty20?\n",
      "Answer: Aleem Sarwar Dar\n",
      "\n",
      "Question: Having the combination of excellent foot speed and bat speed helped Eric Davis, create what kind of outfield for the Los Angeles Dodgers?\n",
      "Answer: \"Outfield of Dreams\"\n",
      "\n",
      "Question: Who is older, Aleksandr Danilovich Aleksandrov or Anatoly Fomenko?\n",
      "Answer: Aleksandr Danilovich Aleksandrov\n",
      "\n",
      "Question: The Organisation that allows a community to influence their operation or use and to enjoy the benefits arisingwas founded in what year?\n",
      "Answer: 2010\n",
      "\n",
      "Question: Tombstone stared an actor born May 17, 1955 known as who?\n",
      "Answer: Bill Paxton\n",
      "\n",
      "Question: In what year was the club founded that played Manchester City in the 1972 FA Charity Shield\n",
      "Answer: 1874\n",
      "\n",
      "Question: which American actor was Candace Kita guest starred with\n",
      "Answer: Bill Murray\n",
      "\n",
      "Question: Which is taller, the Empire State Building or the Bank of America Tower?\n",
      "Answer: The Empire State Building\n",
      "\n",
      "Question: Which company distributed this 1977 American animated film produced by Walt Disney Productions for which Sherman Brothers wrote songs?\n",
      "Answer: Buena Vista Distribution\n",
      "\n",
      "---\n",
      "\n",
      "Follow the following format.\n",
      "\n",
      "Context: may contain relevant facts\n",
      "\n",
      "Question: ${question}\n",
      "\n",
      "Reasoning: Let's think step by step in order to ${produce the answer}. We ...\n",
      "\n",
      "Answer: often between 1 and 5 words\n",
      "\n",
      "---\n",
      "\n",
      "Context:\n",
      "[1] «Tae Kwon Do Times | Tae Kwon Do Times is a magazine devoted to the martial art of taekwondo, and is published in the United States of America. While the title suggests that it focuses on taekwondo exclusively, the magazine also covers other Korean martial arts. \"Tae Kwon Do Times\" has published articles by a wide range of authors, including He-Young Kimm, Thomas Kurz, Scott Shaw, and Mark Van Schuyver.»\n",
      "[2] «Kwon Tae-man | Kwon Tae-man (born 1941) was an early Korean hapkido practitioner and a pioneer of the art, first in Korea and then in the United States. He formed one of the earliest dojang's for hapkido in the United States in Torrance, California, and has been featured in many magazine articles promoting the art.»\n",
      "[3] «Hee Il Cho | Cho Hee Il (born October 13, 1940) is a prominent Korean-American master of taekwondo, holding the rank of 9th \"dan\" in the martial art. He has written 11 martial art books, produced 70 martial art training videos, and has appeared on more than 70 martial arts magazine covers. Cho won several national and international competitions as a taekwondo competitor, and has appeared in several films, including \"Fight to Win\", \"Best of the Best\", \"Bloodsport II\", and \"Bloodsport III\". He founded the Action International Martial Arts Association (AIMAA) in 1980, and is its President. Cho is a member of both \"Black Belt\" magazine's Hall of Fame and \"Tae Kwon Do Times\" magazine's Hall of Fame.»\n",
      "\n",
      "Question: Which magazine has published articles by Scott Shaw, Tae Kwon Do Times or Southwest Art?\n",
      "\n",
      "Reasoning: Let's think step by step in order to determine which magazine has published articles by Scott Shaw. First, we know that Scott Shaw is an author who has written for a wide range of publications. Next, we know that Tae Kwon Do Times is a magazine that covers Korean martial arts and has published articles by Scott Shaw. On the other hand, Southwest Art is a magazine that focuses on art in the American Southwest and does not mention Scott Shaw as a contributor. Therefore, the answer is Tae Kwon Do Times.\n",
      "\n",
      "Answer: Tae Kwon Do Times\n",
      "\n",
      "---\n",
      "\n",
      "Context:\n",
      "[1] «Rosario Dawson | Rosario Isabel Dawson (born May 9, 1979) is an American actress, producer, singer, comic book writer, and political activist. She made her film debut in the 1995 teen drama \"Kids\". Her subsequent film roles include \"He Got Game\", \"Men in Black II\", \"25th Hour\", \"Rent\", \"Sin City\", \"Death Proof\", \"Seven Pounds\", \"\", and \"Top Five\". Dawson has also provided voice-over work for Disney and DC.»\n",
      "[2] «Sarai Gonzalez | Sarai Isaura Gonzalez (born 2005) is an American Latina child actress who made her professional debut at the age of 11 on the Spanish-language \"\"Soy Yo\"\" (\"That's Me\") music video by Bomba Estéreo. Cast as a \"nerdy\" tween with a \"sassy\" and \"confident\" attitude, her performance turned her into a \"Latina icon\" for \"female empowerment, identity and self-worth\". She subsequently appeared in two get out the vote videos for Latinos in advance of the 2016 United States elections.»\n",
      "[3] «Gabriela (2001 film) | Gabriela is a 2001 American romance film, starring Seidy Lopez in the title role alongside Jaime Gomez as her admirer Mike. The film has been cited as an inspiration behind the Premiere Weekend Club, which supports Latino film-making.»\n",
      "\n",
      "Question: Which American actress who made their film debut in the 1995 teen drama \"Kids\" was the co-founder of Voto Latino?\n",
      "\n",
      "Reasoning: Let's think step by step in order to find the answer. We know that the actress made her film debut in 1995 and has since appeared in various films. We also know that she is involved in political activism, specifically in promoting Latino voting rights. This information leads us to the conclusion that the actress is Rosario Dawson, who co-founded Voto Latino.\n",
      "\n",
      "Answer: Rosario Dawson\n",
      "\n",
      "---\n",
      "\n",
      "Context:\n",
      "[1] «Battle of Kursk | The Battle of Kursk was a Second World War engagement between German and Soviet forces on the Eastern Front near Kursk (450 km south-west of Moscow) in the Soviet Union during July and August 1943. The battle began with the launch of the German offensive, Operation Citadel (German: \"Unternehmen Zitadelle\" ), on 5 July, which had the objective of pinching off the Kursk salient with attacks on the base of the salient from north and south simultaneously. After the German offensive stalled on the northern side of the salient, on 12 July the Soviets commenced their Kursk Strategic Offensive Operation with the launch of Operation Kutuzov (Russian: Кутузов ) against the rear of the German forces in the northern side. On the southern side, the Soviets also launched powerful counterattacks the same day, one of which led to a large armoured clash, the Battle of Prokhorovka. On 3 August, the Soviets began the second phase of the Kursk Strategic Offensive Operation with the launch of Operation Polkovodets Rumyantsev (Russian: Полководец Румянцев ) against the German forces in the southern side of the Kursk salient.»\n",
      "[2] «Operation Mars | Operation Mars, also known as the Second Rzhev-Sychevka Offensive Operation (Russian: Вторая Ржевско-Сычёвская наступательная операция), was the codename for an offensive launched by Soviet forces against German forces during World War II. It took place between 25 November and 20 December 1942 around the Rzhev salient in the vicinity of Moscow.»\n",
      "[3] «Kholm Pocket | The Kholm Pocket (German: \"Kessel von Cholm\" ; Russian: Холмский котёл ) was the name given for the encirclement of German troops by the Red Army around Kholm south of Leningrad, during World War II on the Eastern Front, from 23 January 1942 until 5 May 1942. A much larger pocket was simultaneously surrounded in Demyansk, about 100 km to the northeast. These were the results of German retreat following their defeat during the Battle of Moscow.»\n",
      "\n",
      "Question: What is the code name for the German offensive that started this Second World War engagement on the Eastern Front (a few hundred kilometers from Moscow) between Soviet and German forces, which included 102nd Infantry Division?\n",
      "\n",
      "Reasoning: Let's think step by step in order to find the answer. We know that the battle was called the Battle of Kursk and that it was part of Operation Citadel. We also know that the battle began on July 5th and that it was a Second World War engagement between German and Soviet forces. Additionally, we know that the battle took place near Kursk, which is about 450 km southwest of Moscow. Finally, we know that the battle involved the 102nd Infantry Division.\n",
      "\n",
      "Answer: Operation Citadel\n",
      "\n",
      "---\n",
      "\n",
      "Context:\n",
      "[1] «Kerry Condon | Kerry Condon (born 4 January 1983) is an Irish television and film actress, best known for her role as Octavia of the Julii in the HBO/BBC series \"Rome,\" as Stacey Ehrmantraut in AMC's \"Better Call Saul\" and as the voice of F.R.I.D.A.Y. in various films in the Marvel Cinematic Universe. She is also the youngest actress ever to play Ophelia in a Royal Shakespeare Company production of \"Hamlet.\"»\n",
      "[2] «Corona Riccardo | Corona Riccardo (c. 1878October 15, 1917) was an Italian born American actress who had a brief Broadway stage career before leaving to become a wife and mother. Born in Naples she came to acting in 1894 playing a Mexican girl in a play at the Empire Theatre. Wilson Barrett engaged her for a role in his play \"The Sign of the Cross\" which he took on tour of the United States. Riccardo played the role of Ancaria and later played Berenice in the same play. Robert B. Mantell in 1898 who struck by her beauty also cast her in two Shakespeare plays, \"Romeo and Juliet\" and \"Othello\". Author Lewis Strang writing in 1899 said Riccardo was the most promising actress in America at the time. Towards the end of 1898 Mantell chose her for another Shakespeare part, Ophelia im Hamlet. Afterwards she was due to join Augustin Daly's Theatre Company but Daly died in 1899. In 1899 she gained her biggest fame by playing Iras in the first stage production of Ben-Hur.»\n",
      "[3] «Judi Dench | Dame Judith Olivia \"Judi\" Dench, {'1': \", '2': \", '3': \", '4': \"} (born 9 December 1934) is an English actress and author. Dench made her professional debut in 1957 with the Old Vic Company. Over the following few years, she performed in several of Shakespeare's plays in such roles as Ophelia in \"Hamlet\", Juliet in \"Romeo and Juliet\", and Lady Macbeth in \"Macbeth\". Although most of her work during this period was in theatre, she also branched into film work and won a BAFTA Award as Most Promising Newcomer. She drew strong reviews for her leading role in the musical \"Cabaret\" in 1968.»\n",
      "\n",
      "Question: Who acted in the shot film The Shore and is also the youngest actress ever to play Ophelia in a Royal Shakespeare Company production of \"Hamlet.\" ?\n",
      "\n",
      "Reasoning: Let's think step by step in order to find the answer. We know that the actress is Irish and was born on January 4, 1983. She is also known for her role in the HBO/BBC series \"Rome\" and as the voice of F.R.I.D.A.Y. in various films in the Marvel Cinematic Universe. She is also the youngest actress ever to play Ophelia in a Royal Shakespeare Company production of \"Hamlet.\"\n",
      "\n",
      "Answer: Kerry Condon\n",
      "\n",
      "---\n",
      "\n",
      "Context:\n",
      "[1] «David Gregory (physician) | David Gregory (20 December 1625 – 1720) was a Scottish physician and inventor. His surname is sometimes spelt as Gregorie, the original Scottish spelling. He inherited Kinnairdy Castle in 1664. Three of his twenty-nine children became mathematics professors. He is credited with inventing a military cannon that Isaac Newton described as \"being destructive to the human species\". Copies and details of the model no longer exist. Gregory's use of a barometer to predict farming-related weather conditions led him to be accused of witchcraft by Presbyterian ministers from Aberdeen, although he was never convicted.»\n",
      "[2] «Gregory Tarchaneiotes | Gregory Tarchaneiotes (Greek: Γρηγόριος Ταρχανειώτης , Italian: \"Gregorio Tracanioto\" or \"Tracamoto\" ) was a \"protospatharius\" and the long-reigning catepan of Italy from 998 to 1006. In December 999, and again on February 2, 1002, he reinstituted and confirmed the possessions of the abbey and monks of Monte Cassino in Ascoli. In 1004, he fortified and expanded the castle of Dragonara on the Fortore. He gave it three circular towers and one square one. He also strengthened Lucera.»\n",
      "[3] «David Gregory (mathematician) | David Gregory (originally spelt Gregorie) FRS (? 1659 – 10 October 1708) was a Scottish mathematician and astronomer. He was professor of mathematics at the University of Edinburgh, Savilian Professor of Astronomy at the University of Oxford, and a commentator on Isaac Newton's \"Principia\".»\n",
      "\n",
      "Question: What castle did David Gregory inherit?\n",
      "\n",
      "Reasoning: Let's think step by step in order to\u001b[32m find the answer. We know that David Gregory was a Scottish physician and inventor who lived from 1625 to 1720. He is also credited with inventing a military cannon and inherited a castle. Additionally, we know that he had 29 children, three of whom became mathematics professors.\n",
      "\n",
      "Answer: Kinnairdy Castle\u001b[0m\n",
      "\n",
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'\\n\\n\\nAnswer questions with short factoid answers.\\n\\n---\\n\\nQuestion: At My Window was released by which American singer-songwriter?\\nAnswer: John Townes Van Zandt\\n\\nQuestion: \"Everything Has Changed\" is a song from an album released under which record label ?\\nAnswer: Big Machine Records\\n\\nQuestion: The Victorians - Their Story In Pictures is a documentary series written by an author born in what year?\\nAnswer: 1950\\n\\nQuestion: Which Pakistani cricket umpire who won 3 consecutive ICC umpire of the year awards in 2009, 2010, and 2011 will be in the ICC World Twenty20?\\nAnswer: Aleem Sarwar Dar\\n\\nQuestion: Having the combination of excellent foot speed and bat speed helped Eric Davis, create what kind of outfield for the Los Angeles Dodgers?\\nAnswer: \"Outfield of Dreams\"\\n\\nQuestion: Who is older, Aleksandr Danilovich Aleksandrov or Anatoly Fomenko?\\nAnswer: Aleksandr Danilovich Aleksandrov\\n\\nQuestion: The Organisation that allows a community to influence their operation or use and to enjoy the benefits arisingwas founded in what year?\\nAnswer: 2010\\n\\nQuestion: Tombstone stared an actor born May 17, 1955 known as who?\\nAnswer: Bill Paxton\\n\\nQuestion: In what year was the club founded that played Manchester City in the 1972 FA Charity Shield\\nAnswer: 1874\\n\\nQuestion: which American actor was Candace Kita guest starred with\\nAnswer: Bill Murray\\n\\nQuestion: Which is taller, the Empire State Building or the Bank of America Tower?\\nAnswer: The Empire State Building\\n\\nQuestion: Which company distributed this 1977 American animated film produced by Walt Disney Productions for which Sherman Brothers wrote songs?\\nAnswer: Buena Vista Distribution\\n\\n---\\n\\nFollow the following format.\\n\\nContext: may contain relevant facts\\n\\nQuestion: ${question}\\n\\nReasoning: Let\\'s think step by step in order to ${produce the answer}. We ...\\n\\nAnswer: often between 1 and 5 words\\n\\n---\\n\\nContext:\\n[1] «Tae Kwon Do Times | Tae Kwon Do Times is a magazine devoted to the martial art of taekwondo, and is published in the United States of America. While the title suggests that it focuses on taekwondo exclusively, the magazine also covers other Korean martial arts. \"Tae Kwon Do Times\" has published articles by a wide range of authors, including He-Young Kimm, Thomas Kurz, Scott Shaw, and Mark Van Schuyver.»\\n[2] «Kwon Tae-man | Kwon Tae-man (born 1941) was an early Korean hapkido practitioner and a pioneer of the art, first in Korea and then in the United States. He formed one of the earliest dojang\\'s for hapkido in the United States in Torrance, California, and has been featured in many magazine articles promoting the art.»\\n[3] «Hee Il Cho | Cho Hee Il (born October 13, 1940) is a prominent Korean-American master of taekwondo, holding the rank of 9th \"dan\" in the martial art. He has written 11 martial art books, produced 70 martial art training videos, and has appeared on more than 70 martial arts magazine covers. Cho won several national and international competitions as a taekwondo competitor, and has appeared in several films, including \"Fight to Win\", \"Best of the Best\", \"Bloodsport II\", and \"Bloodsport III\". He founded the Action International Martial Arts Association (AIMAA) in 1980, and is its President. Cho is a member of both \"Black Belt\" magazine\\'s Hall of Fame and \"Tae Kwon Do Times\" magazine\\'s Hall of Fame.»\\n\\nQuestion: Which magazine has published articles by Scott Shaw, Tae Kwon Do Times or Southwest Art?\\n\\nReasoning: Let\\'s think step by step in order to determine which magazine has published articles by Scott Shaw. First, we know that Scott Shaw is an author who has written for a wide range of publications. Next, we know that Tae Kwon Do Times is a magazine that covers Korean martial arts and has published articles by Scott Shaw. On the other hand, Southwest Art is a magazine that focuses on art in the American Southwest and does not mention Scott Shaw as a contributor. Therefore, the answer is Tae Kwon Do Times.\\n\\nAnswer: Tae Kwon Do Times\\n\\n---\\n\\nContext:\\n[1] «Rosario Dawson | Rosario Isabel Dawson (born May 9, 1979) is an American actress, producer, singer, comic book writer, and political activist. She made her film debut in the 1995 teen drama \"Kids\". Her subsequent film roles include \"He Got Game\", \"Men in Black II\", \"25th Hour\", \"Rent\", \"Sin City\", \"Death Proof\", \"Seven Pounds\", \"\", and \"Top Five\". Dawson has also provided voice-over work for Disney and DC.»\\n[2] «Sarai Gonzalez | Sarai Isaura Gonzalez (born 2005) is an American Latina child actress who made her professional debut at the age of 11 on the Spanish-language \"\"Soy Yo\"\" (\"That\\'s Me\") music video by Bomba Estéreo. Cast as a \"nerdy\" tween with a \"sassy\" and \"confident\" attitude, her performance turned her into a \"Latina icon\" for \"female empowerment, identity and self-worth\". She subsequently appeared in two get out the vote videos for Latinos in advance of the 2016 United States elections.»\\n[3] «Gabriela (2001 film) | Gabriela is a 2001 American romance film, starring Seidy Lopez in the title role alongside Jaime Gomez as her admirer Mike. The film has been cited as an inspiration behind the Premiere Weekend Club, which supports Latino film-making.»\\n\\nQuestion: Which American actress who made their film debut in the 1995 teen drama \"Kids\" was the co-founder of Voto Latino?\\n\\nReasoning: Let\\'s think step by step in order to find the answer. We know that the actress made her film debut in 1995 and has since appeared in various films. We also know that she is involved in political activism, specifically in promoting Latino voting rights. This information leads us to the conclusion that the actress is Rosario Dawson, who co-founded Voto Latino.\\n\\nAnswer: Rosario Dawson\\n\\n---\\n\\nContext:\\n[1] «Battle of Kursk | The Battle of Kursk was a Second World War engagement between German and Soviet forces on the Eastern Front near Kursk (450 km south-west of Moscow) in the Soviet Union during July and August 1943. The battle began with the launch of the German offensive, Operation Citadel (German: \"Unternehmen Zitadelle\" ), on 5 July, which had the objective of pinching off the Kursk salient with attacks on the base of the salient from north and south simultaneously. After the German offensive stalled on the northern side of the salient, on 12 July the Soviets commenced their Kursk Strategic Offensive Operation with the launch of Operation Kutuzov (Russian: Кутузов ) against the rear of the German forces in the northern side. On the southern side, the Soviets also launched powerful counterattacks the same day, one of which led to a large armoured clash, the Battle of Prokhorovka. On 3 August, the Soviets began the second phase of the Kursk Strategic Offensive Operation with the launch of Operation Polkovodets Rumyantsev (Russian: Полководец Румянцев ) against the German forces in the southern side of the Kursk salient.»\\n[2] «Operation Mars | Operation Mars, also known as the Second Rzhev-Sychevka Offensive Operation (Russian: Вторая Ржевско-Сычёвская наступательная операция), was the codename for an offensive launched by Soviet forces against German forces during World War II. It took place between 25 November and 20 December 1942 around the Rzhev salient in the vicinity of Moscow.»\\n[3] «Kholm Pocket | The Kholm Pocket (German: \"Kessel von Cholm\" ; Russian: Холмский котёл ) was the name given for the encirclement of German troops by the Red Army around Kholm south of Leningrad, during World War II on the Eastern Front, from 23 January 1942 until 5 May 1942. A much larger pocket was simultaneously surrounded in Demyansk, about 100 km to the northeast. These were the results of German retreat following their defeat during the Battle of Moscow.»\\n\\nQuestion: What is the code name for the German offensive that started this Second World War engagement on the Eastern Front (a few hundred kilometers from Moscow) between Soviet and German forces, which included 102nd Infantry Division?\\n\\nReasoning: Let\\'s think step by step in order to find the answer. We know that the battle was called the Battle of Kursk and that it was part of Operation Citadel. We also know that the battle began on July 5th and that it was a Second World War engagement between German and Soviet forces. Additionally, we know that the battle took place near Kursk, which is about 450 km southwest of Moscow. Finally, we know that the battle involved the 102nd Infantry Division.\\n\\nAnswer: Operation Citadel\\n\\n---\\n\\nContext:\\n[1] «Kerry Condon | Kerry Condon (born 4 January 1983) is an Irish television and film actress, best known for her role as Octavia of the Julii in the HBO/BBC series \"Rome,\" as Stacey Ehrmantraut in AMC\\'s \"Better Call Saul\" and as the voice of F.R.I.D.A.Y. in various films in the Marvel Cinematic Universe. She is also the youngest actress ever to play Ophelia in a Royal Shakespeare Company production of \"Hamlet.\"»\\n[2] «Corona Riccardo | Corona Riccardo (c. 1878October 15, 1917) was an Italian born American actress who had a brief Broadway stage career before leaving to become a wife and mother. Born in Naples she came to acting in 1894 playing a Mexican girl in a play at the Empire Theatre. Wilson Barrett engaged her for a role in his play \"The Sign of the Cross\" which he took on tour of the United States. Riccardo played the role of Ancaria and later played Berenice in the same play. Robert B. Mantell in 1898 who struck by her beauty also cast her in two Shakespeare plays, \"Romeo and Juliet\" and \"Othello\". Author Lewis Strang writing in 1899 said Riccardo was the most promising actress in America at the time. Towards the end of 1898 Mantell chose her for another Shakespeare part, Ophelia im Hamlet. Afterwards she was due to join Augustin Daly\\'s Theatre Company but Daly died in 1899. In 1899 she gained her biggest fame by playing Iras in the first stage production of Ben-Hur.»\\n[3] «Judi Dench | Dame Judith Olivia \"Judi\" Dench, {\\'1\\': \", \\'2\\': \", \\'3\\': \", \\'4\\': \"} (born 9 December 1934) is an English actress and author. Dench made her professional debut in 1957 with the Old Vic Company. Over the following few years, she performed in several of Shakespeare\\'s plays in such roles as Ophelia in \"Hamlet\", Juliet in \"Romeo and Juliet\", and Lady Macbeth in \"Macbeth\". Although most of her work during this period was in theatre, she also branched into film work and won a BAFTA Award as Most Promising Newcomer. She drew strong reviews for her leading role in the musical \"Cabaret\" in 1968.»\\n\\nQuestion: Who acted in the shot film The Shore and is also the youngest actress ever to play Ophelia in a Royal Shakespeare Company production of \"Hamlet.\" ?\\n\\nReasoning: Let\\'s think step by step in order to find the answer. We know that the actress is Irish and was born on January 4, 1983. She is also known for her role in the HBO/BBC series \"Rome\" and as the voice of F.R.I.D.A.Y. in various films in the Marvel Cinematic Universe. She is also the youngest actress ever to play Ophelia in a Royal Shakespeare Company production of \"Hamlet.\"\\n\\nAnswer: Kerry Condon\\n\\n---\\n\\nContext:\\n[1] «David Gregory (physician) | David Gregory (20 December 1625 – 1720) was a Scottish physician and inventor. His surname is sometimes spelt as Gregorie, the original Scottish spelling. He inherited Kinnairdy Castle in 1664. Three of his twenty-nine children became mathematics professors. He is credited with inventing a military cannon that Isaac Newton described as \"being destructive to the human species\". Copies and details of the model no longer exist. Gregory\\'s use of a barometer to predict farming-related weather conditions led him to be accused of witchcraft by Presbyterian ministers from Aberdeen, although he was never convicted.»\\n[2] «Gregory Tarchaneiotes | Gregory Tarchaneiotes (Greek: Γρηγόριος Ταρχανειώτης , Italian: \"Gregorio Tracanioto\" or \"Tracamoto\" ) was a \"protospatharius\" and the long-reigning catepan of Italy from 998 to 1006. In December 999, and again on February 2, 1002, he reinstituted and confirmed the possessions of the abbey and monks of Monte Cassino in Ascoli. In 1004, he fortified and expanded the castle of Dragonara on the Fortore. He gave it three circular towers and one square one. He also strengthened Lucera.»\\n[3] «David Gregory (mathematician) | David Gregory (originally spelt Gregorie) FRS (? 1659 – 10 October 1708) was a Scottish mathematician and astronomer. He was professor of mathematics at the University of Edinburgh, Savilian Professor of Astronomy at the University of Oxford, and a commentator on Isaac Newton\\'s \"Principia\".»\\n\\nQuestion: What castle did David Gregory inherit?\\n\\nReasoning: Let\\'s think step by step in order to\\x1b[32m find the answer. We know that David Gregory was a Scottish physician and inventor who lived from 1625 to 1720. He is also credited with inventing a military cannon and inherited a castle. Additionally, we know that he had 29 children, three of whom became mathematics professors.\\n\\nAnswer: Kinnairdy Castle\\x1b[0m\\n\\n\\n'"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "turbo.inspect_history(n=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Question: Who is the first president of the USA?\n",
      "Predicted Answer: George Washington\n",
      "Retrieved Contexts (truncated): ['George Washington (disambiguation) | George Washington (1732–1799) was the first President of the United States....', 'Lansdowne portrait | The Lansdowne portrait is an iconic oil-on-canvas portrait of George Washington, the first President of the United States. The portrait was painted by Gilbert Stuart on April 12, ...', 'His Excellency: George Washington | His Excellency: George Washington is a 2004 biography of the first President of the United States, General George Washington. It was written by Joseph Ellis, a prof...']\n"
     ]
    }
   ],
   "source": [
    "# Ask any question you like to this simple RAG program.\n",
    "my_question = \"Who is the first president of the USA?\"\n",
    "\n",
    "# Get the prediction. This contains `pred.context` and `pred.answer`.\n",
    "pred = compiled_rag(my_question)\n",
    "\n",
    "# Print the contexts and the answer.\n",
    "print(f\"Question: {my_question}\")\n",
    "print(f\"Predicted Answer: {pred.answer}\")\n",
    "print(f\"Retrieved Contexts (truncated): {[c[:200] + '...' for c in pred.context]}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Question: Who is the first female prime minister of India?\n",
      "Predicted Answer: Indira Gandhi\n",
      "Retrieved Contexts (truncated): ['Indira Gandhi | Indira Priyadarshini Gandhi (; Nehru; 19 November 1917 – 31 October 1984) was an Indian politician and central figure of the Indian National Congress party. She was the first and to da...', 'First Indira Gandhi ministry | Indira Gandhi was sworn in as Prime Minister of India for the first time on 24 January 1966. In her ministry, the ministers were as follows....', 'Manorama Madhwaraj | Manorama Madhwaraj was the first woman cabinet minister of India elected as an MLA in the 5th, 8th and 9th Karnataka Legislative Assembly. On all of these occasions was elected fr...']\n"
     ]
    }
   ],
   "source": [
    "# Ask any question you like to this simple RAG program.\n",
    "my_question = \"Who is the first female prime minister of India?\"\n",
    "\n",
    "# Get the prediction. This contains `pred.context` and `pred.answer`.\n",
    "pred = compiled_rag(my_question)\n",
    "\n",
    "# Print the contexts and the answer.\n",
    "print(f\"Question: {my_question}\")\n",
    "print(f\"Predicted Answer: {pred.answer}\")\n",
    "print(f\"Retrieved Contexts (truncated): {[c[:200] + '...' for c in pred.context]}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Inspect Learned Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generate_answer\n",
      "Example({'augmented': True, 'context': ['Tae Kwon Do Times | Tae Kwon Do Times is a magazine devoted to the martial art of taekwondo, and is published in the United States of America. While the title suggests that it focuses on taekwondo exclusively, the magazine also covers other Korean martial arts. \"Tae Kwon Do Times\" has published articles by a wide range of authors, including He-Young Kimm, Thomas Kurz, Scott Shaw, and Mark Van Schuyver.', \"Kwon Tae-man | Kwon Tae-man (born 1941) was an early Korean hapkido practitioner and a pioneer of the art, first in Korea and then in the United States. He formed one of the earliest dojang's for hapkido in the United States in Torrance, California, and has been featured in many magazine articles promoting the art.\", 'Hee Il Cho | Cho Hee Il (born October 13, 1940) is a prominent Korean-American master of taekwondo, holding the rank of 9th \"dan\" in the martial art. He has written 11 martial art books, produced 70 martial art training videos, and has appeared on more than 70 martial arts magazine covers. Cho won several national and international competitions as a taekwondo competitor, and has appeared in several films, including \"Fight to Win\", \"Best of the Best\", \"Bloodsport II\", and \"Bloodsport III\". He founded the Action International Martial Arts Association (AIMAA) in 1980, and is its President. Cho is a member of both \"Black Belt\" magazine\\'s Hall of Fame and \"Tae Kwon Do Times\" magazine\\'s Hall of Fame.'], 'question': 'Which magazine has published articles by Scott Shaw, Tae Kwon Do Times or Southwest Art?', 'rationale': 'determine which magazine has published articles by Scott Shaw. First, we know that Scott Shaw is an author who has written for a wide range of publications. Next, we know that Tae Kwon Do Times is a magazine that covers Korean martial arts and has published articles by Scott Shaw. On the other hand, Southwest Art is a magazine that focuses on art in the American Southwest and does not mention Scott Shaw as a contributor. Therefore, the answer is Tae Kwon Do Times.', 'answer': 'Tae Kwon Do Times'}) (input_keys=None)\n"
     ]
    }
   ],
   "source": [
    "for name, parameter in compiled_rag.named_predictors():\n",
    "    print(name)\n",
    "    print(parameter.demos[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluate Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style type=\"text/css\">\n",
       "#T_fb7ca th {\n",
       "  text-align: left;\n",
       "}\n",
       "#T_fb7ca td {\n",
       "  text-align: left;\n",
       "}\n",
       "#T_fb7ca_row0_col0, #T_fb7ca_row0_col1, #T_fb7ca_row0_col2, #T_fb7ca_row0_col3, #T_fb7ca_row0_col4, #T_fb7ca_row0_col5, #T_fb7ca_row1_col0, #T_fb7ca_row1_col1, #T_fb7ca_row1_col2, #T_fb7ca_row1_col3, #T_fb7ca_row1_col4, #T_fb7ca_row1_col5, #T_fb7ca_row2_col0, #T_fb7ca_row2_col1, #T_fb7ca_row2_col2, #T_fb7ca_row2_col3, #T_fb7ca_row2_col4, #T_fb7ca_row2_col5, #T_fb7ca_row3_col0, #T_fb7ca_row3_col1, #T_fb7ca_row3_col2, #T_fb7ca_row3_col3, #T_fb7ca_row3_col4, #T_fb7ca_row3_col5, #T_fb7ca_row4_col0, #T_fb7ca_row4_col1, #T_fb7ca_row4_col2, #T_fb7ca_row4_col3, #T_fb7ca_row4_col4, #T_fb7ca_row4_col5 {\n",
       "  text-align: left;\n",
       "  white-space: pre-wrap;\n",
       "  word-wrap: break-word;\n",
       "  max-width: 400px;\n",
       "}\n",
       "</style>\n",
       "<table id=\"T_fb7ca\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th id=\"T_fb7ca_level0_col0\" class=\"col_heading level0 col0\" >question</th>\n",
       "      <th id=\"T_fb7ca_level0_col1\" class=\"col_heading level0 col1\" >example_answer</th>\n",
       "      <th id=\"T_fb7ca_level0_col2\" class=\"col_heading level0 col2\" >gold_titles</th>\n",
       "      <th id=\"T_fb7ca_level0_col3\" class=\"col_heading level0 col3\" >context</th>\n",
       "      <th id=\"T_fb7ca_level0_col4\" class=\"col_heading level0 col4\" >pred_answer</th>\n",
       "      <th id=\"T_fb7ca_level0_col5\" class=\"col_heading level0 col5\" >answer_exact_match</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_fb7ca_level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
       "      <td id=\"T_fb7ca_row0_col0\" class=\"data row0 col0\" >Give a Girl a Break featured Marge Champion as what kind of talent?</td>\n",
       "      <td id=\"T_fb7ca_row0_col1\" class=\"data row0 col1\" >dancer</td>\n",
       "      <td id=\"T_fb7ca_row0_col2\" class=\"data row0 col2\" >{'Marge Champion', 'Give a Girl a Break'}</td>\n",
       "      <td id=\"T_fb7ca_row0_col3\" class=\"data row0 col3\" >['Give a Girl a Break | Give a Girl a Break is a 1953 musical comedy film directed by Stanley Donen, starring Debbie Reynolds and...</td>\n",
       "      <td id=\"T_fb7ca_row0_col4\" class=\"data row0 col4\" >Dancer</td>\n",
       "      <td id=\"T_fb7ca_row0_col5\" class=\"data row0 col5\" >✔️ [True]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_fb7ca_level0_row1\" class=\"row_heading level0 row1\" >1</th>\n",
       "      <td id=\"T_fb7ca_row1_col0\" class=\"data row1 col0\" >In what year was the film that Steve Hoban is best known for released?</td>\n",
       "      <td id=\"T_fb7ca_row1_col1\" class=\"data row1 col1\" >2009</td>\n",
       "      <td id=\"T_fb7ca_row1_col2\" class=\"data row1 col2\" >{'Splice (film)', 'Steve Hoban'}</td>\n",
       "      <td id=\"T_fb7ca_row1_col3\" class=\"data row1 col3\" >['Steve Hoban | Steven \"Steve\" Hoban (born 1964) is a Canadian film producer. He has been nominated for three Genie Awards and won another. He...</td>\n",
       "      <td id=\"T_fb7ca_row1_col4\" class=\"data row1 col4\" >2009</td>\n",
       "      <td id=\"T_fb7ca_row1_col5\" class=\"data row1 col5\" >✔️ [True]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_fb7ca_level0_row2\" class=\"row_heading level0 row2\" >2</th>\n",
       "      <td id=\"T_fb7ca_row2_col0\" class=\"data row2 col0\" >What football player left the Superliga side OB when he won Goalkeeper of the Year award in Denmark and Norway in 2010?</td>\n",
       "      <td id=\"T_fb7ca_row2_col1\" class=\"data row2 col1\" >Anders Lindegaard</td>\n",
       "      <td id=\"T_fb7ca_row2_col2\" class=\"data row2 col2\" >{'Sten Grytebust', 'Anders Lindegaard'}</td>\n",
       "      <td id=\"T_fb7ca_row2_col3\" class=\"data row2 col3\" >['Anders Lindegaard | Anders Rozenkrantz Lindegaard (] ; born 13 April 1984) is a Danish footballer who plays for English club Burnley as a goalkeeper....</td>\n",
       "      <td id=\"T_fb7ca_row2_col4\" class=\"data row2 col4\" >Anders Lindegaard</td>\n",
       "      <td id=\"T_fb7ca_row2_col5\" class=\"data row2 col5\" >✔️ [True]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_fb7ca_level0_row3\" class=\"row_heading level0 row3\" >3</th>\n",
       "      <td id=\"T_fb7ca_row3_col0\" class=\"data row3 col0\" >Álvaro Raposo De Oliveira was born in a city located in the valleys of which rivers ?</td>\n",
       "      <td id=\"T_fb7ca_row3_col1\" class=\"data row3 col1\" >the Chillón, Rímac and Lurín rivers</td>\n",
       "      <td id=\"T_fb7ca_row3_col2\" class=\"data row3 col2\" >{'Lima', 'Álvaro Raposo de Oliveira'}</td>\n",
       "      <td id=\"T_fb7ca_row3_col3\" class=\"data row3 col3\" >['Álvaro Raposo de Oliveira | Álvaro Raposo De Oliveira (born September 5, 1990, in Lima) is a Peru professional tennis player.', 'Miguel Sutil | Miguel...</td>\n",
       "      <td id=\"T_fb7ca_row3_col4\" class=\"data row3 col4\" >Amazon River</td>\n",
       "      <td id=\"T_fb7ca_row3_col5\" class=\"data row3 col5\" >False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_fb7ca_level0_row4\" class=\"row_heading level0 row4\" >4</th>\n",
       "      <td id=\"T_fb7ca_row4_col0\" class=\"data row4 col0\" >What are the accounting companies that make up the \"Big Four\" excluding Ernst & Young?</td>\n",
       "      <td id=\"T_fb7ca_row4_col1\" class=\"data row4 col1\" >PricewaterhouseCoopers, Deloitte Touche Tohmatsu</td>\n",
       "      <td id=\"T_fb7ca_row4_col2\" class=\"data row4 col2\" >{'Arthur Andersen', 'Ernst &amp; Young'}</td>\n",
       "      <td id=\"T_fb7ca_row4_col3\" class=\"data row4 col3\" >['Ernst &amp; Young | EY (formerly Ernst & Young) is a multinational professional services firm headquartered in London, England. EY is one of the largest...</td>\n",
       "      <td id=\"T_fb7ca_row4_col4\" class=\"data row4 col4\" >Deloitte and PricewaterhouseCoopers</td>\n",
       "      <td id=\"T_fb7ca_row4_col5\" class=\"data row4 col5\" >False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x3239370d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "                <div style='\n",
       "                    text-align: center;\n",
       "                    font-size: 16px;\n",
       "                    font-weight: bold;\n",
       "                    color: #555;\n",
       "                    margin: 10px 0;'>\n",
       "                    ... 45 more rows not displayed ...\n",
       "                </div>\n",
       "                "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "40.0"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from dspy.evaluate.evaluate import Evaluate\n",
    "\n",
    "# Set up the `evaluate_on_hotpotqa` function. We'll use this many times below.\n",
    "evaluate_on_hotpotqa = Evaluate(devset=devset, num_threads=1, display_progress=False, display_table=5)\n",
    "\n",
    "# Evaluate the `compiled_rag` program with the `answer_exact_match` metric.\n",
    "metric = dspy.evaluate.answer_exact_match\n",
    "evaluate_on_hotpotqa(compiled_rag, metric=metric)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style type=\"text/css\">\n",
       "#T_16428 th {\n",
       "  text-align: left;\n",
       "}\n",
       "#T_16428 td {\n",
       "  text-align: left;\n",
       "}\n",
       "#T_16428_row0_col0, #T_16428_row0_col1, #T_16428_row0_col2, #T_16428_row0_col3, #T_16428_row0_col4, #T_16428_row0_col5, #T_16428_row1_col0, #T_16428_row1_col1, #T_16428_row1_col2, #T_16428_row1_col3, #T_16428_row1_col4, #T_16428_row1_col5, #T_16428_row2_col0, #T_16428_row2_col1, #T_16428_row2_col2, #T_16428_row2_col3, #T_16428_row2_col4, #T_16428_row2_col5, #T_16428_row3_col0, #T_16428_row3_col1, #T_16428_row3_col2, #T_16428_row3_col3, #T_16428_row3_col4, #T_16428_row3_col5, #T_16428_row4_col0, #T_16428_row4_col1, #T_16428_row4_col2, #T_16428_row4_col3, #T_16428_row4_col4, #T_16428_row4_col5 {\n",
       "  text-align: left;\n",
       "  white-space: pre-wrap;\n",
       "  word-wrap: break-word;\n",
       "  max-width: 400px;\n",
       "}\n",
       "</style>\n",
       "<table id=\"T_16428\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th id=\"T_16428_level0_col0\" class=\"col_heading level0 col0\" >question</th>\n",
       "      <th id=\"T_16428_level0_col1\" class=\"col_heading level0 col1\" >example_answer</th>\n",
       "      <th id=\"T_16428_level0_col2\" class=\"col_heading level0 col2\" >gold_titles</th>\n",
       "      <th id=\"T_16428_level0_col3\" class=\"col_heading level0 col3\" >context</th>\n",
       "      <th id=\"T_16428_level0_col4\" class=\"col_heading level0 col4\" >pred_answer</th>\n",
       "      <th id=\"T_16428_level0_col5\" class=\"col_heading level0 col5\" >gold_passages_retrieved</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_16428_level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
       "      <td id=\"T_16428_row0_col0\" class=\"data row0 col0\" >Give a Girl a Break featured Marge Champion as what kind of talent?</td>\n",
       "      <td id=\"T_16428_row0_col1\" class=\"data row0 col1\" >dancer</td>\n",
       "      <td id=\"T_16428_row0_col2\" class=\"data row0 col2\" >{'Marge Champion', 'Give a Girl a Break'}</td>\n",
       "      <td id=\"T_16428_row0_col3\" class=\"data row0 col3\" >['Give a Girl a Break | Give a Girl a Break is a 1953 musical comedy film directed by Stanley Donen, starring Debbie Reynolds and...</td>\n",
       "      <td id=\"T_16428_row0_col4\" class=\"data row0 col4\" >Dancer</td>\n",
       "      <td id=\"T_16428_row0_col5\" class=\"data row0 col5\" >✔️ [True]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_16428_level0_row1\" class=\"row_heading level0 row1\" >1</th>\n",
       "      <td id=\"T_16428_row1_col0\" class=\"data row1 col0\" >In what year was the film that Steve Hoban is best known for released?</td>\n",
       "      <td id=\"T_16428_row1_col1\" class=\"data row1 col1\" >2009</td>\n",
       "      <td id=\"T_16428_row1_col2\" class=\"data row1 col2\" >{'Splice (film)', 'Steve Hoban'}</td>\n",
       "      <td id=\"T_16428_row1_col3\" class=\"data row1 col3\" >['Steve Hoban | Steven \"Steve\" Hoban (born 1964) is a Canadian film producer. He has been nominated for three Genie Awards and won another. He...</td>\n",
       "      <td id=\"T_16428_row1_col4\" class=\"data row1 col4\" >2009</td>\n",
       "      <td id=\"T_16428_row1_col5\" class=\"data row1 col5\" >False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_16428_level0_row2\" class=\"row_heading level0 row2\" >2</th>\n",
       "      <td id=\"T_16428_row2_col0\" class=\"data row2 col0\" >What football player left the Superliga side OB when he won Goalkeeper of the Year award in Denmark and Norway in 2010?</td>\n",
       "      <td id=\"T_16428_row2_col1\" class=\"data row2 col1\" >Anders Lindegaard</td>\n",
       "      <td id=\"T_16428_row2_col2\" class=\"data row2 col2\" >{'Sten Grytebust', 'Anders Lindegaard'}</td>\n",
       "      <td id=\"T_16428_row2_col3\" class=\"data row2 col3\" >['Anders Lindegaard | Anders Rozenkrantz Lindegaard (] ; born 13 April 1984) is a Danish footballer who plays for English club Burnley as a goalkeeper....</td>\n",
       "      <td id=\"T_16428_row2_col4\" class=\"data row2 col4\" >Anders Lindegaard</td>\n",
       "      <td id=\"T_16428_row2_col5\" class=\"data row2 col5\" >False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_16428_level0_row3\" class=\"row_heading level0 row3\" >3</th>\n",
       "      <td id=\"T_16428_row3_col0\" class=\"data row3 col0\" >Álvaro Raposo De Oliveira was born in a city located in the valleys of which rivers ?</td>\n",
       "      <td id=\"T_16428_row3_col1\" class=\"data row3 col1\" >the Chillón, Rímac and Lurín rivers</td>\n",
       "      <td id=\"T_16428_row3_col2\" class=\"data row3 col2\" >{'Lima', 'Álvaro Raposo de Oliveira'}</td>\n",
       "      <td id=\"T_16428_row3_col3\" class=\"data row3 col3\" >['Álvaro Raposo de Oliveira | Álvaro Raposo De Oliveira (born September 5, 1990, in Lima) is a Peru professional tennis player.', 'Miguel Sutil | Miguel...</td>\n",
       "      <td id=\"T_16428_row3_col4\" class=\"data row3 col4\" >Amazon River</td>\n",
       "      <td id=\"T_16428_row3_col5\" class=\"data row3 col5\" >False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_16428_level0_row4\" class=\"row_heading level0 row4\" >4</th>\n",
       "      <td id=\"T_16428_row4_col0\" class=\"data row4 col0\" >What are the accounting companies that make up the \"Big Four\" excluding Ernst & Young?</td>\n",
       "      <td id=\"T_16428_row4_col1\" class=\"data row4 col1\" >PricewaterhouseCoopers, Deloitte Touche Tohmatsu</td>\n",
       "      <td id=\"T_16428_row4_col2\" class=\"data row4 col2\" >{'Arthur Andersen', 'Ernst &amp; Young'}</td>\n",
       "      <td id=\"T_16428_row4_col3\" class=\"data row4 col3\" >['Ernst &amp; Young | EY (formerly Ernst & Young) is a multinational professional services firm headquartered in London, England. EY is one of the largest...</td>\n",
       "      <td id=\"T_16428_row4_col4\" class=\"data row4 col4\" >Deloitte and PricewaterhouseCoopers</td>\n",
       "      <td id=\"T_16428_row4_col5\" class=\"data row4 col5\" >✔️ [True]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x167255270>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "                <div style='\n",
       "                    text-align: center;\n",
       "                    font-size: 16px;\n",
       "                    font-weight: bold;\n",
       "                    color: #555;\n",
       "                    margin: 10px 0;'>\n",
       "                    ... 45 more rows not displayed ...\n",
       "                </div>\n",
       "                "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def gold_passages_retrieved(example, pred, trace=None):\n",
    "    gold_titles = set(map(dspy.evaluate.normalize_text, example['gold_titles']))\n",
    "    found_titles = set(map(dspy.evaluate.normalize_text, [c.split(' | ')[0] for c in pred.context]))\n",
    "\n",
    "    return gold_titles.issubset(found_titles)\n",
    "\n",
    "compiled_rag_retrieval_score = evaluate_on_hotpotqa(compiled_rag, metric=gold_passages_retrieved)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "38.0"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "compiled_rag_retrieval_score"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "env-dspy",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}