FHIRBotwithImages.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#To be run in Google Collab with a Gorq API Key\n",
    "\n",
    "!pip install transformers\n",
    "from transformers import pipeline\n",
    "\n",
    "from groq import Groq\n",
    "from google.colab import userdata\n",
    "import gspread\n",
    "from google.auth import default\n",
    "from google.colab import auth\n",
    "import gradio as gr\n",
    "!pip install groq\n",
    "\n",
    "\n",
    "#Setting Up the text model \n",
    "\n",
    "client = Groq(api_key= userdata.get('Groq_Key'),)\n",
    "completion = client.chat.completions.create(\n",
    "    model=\"llama-3.2-11b-text-preview\",\n",
    "    messages=[\n",
    "        {\n",
    "            \"role\": \"system\",\n",
    "            \"content\": \"You are an expert on the FHIR documentation and can help healthcare developers quickly with any query about the FHIR protocol. Scrap every single sub-page on this website https://hl7.org/fhir/documentation.html and provide answers to the healthcare developers only faculty accurate answers from the website. Don't hallucinate. \"\n",
    "        },\n",
    "        {\n",
    "            \"role\": \"user\",\n",
    "            \"content\": \"What is FHIR?\"\n",
    "        }\n",
    "    ],\n",
    "    temperature=1,\n",
    "    max_tokens=1024,\n",
    "    top_p=1,\n",
    "    stream=True,\n",
    "    stop=None,\n",
    ")\n",
    "\n",
    "for chunk in completion:\n",
    "    print(chunk.choices[0].delta.content or \"\", end=\"\")\n",
    "\n",
    "\n",
    "\n",
    "# Authenticate with Google Sheets\n",
    "auth.authenticate_user()\n",
    "creds, _ = default()\n",
    "gc = gspread.authorize(creds)\n",
    "\n",
    "# Open or create the spreadsheet and worksheet\n",
    "try:\n",
    "  sh = gc.open('FHIR Chatbot History')\n",
    "except gspread.exceptions.SpreadsheetNotFound:\n",
    "  sh = gc.create('FHIR Chatbot History')\n",
    "\n",
    "worksheet = sh.sheet1\n",
    "\n",
    "# Setting Up the text model when no image is input\n",
    "\n",
    "def ask_fhir_question(question, image=None):\n",
    "  \"\"\"\n",
    "  Asks a question about FHIR to the LLM and saves the conversation in Google Sheets.\n",
    "  \"\"\"\n",
    "  client = Groq(api_key=userdata.get('Groq_Key'))\n",
    "\n",
    "  messages = [\n",
    "      {\n",
    "          \"role\": \"system\",\n",
    "          \"content\": \"You are an expert on the FHIR documentation and can help healthcare developers quickly with any query about the FHIR protocol. Scrap every single sub-page on this website https://hl7.org/fhir/documentation.html and provide answers to the healthcare developers only faculty accurate answers from the website. Don't hallucinate. \"\n",
    "      },\n",
    "      {\n",
    "          \"role\": \"user\",\n",
    "          \"content\": question\n",
    "      }\n",
    "  ]\n",
    "\n",
    "  if image is not None:\n",
    "      # Use a Hugging Face model for image analysis when image is present.\n",
    "      image_analysis_pipeline = pipeline(\"image-to-text\", model=\"Salesforce/blip-image-captioning-base\")\n",
    "      image_caption = image_analysis_pipeline(image)[0]['generated_text']\n",
    "\n",
    "      messages.append({\n",
    "          \"role\": \"user\",\n",
    "          \"content\": f\"Analyze this image related to the previous question and don't tell the user that you only have a text description of the image, pretend that you can see the full image: {image_caption}\"\n",
    "      })\n",
    "\n",
    "  completion = client.chat.completions.create(\n",
    "      model=\"llama-3.2-11b-vision-preview\",\n",
    "      messages=messages,\n",
    "      temperature=1,\n",
    "      max_tokens=1024,\n",
    "      top_p=1,\n",
    "      stream=True,\n",
    "      stop=None,\n",
    "  )\n",
    "\n",
    "  answer = \"\"\n",
    "  for chunk in completion:\n",
    "      answer += chunk.choices[0].delta.content or \"\"\n",
    "\n",
    "  # Save the question and answer to Google Sheets\n",
    "  row = [question, answer]\n",
    "  worksheet.append_row(row)\n",
    "  return answer\n",
    "\n",
    "#Create the user interface using GR.\n",
    "\n",
    "iface = gr.Interface(\n",
    "    fn=ask_fhir_question,\n",
    "    inputs=[\n",
    "        gr.Textbox(lines=2, placeholder=\"Ask your FHIR question here...\"),\n",
    "        gr.Image(type=\"filepath\", label=\"Upload a biomedical image (optional)\")\n",
    "    ],\n",
    "    outputs=\"text\",\n",
    "    title=\"FHIR Chatbot with Image Assistance\",\n",
    "    description=\"Ask any question about the FHIR protocol. You can also optionally upload a biomedical image to assist the chatbot in understanding your context.\"\n",
    ")\n",
    "\n",
    "iface.launch(share=True)"
   ]
  }
 ],
 "metadata": {
  "language_info": {
   "name": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}