Merge pull request #4 from jasonacox/fastapi

Port Chatbot to FastAPI and Uvicorn ASGI

.gitignore

@@ -165,12 +165,6 @@ cython_debug/
 # Exclude sandbox
 sandbox/
 .DS_Store
-chatbot/test.sh
-chatbot/test8080.sh
-chatbot/llamafile.sh
 chatbot/prompts.json
 chatbot/.tinyllm
-chatbot/uploadtest.sh
-chatbot/f.sh
-chatbot/main.sh
-chatbot/r
+chatbot/local

README.md

@@ -111,7 +111,8 @@ docker run \
     -e OPENAI_API_BASE="http://localhost:8000/v1" \
     -e LLM_MODEL="tinyllm" \
     -e USE_SYSTEM="false" \
-    -v $PWD/prompts.json:/app/prompts.json \
+    -e SENTENCE_TRANSFORMERS_HOME=/app/.tinyllm \
+    -v $PWD/.tinyllm:/app/.tinyllm \
     --name chatbot \
     --restart unless-stopped \
     jasonacox/chatbot

RELEASE.md

@@ -1,5 +1,12 @@
 # Releases
 
+## 0.12.0 - FastAPI and Uvicorn
+
+* Ported Chatbot to the async FastAPI and Uvicorn ASGI high speed web server implementation (https://github.com/jasonacox/TinyLLM/issues/3).
+* Added /stats page to display configuration settings and current stats (optional `?format=json`)
+* UI updated to help enforce focus on text entry box.
+* Moved `prompts.json` and Sentence Transformer model location to a `./.tinyllm` for Docker support. 
+
 ## 0.11.4 - Stats Page
 
 * Add `/stats` URL to Chatbot for settings and current status information.

chatbot/Dockerfile

@@ -1,5 +1,5 @@
 # Dockerfile for chatbot - OpenAI API ChatBot
-# - Basic Version: NO Vector Search Support
+# - Full Version: WITH Vector Search Support for RAG
 #
 #   Author: Jason A. Cox
 #   23 Sept 2023
@@ -9,23 +9,23 @@
 FROM python:3.10-slim
 
 # Setting build related env vars
-ENV PORT=5000
-ENV OPENAI_API_KEY="DEFAULT_API_KEY"
-ENV OPENAI_API_BASE="http://localhost:8000/v1"
-ENV AGENT_NAME="Jarvis"
-ENV MY_MODEL="models/7B/gguf-model.bin"
-ENV DEBUG="False"
-ENV DEVICE="cpu"
-ENV STMODEL="all-MiniLM-L6-v2"
-ENV QDRANT_HOST=""
-ENV RESULTS=1
-ENV USE_SYSTEM="false"
+ENV PORT 5000
+ENV OPENAI_API_KEY "DEFAULT_API_KEY"
+ENV OPENAI_API_BASE "http://localhost:8000/v1"
+ENV AGENT_NAME "Jarvis"
+ENV MY_MODEL "models/7B/gguf-model.bin"
+ENV DEBUG "False"
+ENV DEVICE "cpu"
+ENV STMODEL "all-MiniLM-L6-v2"
+ENV QDRANT_HOST ""
+ENV RESULTS 1
+ENV USE_SYSTEM "false"
 
 # Set the working directory
 WORKDIR /app
 
 # Install depencencies - Mini Install - No Vector Search
-RUN pip install flask flask-socketio bs4 requests pypdf openai lxml
+RUN pip install fastapi uvicorn python-socketio jinja2 openai bs4 pypdf requests lxml
 
 # Add Suport for Vector Search - CPU Only - Comment out to disable vector search
 # RUN pip install torch qdrant-client sentence-transformers --extra-index-url https://download.pytorch.org/whl/cpu
@@ -34,12 +34,12 @@ RUN pip install flask flask-socketio bs4 requests pypdf openai lxml
 COPY server.py /app/server.py
 COPY templates /app/templates
 
-# Run the server
-CMD ["python3", "server.py"]
-
 # Network
 EXPOSE $PORT
 
+# Run the server
+CMD uvicorn server:app --host 0.0.0.0 --port $PORT
+
 # Example docker run command
 # docker run \
 #     -d \
@@ -54,4 +54,4 @@ EXPOSE $PORT
 #     -v $PWD/prompts.json:/app/prompts.json \
 #     --name chatbot \
 #     --restart unless-stopped \
-#     chatbot:0.11.0
+#     chatbot:0.12.0

chatbot/Dockerfile.basic

@@ -1,5 +1,5 @@
 # Dockerfile for chatbot - OpenAI API ChatBot
-# - Basic Version: NO Vector Search Support
+# - Full Version: WITH Vector Search Support for RAG
 #
 #   Author: Jason A. Cox
 #   23 Sept 2023
@@ -9,23 +9,23 @@
 FROM python:3.10-slim
 
 # Setting build related env vars
-ENV PORT=5000
-ENV OPENAI_API_KEY="DEFAULT_API_KEY"
-ENV OPENAI_API_BASE="http://localhost:8000/v1"
-ENV AGENT_NAME="Jarvis"
-ENV MY_MODEL="models/7B/gguf-model.bin"
-ENV DEBUG="False"
-ENV DEVICE="cpu"
-ENV STMODEL="all-MiniLM-L6-v2"
-ENV QDRANT_HOST=""
-ENV RESULTS=1
-ENV USE_SYSTEM="false"
+ENV PORT 5000
+ENV OPENAI_API_KEY "DEFAULT_API_KEY"
+ENV OPENAI_API_BASE "http://localhost:8000/v1"
+ENV AGENT_NAME "Jarvis"
+ENV MY_MODEL "models/7B/gguf-model.bin"
+ENV DEBUG "False"
+ENV DEVICE "cpu"
+ENV STMODEL "all-MiniLM-L6-v2"
+ENV QDRANT_HOST ""
+ENV RESULTS 1
+ENV USE_SYSTEM "false"
 
 # Set the working directory
 WORKDIR /app
 
 # Install depencencies - Mini Install - No Vector Search
-RUN pip install flask flask-socketio bs4 requests pypdf openai lxml
+RUN pip install fastapi uvicorn python-socketio jinja2 openai bs4 pypdf requests lxml
 
 # Add Suport for Vector Search - CPU Only - Comment out to disable vector search
 # RUN pip install torch qdrant-client sentence-transformers --extra-index-url https://download.pytorch.org/whl/cpu
@@ -34,12 +34,12 @@ RUN pip install flask flask-socketio bs4 requests pypdf openai lxml
 COPY server.py /app/server.py
 COPY templates /app/templates
 
-# Run the server
-CMD ["python3", "server.py"]
-
 # Network
 EXPOSE $PORT
 
+# Run the server
+CMD uvicorn server:app --host 0.0.0.0 --port $PORT
+
 # Example docker run command
 # docker run \
 #     -d \
@@ -51,7 +51,8 @@ EXPOSE $PORT
 #     -e DEVICE="cpu" \
 #     -e RESULTS=1 \
 #     -e USE_SYSTEM="false" \
-#     -v $PWD/prompts.json:/app/prompts.json \
+#     -e SENTENCE_TRANSFORMERS_HOME=/app/.tinyllm \
+#     -v $PWD/.tinyllm:/app/.tinyllm \
 #     --name chatbot \
 #     --restart unless-stopped \
-#     chatbot:0.11.0
+#     chatbot:0.12.0

chatbot/Dockerfile.rag

@@ -9,23 +9,23 @@
 FROM python:3.10-slim
 
 # Setting build related env vars
-ENV PORT=5000
-ENV OPENAI_API_KEY="DEFAULT_API_KEY"
-ENV OPENAI_API_BASE="http://localhost:8000/v1"
-ENV AGENT_NAME="Jarvis"
-ENV MY_MODEL="models/7B/gguf-model.bin"
-ENV DEBUG="False"
-ENV DEVICE="cpu"
-ENV STMODEL="all-MiniLM-L6-v2"
-ENV QDRANT_HOST=""
-ENV RESULTS=1
-ENV USE_SYSTEM="false"
+ENV PORT 5000
+ENV OPENAI_API_KEY "DEFAULT_API_KEY"
+ENV OPENAI_API_BASE "http://localhost:8000/v1"
+ENV AGENT_NAME "Jarvis"
+ENV MY_MODEL "models/7B/gguf-model.bin"
+ENV DEBUG "False"
+ENV DEVICE "cpu"
+ENV STMODEL "all-MiniLM-L6-v2"
+ENV QDRANT_HOST ""
+ENV RESULTS 1
+ENV USE_SYSTEM "false"
 
 # Set the working directory
 WORKDIR /app
 
 # Install depencencies - Mini Install - No Vector Search
-RUN pip install flask flask-socketio bs4 requests pypdf openai lxml
+RUN pip install fastapi uvicorn python-socketio jinja2 openai bs4 pypdf requests lxml
 
 # Add Suport for Vector Search - CPU Only - Comment out to disable vector search
 RUN pip install torch qdrant-client sentence-transformers --extra-index-url https://download.pytorch.org/whl/cpu
@@ -34,12 +34,12 @@ RUN pip install torch qdrant-client sentence-transformers --extra-index-url http
 COPY server.py /app/server.py
 COPY templates /app/templates
 
-# Run the server
-CMD ["python3", "server.py"]
-
 # Network
 EXPOSE $PORT
 
+# Run the server
+CMD uvicorn server:app --host 0.0.0.0 --port $PORT
+
 # Example docker run command
 # docker run \
 #     -d \
@@ -51,7 +51,8 @@ EXPOSE $PORT
 #     -e DEVICE="cpu" \
 #     -e RESULTS=1 \
 #     -e USE_SYSTEM="false" \
-#     -v $PWD/prompts.json:/app/prompts.json \
+#     -e SENTENCE_TRANSFORMERS_HOME=/app/.tinyllm \
+#     -v $PWD/.tinyllm:/app/.tinyllm \
 #     --name chatbot \
 #     --restart unless-stopped \
-#     chatbot:0.11.0
+#     chatbot:0.12.0

chatbot/build.sh

@@ -8,3 +8,5 @@ version=$(echo $string | awk '{print $NF}' | sed 's/v//' | sed 's/"//g')
 # Create Container
 docker build -t chatbot:$version .
 
+# Done
+echo "Built: chatbot:$version"

chatbot/run-openai.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+# 
+# Run the TinyLLM Chatbot using OpenAI GPT-3.5
+#
+# Author: Jason A. Cox
+# 11 Feb 2024
+# https://github.com/jasonacox/TinyLLM
+
+echo "Run TinyLLM Chatbot - OpenAI GPT-3.5"
+echo "------------------------------------"
+
+# Stop and remove the chatbot container
+echo "Removing old chatbot container..."
+docker stop chatbot
+docker rm chatbot
+
+# Start the chatbot container
+echo "Starting new chatbot container..."
+if [ ! -d "./.tinyllm" ]; then
+    echo "Creating .tinyllm directory..."
+    mkdir .tinyllm
+fi
+
+docker run \
+    -d \
+    -p 5000:5000 \
+    -e PORT=5000 \
+    -e OPENAI_API_KEY="YOUR_OPENAI_KEY" \
+    -e OPENAI_API_BASE="https://api.openai.com/v1" \
+    -e LLM_MODEL="gpt-3.5-turbo" \
+    -e USE_SYSTEM="false" \
+    -e MAXCLIENTS=1000 \
+    -e MAXTOKENS=4000 \
+    -e TEMPERATURE=0.0 \
+    -e QDRANT_HOST="localhost" \
+    -e SENTENCE_TRANSFORMERS_HOME=/app/.tinyllm \
+    -v $PWD/.tinyllm:/app/.tinyllm \
+    --name chatbot \
+    --restart unless-stopped \
+    jasonacox/chatbot
+echo "Chatbot container started."
+echo ""
+
+# Show the logs
+echo "Viewing chatbot container logs... ^C to exit"
+echo ""
+docker logs -f chatbot

chatbot/run.sh

@@ -6,34 +6,16 @@
 # 6 Feb 2024
 # https://github.com/jasonacox/TinyLLM
 
-# Check to see if container name exists already
-EXISTING=$(docker ps -a --format '{{.Names}}' | grep chatbot)
-if [ ! -z "$EXISTING" ]; then
-    echo "Existing chatbot container found: $EXISTING"
-    echo ""
-    # Ask if we should stop and remove the existing container
-    read -p "Would you like to stop and remove the existing container? (y/N) " -n 1 -r
-    echo ""
-    if [[ $REPLY =~ ^[Yy]$ ]]; then
-        # Check if container is in running state
-        RUNNING=$(docker inspect --format="{{.State.Running}}" chatbot 2>/dev/null)
-        if [ "$RUNNING" == "true" ]; then
-            # Send restarting alert message to chatbot users
-            curl -s -X POST -H "Content-Type: application/json" -d '{"token": "secret", "message": "Restarting"}' http://localhost:5000/alert > /dev/null 2>&1
-            echo "Stopping chatbot container..."
-            docker stop chatbot
-        fi
-        echo "Removing existing chatbot container..."
-        docker rm chatbot
-    else
-        echo "Exiting..."
-        exit 1
-    fi
-    echo ""
-fi
+echo "Run TinyLLM Chatbot - Local Model"
+echo "---------------------------------"
+
+# Stop and remove the chatbot container
+echo "Removing old chatbot container..."
+docker stop chatbot
+docker rm chatbot
 
 # Start the chatbot container
-echo "Starting chatbot container..."
+echo "Starting new chatbot container..."
 if [ ! -d "./.tinyllm" ]; then
     echo "Creating .tinyllm directory..."
     mkdir .tinyllm
@@ -51,8 +33,8 @@ docker run \
     -e MAXTOKENS=16384 \
     -e TEMPERATURE=0.0 \
     -e QDRANT_HOST="localhost" \
+    -e SENTENCE_TRANSFORMERS_HOME=/app/.tinyllm \
     -v $PWD/.tinyllm:/app/.tinyllm \
-    --network="host" \
     --name chatbot \
     --restart unless-stopped \
     jasonacox/chatbot

chatbot/server-flask.py

@@ -269,6 +269,7 @@ def base_prompt(content=None):
 def ask(prompt, sid=None):
     global client
     stats["ask"] += 1
+
     response = False
     log(f"Context size = {len(context)}")
     while not response:
@@ -396,7 +397,7 @@ def get_news(topic, max=10):
     log(f"Fetching news for {topic} from {url}")
     response = get_top_articles(url, max)
     return response
-    
+
 # Function - Extract text from URL
 def extract_text_from_url(url):
     try: