Add response statistics to chatbot footer

jasonacox · May 5, 2024 · 4007dcb · 4007dcb
1 parent 19a43ad
commit 4007dcb
Show file tree

Hide file tree

Showing 5 changed files with 6,078 additions and 1 deletion.
diff --git a/.gitignore b/.gitignore
@@ -171,3 +171,6 @@ chatbot/local
 chatbot/oneshot/test.sh
 prompts.json
 chatbot/t
+agents/r
+chatbot/tlocal
+chatbot/tt
diff --git a/RELEASE.md b/RELEASE.md
@@ -2,7 +2,11 @@
 
 ## 0.14.6 - News Links
 
+Chatbot
+
 * Expand `/news/` RAG command to include reference URL links in news article headlines.
+* Add response statistics (number of tokens and tokens per second) to footer.
+* Serve up local copy of socket.io.js library to help with air-gap installations.
 
 ## 0.14.5 - Ollama Support
 

diff --git a/chatbot/server.py b/chatbot/server.py
@@ -72,6 +72,7 @@
 from fastapi import FastAPI, Request
 from fastapi.responses import HTMLResponse
 from fastapi.templating import Jinja2Templates
+from fastapi.responses import FileResponse
 from pypdf import PdfReader
 import aiohttp
 
@@ -534,6 +535,11 @@ async def extract_text_from_html(response):
 async def index(request: Request):
     return templates.TemplateResponse("index.html", {"request": request})
 
+# Serve static socket.io.js
+@app.get("/socket.io.js")
+def serve_socket_io_js():
+    return FileResponse("templates/socket.io.js", media_type="application/javascript")
+
 # Display settings and stats
 @app.get("/stats")
 async def home(format: str = None):
@@ -686,16 +692,23 @@ async def send_update(session_id):
                         # Ask LLM for answers
                         response= await ask(client[session_id]["prompt"],session_id)
                         completion_text = ''
+                        tokens = 0
                         # iterate through the stream of events and print it
+                        stime = time.time()
                         for event in response:
                             event_text = event.choices[0].delta.content
+                            tokens += 1
                             if event_text:
                                 chunk = event_text
                                 completion_text += chunk
                                 if DEBUG:
                                     print(string_to_hex(chunk), end="")
                                     print(f" = [{chunk}]")
                                 await sio.emit('update', {'update': chunk, 'voice': 'ai'},room=session_id)
+                        # Update footer with stats
+                        await sio.emit('update', {'update': 
+                                                  f"TinyLLM Chatbot {VERSION} - {mymodel} - Tokens: {tokens} - TPS: {tokens/(time.time()-stime):.1f}", 
+                                                  'voice': 'footer'},room=session_id)
                         # Check for link injection
                         if client[session_id]["links"]:
                             await sio.emit('update', {'update': json.dumps(client[session_id]["links"]), 'voice': 'links'},room=session_id)

diff --git a/chatbot/templates/index.html b/chatbot/templates/index.html
@@ -8,6 +8,12 @@
             font-family: Arial, sans-serif;
             overflow: hidden; /* Hide scrollbars */
         }
+        /* Style for links */
+        a {
+            color: #6d6d6d;
+            font-size: 12px;
+            text-decoration: none;
+        }
         #container {
             display: flex;
             flex-direction: column;
@@ -380,7 +386,7 @@ <h2 class="popup-title">System Settings</h2>
         }
     </script>
     <!-- END: pop-up-dialogue -->
-    <script src="https://cdnjs.cloudflare.com/ajax/libs/socket.io/4.0.1/socket.io.js"></script>
+    <script src="/socket.io.js"></script>
 </head>
 <body>
     <div id="container">
@@ -554,6 +560,11 @@ <h2 class="popup-title">System Settings</h2>
                 getVersion();
             }
 
+            if (data.voice === "footer") {
+                // Update the footer with the update message
+                document.getElementById('footer').innerHTML = data.update;
+            }
+
             // Ensure we always show the latest
             updateContainer.scrollTop = updateContainer.scrollHeight;
         });