Add image and PDF support to chat with context file integration

Features:
- Implemented pasting and drag-and-drop for images and PDFs in ChatView
- Added PDF extraction functionality using Marker API or local instance
- Integrated extracted content into chat context for AI interaction

Enhancements:
- Improved file handling and management in ContextFileManager
- Added support for Markdown files in context file manager
- Implemented file type validation and user notifications for unsupported formats

UI/UX:
- Added icons for PDF and audio files in chat file manager
- Enhanced loading overlay with progress indication

Performance:
- Optimized PDF processing and content extraction
- Reduced chunk size for transcription to handle content size limit errors

Documentation:
- Updated API documentation to reflect new file handling capabilities
- Added instructions for setting up Marker API key or local instance

Important Notes:
- This is an initial release of PDF and image support, subject to further refinement
- Feature currently requires Marker API key or local Marker instance for full functionality
- Encouraged user feedback for bug reports and improvement suggestions

manifest.json

@@ -6,6 +6,6 @@
   "authorUrl": "systemsculpt.com",
   "fundingUrl": "https://www.patreon.com/SystemSculpt",
   "minAppVersion": "1.5.0",
-  "version": "0.5.3",
+  "version": "0.6.0",
   "isDesktopOnly": true
 }

package.json

@@ -29,6 +29,7 @@
     "lodash": "^4.17.21",
     "marked": "^12.0.2",
     "natural": "^7.1.0",
-    "openai": "^4.29.0"
+    "openai": "^4.29.0",
+    "pdfjs": "^2.5.3"
   }
 }

src/api/UnifiedAIService.ts

@@ -166,15 +166,36 @@ export class UnifiedAIService implements AIServiceInterface {
 
   async createStreamingConversationWithCallback(
     systemPrompt: string,
-    messages: { role: string; content: string }[],
+    messages: { role: string; content: string | { type: string; text?: string; image_url?: { url: string } }[] }[],
     modelId: string,
     maxOutputTokens: number,
     callback: (chunk: string) => void,
     abortSignal?: AbortSignal
   ): Promise<void> {
+    const formattedMessages = [
+      { role: 'system', content: systemPrompt },
+      ...messages.map(msg => {
+        if (typeof msg.content === 'string') {
+          return msg;
+        } else {
+          return {
+            role: msg.role,
+            content: msg.content.map(item => {
+              if (item.type === 'text') {
+                return { type: 'text', text: item.text };
+              } else if (item.type === 'image_url') {
+                return { type: 'image_url', image_url: item.image_url };
+              }
+              return item;
+            })
+          };
+        }
+      })
+    ];
+
     const requestData = JSON.stringify({
       model: modelId,
-      messages: [{ role: 'system', content: systemPrompt }, ...messages],
+      messages: formattedMessages,
       stream: true,
       max_tokens: maxOutputTokens,
       temperature: this.settings.temperature,
@@ -199,6 +220,8 @@ export class UnifiedAIService implements AIServiceInterface {
       headers['X-Title'] = 'SystemSculpt AI for Obsidian';
     }
 
+    console.log('Request payload:', requestData);
+
     const req = await fetch(`${this.endpoint}/chat/completions`, {
       method: 'POST',
       headers: headers,

src/css/chat-view.css

@@ -376,6 +376,46 @@
   padding: 3px;
   border-bottom: 1px solid var(--background-modifier-border);
   white-space: nowrap;
+  position: relative;
+}
+
+.context-file-preview {
+  width: 40px;
+  height: 40px;
+  object-fit: cover;
+  margin-right: 10px;
+  border-radius: 4px;
+}
+
+.audio-icon,
+.pdf-icon {
+  width: 40px;
+  height: 40px;
+  margin-right: 10px;
+  color: var(--text-normal);
+}
+
+.context-file-remove {
+  position: absolute;
+  top: 0;
+  left: 0;
+  background: rgba(0, 0, 0, 0.5);
+  color: white;
+  border: none;
+  border-radius: 50%;
+  width: 20px;
+  height: 20px;
+  font-size: 12px;
+  display: flex;
+  align-items: center;
+  justify-content: center;
+  cursor: pointer;
+  opacity: 0;
+  transition: opacity 0.3s ease;
+}
+
+.context-file:hover .context-file-remove {
+  opacity: 1;
 }
 
 .context-file:last-child {
@@ -424,49 +464,77 @@
   color: var(--interactive-accent);
 }
 
-.loading-container {
-  display: none;
-  padding: 15px;
-  background-color: var(--background-primary-alt);
-  border-top: 1px solid var(--background-modifier-border);
-  border-bottom: 1px solid var(--background-modifier-border);
-  text-align: center;
-  flex-direction: column;
+.loading-overlay {
+  position: absolute;
+  top: 0;
+  left: 0;
+  right: 0;
+  bottom: 0;
+  background-color: rgba(var(--background-primary-rgb), 0.8);
+  display: flex;
   justify-content: center;
   align-items: center;
+  z-index: 1000;
+  opacity: 0;
+  visibility: hidden;
+  transition: opacity 0.3s ease, visibility 0.3s ease;
 }
 
-.loading-container.visible {
-  display: flex;
+.loading-overlay.visible {
+  opacity: 1;
+  visibility: visible;
+}
+
+.loading-container {
+  background-color: var(--background-secondary);
+  border-radius: 8px;
+  padding: 20px;
+  box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1);
+  text-align: center;
+  max-width: 80%;
+  width: 300px;
 }
 
 .loading-spinner {
-  border: 4px solid rgba(255, 255, 255, 0.3);
-  border-top: 4px solid white;
+  border: 4px solid var(--background-modifier-border);
+  border-top: 4px solid var(--interactive-accent);
   border-radius: 50%;
   width: 40px;
   height: 40px;
   animation: spin 1s linear infinite;
-  margin-bottom: 10px;
+  margin: 0 auto 15px;
 }
 
 .loading-text {
   font-size: 16px;
-  color: var(--text-muted);
+  color: var(--text-normal);
+  margin-bottom: 15px;
+  font-weight: bold;
+}
+
+.progress-bar {
+  width: 100%;
+  height: 6px;
+  background-color: var(--background-modifier-border);
+  border-radius: 3px;
+  overflow: hidden;
   margin-bottom: 10px;
 }
 
-@keyframes spin {
-  0% {
-    transform: rotate(0deg);
-  }
-  100% {
-    transform: rotate(360deg);
-  }
+.progress-bar-fill {
+  height: 100%;
+  background-color: var(--interactive-accent);
+  transition: width 0.3s ease;
 }
 
-.loading-container.scaled-down {
-  transform: scale(0.5);
+.progress-text {
+  font-size: 14px;
+  color: var(--text-muted);
+}
+
+@keyframes spin {
+  0% { transform: rotate(0deg); }
+  100% { transform: rotate(360deg); }
 }
 
 .title-edit-modal {

src/main.ts

@@ -1,4 +1,4 @@
-import { Plugin, TFile, Notice, WorkspaceLeaf } from 'obsidian';
+import { Plugin, TFile, Notice, WorkspaceLeaf, Menu } from 'obsidian';
 import {
   SystemSculptSettings,
   DEFAULT_SETTINGS,
@@ -71,6 +71,36 @@ export default class SystemSculptPlugin extends Plugin {
     // Register the context menu item for .mp3 files using the new events module
     registerMp3ContextMenu(this, this.recorderModule);
 
+    // Register the context menu item for PDF extraction
+    this.addCommand({
+      id: 'extract-pdf',
+      name: 'Extract PDF with SystemSculpt',
+      checkCallback: (checking: boolean) => {
+        const file = this.app.workspace.getActiveFile();
+        if (file && file.extension === 'pdf') {
+          if (!checking) {
+            this.chatModule.extractPDF(file);
+          }
+          return true;
+        }
+        return false;
+      },
+    });
+
+    // Register the context menu item for PDF files
+    this.registerEvent(
+      this.app.workspace.on('file-menu', (menu, file) => {
+        if (file instanceof TFile && file.extension === 'pdf') {
+          menu.addItem((item) => {
+            item
+              .setTitle('Extract PDF with SystemSculpt')
+              .setIcon('file-text')
+              .onClick(() => this.chatModule.extractPDF(file));
+          });
+        }
+      })
+    );
+
     // Add commands and event listeners
     this.addCommands();
     this.registerEvents();
@@ -113,16 +143,21 @@ export default class SystemSculptPlugin extends Plugin {
       this.app.workspace.on('file-open', async file => {
         if (
           file &&
-          file.path.startsWith('SystemSculpt/Chats') &&
-          file instanceof TFile
+          file instanceof TFile &&
+          file.parent &&
+          file.parent.path === this.chatModule.settings.chatsPath &&
+          file.extension === 'md'
         ) {
           let chatLeaf = this.app.workspace.getLeavesOfType(VIEW_TYPE_CHAT)[0];
 
           if (!chatLeaf) {
-            //@ts-ignore
-            chatLeaf = this.app.workspace.getRightLeaf(false);
-            if (chatLeaf) {
-              await chatLeaf.setViewState({ type: VIEW_TYPE_CHAT });
+            const newLeaf = this.app.workspace.getRightLeaf(false);
+            if (newLeaf) {
+              await newLeaf.setViewState({ type: VIEW_TYPE_CHAT });
+              chatLeaf = newLeaf;
+            } else {
+              // Handle the case where a new leaf couldn't be created
+              return;
             }
           } else {
             this.app.workspace.revealLeaf(chatLeaf);
@@ -135,6 +170,7 @@ export default class SystemSculptPlugin extends Plugin {
       })
     );
 
+
     // Register file change event to update chat view when the chat file changes
     this.registerEvent(
       this.app.vault.on('modify', async file => {
@@ -215,4 +251,4 @@ export default class SystemSculptPlugin extends Plugin {
   onClickEvent(element: HTMLElement, callback: () => void) {
     element.addEventListener('click', callback);
   }
-}
+}

src/modules/chat/ChatFileManager.ts

@@ -134,4 +134,9 @@ export class ChatFileManager {
 
     return contextFiles;
   }
-}
+
+  isDirectlyInChatsDirectory(file: TFile): boolean {
+    const chatsPath = this.chatModule.settings.chatsPath;
+    return file.parent?.path === chatsPath;
+  }
+}