Merge pull request #176 from NetSepio/vaibhavvvvv/dev

summary: reduced response time + reduced prompt size

api/v1/summary/create_summary.go

@@ -20,8 +20,8 @@ func summarizeContent(contents []string) string {
 	}
 
 	prompt := builder.String()
-	if len(prompt) > 128000 {
-		prompt = prompt[:127999]
+	if len(prompt) > 10000 {
+		prompt = prompt[:9900]
 	}
 
 	open_ai_key := envconfig.EnvVars.OPENAI_API_KEY

api/v1/summary/scrape_urls.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"fmt"
 	"strings"
-	"time"
 
 	"github.com/chromedp/cdproto/cdp"
 	"github.com/chromedp/chromedp"
@@ -23,16 +22,13 @@ func extractLinksFromURL(url string) ([]string, error) {
 	defer cancel()
 	ctx, cancel := chromedp.NewContext(parentCtx)
 	defer cancel()
+
 	var links []string
 
 	if err := chromedp.Run(ctx, navigateToWebsite(url)); err != nil {
 		return nil, fmt.Errorf("failed to navigate to website: %v", err)
 	}
 
-	if err := chromedp.Run(ctx, chromedp.Sleep(3*time.Second)); err != nil {
-		return nil, fmt.Errorf("failed to wait: %v", err)
-	}
-
 	if err := chromedp.Run(ctx, extractLinks(&links)); err != nil {
 		return nil, fmt.Errorf("failed to extract links: %v", err)
 	}
@@ -57,7 +53,7 @@ func extractLinks(links *[]string) chromedp.Action {
 				*links = append(*links, href)
 			}
 		}
-
+		// fmt.Println("AL: : ", links)
 		return nil
 	})
 }
@@ -82,10 +78,6 @@ func extractContentFromLink(link string) (string, error) {
 		return "", fmt.Errorf("failed to navigate to link: %v", err)
 	}
 
-	if err := chromedp.Run(ctx, chromedp.Sleep(3*time.Second)); err != nil {
-		return "", fmt.Errorf("failed to wait: %v", err)
-	}
-
 	if err := chromedp.Run(ctx, extractText(&content)); err != nil {
 		return "", fmt.Errorf("failed to extract text content: %v", err)
 	}
@@ -111,6 +103,7 @@ func containsTermsKeywords(link string) bool {
 	link = strings.ToLower(link)
 	for _, keyword := range termsKeywords {
 		if strings.Contains(link, keyword) {
+			// fmt.Println("imp:  ", link)
 			return true
 		}
 	}
@@ -122,6 +115,7 @@ func containsPrivacyKeywords(link string) bool {
 	link = strings.ToLower(link)
 	for _, keyword := range privacyKeywords {
 		if strings.Contains(link, keyword) {
+			// fmt.Println(link)
 			return true
 		}
 	}
@@ -136,14 +130,18 @@ func summarizeLinksContent(links []string) (termsSummary, privacySummary string)
 			content, err := extractContentFromLink(link)
 			if err == nil {
 				termsContents = append(termsContents, content)
-				fmt.Println(termsContents)
+				// fmt.Println(termsContents)
+			} else {
+				fmt.Printf("Error extracting content from terms link %s: %v\n", link, err)
 			}
 		}
 		if containsPrivacyKeywords(link) {
 			content, err := extractContentFromLink(link)
 			if err == nil {
 				privacyContents = append(privacyContents, content)
-				fmt.Println(privacyContents)
+				// fmt.Println(privacyContents)
+			} else {
+				fmt.Printf("Error extracting content from privacy link %s: %v\n", link, err)
 			}
 		}
 	}