diff --git a/package.json b/package.json index f3dd84d..cbadfc4 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,7 @@ "dependencies": { "@mendable/firecrawl-js": "^1.8.1", "@supabase/supabase-js": "^2.46.1", + "axios": "^1.7.9", "dotenv": "^14.3.2", "express": "^4.18.2", "express-async-handler": "^1.2.0", @@ -31,6 +32,8 @@ "playht": "^0.13.0", "resend": "^4.0.1-alpha.0", "together-ai": "^0.9.0", - "ts-node-dev": "^2.0.0" + "ts-node-dev": "^2.0.0", + "zod": "^3.24.1", + "zod-to-json-schema": "^3.24.1" } } diff --git a/src/controllers/cron.ts b/src/controllers/cron.ts index bd2eb4a..af2b15f 100644 --- a/src/controllers/cron.ts +++ b/src/controllers/cron.ts @@ -2,6 +2,7 @@ import { scrapeSources } from '../services/scrapeSources'; import { getCronSources } from '../services/getCronSources'; import { generateDraft } from '../services/generateDraft'; import { sendDraft } from '../services/sendDraft'; + export const handleCron = async (): Promise => { try { const cronSources = await getCronSources(); @@ -13,4 +14,4 @@ export const handleCron = async (): Promise => { } catch (error) { console.error(error); } -} \ No newline at end of file +} diff --git a/src/index.ts b/src/index.ts index 2a12fbc..b2a2971 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,8 +1,6 @@ +import 'dotenv/config' import { handleCron } from "./controllers/cron" import cron from 'node-cron'; -import dotenv from 'dotenv'; - -dotenv.config(); async function main() { console.log(`Starting process to generate draft...`); @@ -15,4 +13,4 @@ main(); //cron.schedule(`0 17 * * *`, async () => { // console.log(`Starting process to generate draft...`); // await handleCron(); -//}); \ No newline at end of file +//}); diff --git a/src/services/getCronSources.ts b/src/services/getCronSources.ts index 227dfbb..0b8783f 100644 --- a/src/services/getCronSources.ts +++ b/src/services/getCronSources.ts @@ -1,10 +1,4 @@ -import dotenv from 'dotenv'; - -dotenv.config(); - - - export async function getCronSources() { try { console.log("Fetching sources..."); @@ -31,12 +25,11 @@ export async function getCronSources() { { identifier: "https://x.com/nickscamara_"}, { identifier: "https://x.com/ericciarla"}, { identifier: "https://www.firecrawl.dev/blog"}, - + ]; return sources.map(source => source.identifier); } catch (error) { console.error(error); } -} - \ No newline at end of file +} diff --git a/src/services/scrapeSources.ts b/src/services/scrapeSources.ts index 2c377ad..797f114 100644 --- a/src/services/scrapeSources.ts +++ b/src/services/scrapeSources.ts @@ -1,11 +1,8 @@ import FirecrawlApp from '@mendable/firecrawl-js'; -import dotenv from 'dotenv'; import Together from 'together-ai'; import { z } from 'zod'; import { zodToJsonSchema } from 'zod-to-json-schema'; -dotenv.config(); - const app = new FirecrawlApp({ apiKey: process.env.FIRECRAWL_API_KEY }); // 1. Define the schema for your expected JSON @@ -114,9 +111,9 @@ export async function scrapeSources(sources: string[]) { { role: "system", content: `Today is ${currentDate}. Return only today's AI or LLM related story or post headlines and links in JSON format from the scraped content. They must be posted today. The format should be {"stories": [{"headline": "headline1", "link": "link1", "date_posted": "date1"}, ...]}. -If there are no AI or LLM stories from today, return {"stories": []}. The source link is ${source}. -If the story or post link is not absolute, prepend ${source} to make it absolute. -Return only pure JSON in the specified format (no extra text, no markdown, no \`\`\`). +If there are no AI or LLM stories from today, return {"stories": []}. The source link is ${source}. +If the story or post link is not absolute, prepend ${source} to make it absolute. +Return only pure JSON in the specified format (no extra text, no markdown, no \`\`\`). Scraped Content:\n\n${scrapeResponse.markdown}\n\nJSON:`, }, ], diff --git a/src/services/sendDraft.ts b/src/services/sendDraft.ts index 359dccf..93768bd 100644 --- a/src/services/sendDraft.ts +++ b/src/services/sendDraft.ts @@ -1,6 +1,4 @@ import axios from 'axios'; -import dotenv from 'dotenv'; -dotenv.config(); export async function sendDraft(draft_post: string) { try { @@ -21,4 +19,4 @@ export async function sendDraft(draft_post: string) { console.log('error sending draft to Slack'); console.log(error); } -} \ No newline at end of file +}