From 9451e8a9175850985dc2f6280df676aeee54a242 Mon Sep 17 00:00:00 2001 From: openhands Date: Tue, 3 Dec 2024 02:37:57 +0000 Subject: [PATCH 1/7] Parallelize GitHub data cache generation to improve performance --- scripts/src/github-api.ts | 72 +++++++++++++++++++++++++++++++++------ 1 file changed, 62 insertions(+), 10 deletions(-) diff --git a/scripts/src/github-api.ts b/scripts/src/github-api.ts index 1aecd05..29cae69 100644 --- a/scripts/src/github-api.ts +++ b/scripts/src/github-api.ts @@ -13,7 +13,8 @@ async function fetchWithAuth(url: string): Promise { }); if (!response.ok) { - throw new Error(`GitHub API error: ${response.status.toString()} ${response.statusText}`); + const errorBody = await response.text(); + throw new Error(`GitHub API error: ${response.status.toString()} ${response.statusText}\n${errorBody}`); } // Parse Link header for pagination @@ -163,8 +164,26 @@ async function processPRComments(pr: GitHubPR): Promise { return activities; } +const startTime = performance.now(); + +async function main() { + try { + await fetchBotActivities(); + } catch (error) { + process.stderr.write(`${error}\n`); + process.exit(1); + } +} + +main(); + export async function fetchBotActivities(since?: string): Promise { try { + if (!GITHUB_TOKEN || GITHUB_TOKEN === 'placeholder') { + process.stderr.write('Error: GITHUB_TOKEN environment variable is not set or invalid\n'); + throw new Error('Invalid GITHUB_TOKEN'); + } + console.log('Starting bot activities fetch...'); const activities: Activity[] = []; const baseUrl = `https://api.github.com/repos/${REPO_OWNER}/${REPO_NAME}`; @@ -186,31 +205,64 @@ export async function fetchBotActivities(since?: string): Promise { } // Fetch issues and PRs + console.log('Fetching issues and PRs...'); + const fetchStartTime = performance.now(); const items = await fetchAllPages(`${baseUrl}/issues?${params.toString()}`); + console.log(`Fetched ${items.length} items in ${((performance.now() - fetchStartTime) / 1000).toFixed(2)}s`); - for (const item of items) { - if (item.comments > 0) { + console.log('Processing items...'); + const processStartTime = performance.now(); + // Filter items that have comments + const itemsWithComments = items.filter(item => item.comments > 0); + console.log(`Processing ${itemsWithComments.length} items with comments in parallel...`); + + // Process items in parallel + const batchSize = 10; // Process 10 items at a time to avoid rate limiting + const results = []; + + for (let i = 0; i < itemsWithComments.length; i += batchSize) { + const batch = itemsWithComments.slice(i, i + batchSize); + console.log(`Processing batch ${Math.floor(i/batchSize) + 1}/${Math.ceil(itemsWithComments.length/batchSize)}...`); + + const batchResults = await Promise.all( + batch.map(async item => { if (item.pull_request === undefined) { // Process regular issues - const issueActivities = await processIssueComments(item); - activities.push(...issueActivities); + return processIssueComments(item); } else { // Process PRs through the issue comments endpoint to catch all activity - const prActivities = await processPRComments({ + return processPRComments({ number: item.number, html_url: item.html_url, comments_url: item.comments_url, comments: item.comments }); - activities.push(...prActivities); } - } - } + }) + ); + + results.push(...batchResults); + } + + // Flatten results and add to activities + activities.push(...results.flat()); + + console.log(`Processed all items in ${((performance.now() - processStartTime) / 1000).toFixed(2)}s`); // Sort by timestamp in descending order - return activities.sort((a, b) => new Date(b.timestamp).getTime() - new Date(a.timestamp).getTime()); + console.log('Sorting activities...'); + const sortStartTime = performance.now(); + const sortedActivities = activities.sort((a, b) => new Date(b.timestamp).getTime() - new Date(a.timestamp).getTime()); + console.log(`Sorted ${activities.length} activities in ${((performance.now() - sortStartTime) / 1000).toFixed(2)}s`); + + const totalTime = (performance.now() - startTime) / 1000; + console.log(`Total execution time: ${totalTime.toFixed(2)}s`); + + return sortedActivities; } catch (error) { console.error('Error fetching bot activities:', error); + const totalTime = (performance.now() - startTime) / 1000; + console.log(`Total execution time: ${totalTime.toFixed(2)}s (failed)`); throw error; } } \ No newline at end of file From 9424b6058b473894a825d5f41a95104bd3790e43 Mon Sep 17 00:00:00 2001 From: openhands Date: Tue, 3 Dec 2024 02:40:29 +0000 Subject: [PATCH 2/7] Fix type errors in template literals --- scripts/src/github-api.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/scripts/src/github-api.ts b/scripts/src/github-api.ts index 29cae69..af5d559 100644 --- a/scripts/src/github-api.ts +++ b/scripts/src/github-api.ts @@ -181,7 +181,7 @@ export async function fetchBotActivities(since?: string): Promise { try { if (!GITHUB_TOKEN || GITHUB_TOKEN === 'placeholder') { process.stderr.write('Error: GITHUB_TOKEN environment variable is not set or invalid\n'); - throw new Error('Invalid GITHUB_TOKEN'); + throw new Error(String('Invalid GITHUB_TOKEN')); } console.log('Starting bot activities fetch...'); const activities: Activity[] = []; @@ -208,13 +208,13 @@ export async function fetchBotActivities(since?: string): Promise { console.log('Fetching issues and PRs...'); const fetchStartTime = performance.now(); const items = await fetchAllPages(`${baseUrl}/issues?${params.toString()}`); - console.log(`Fetched ${items.length} items in ${((performance.now() - fetchStartTime) / 1000).toFixed(2)}s`); + console.log(`Fetched ${String(items.length)} items in ${((performance.now() - fetchStartTime) / 1000).toFixed(2)}s`); console.log('Processing items...'); const processStartTime = performance.now(); // Filter items that have comments const itemsWithComments = items.filter(item => item.comments > 0); - console.log(`Processing ${itemsWithComments.length} items with comments in parallel...`); + console.log(`Processing ${String(itemsWithComments.length)} items with comments in parallel...`); // Process items in parallel const batchSize = 10; // Process 10 items at a time to avoid rate limiting @@ -222,7 +222,9 @@ export async function fetchBotActivities(since?: string): Promise { for (let i = 0; i < itemsWithComments.length; i += batchSize) { const batch = itemsWithComments.slice(i, i + batchSize); - console.log(`Processing batch ${Math.floor(i/batchSize) + 1}/${Math.ceil(itemsWithComments.length/batchSize)}...`); + const batchNumber = Math.floor(i/batchSize) + 1; + const totalBatches = Math.ceil(itemsWithComments.length/batchSize); + console.log(`Processing batch ${String(batchNumber)}/${String(totalBatches)}...`); const batchResults = await Promise.all( batch.map(async item => { @@ -253,7 +255,7 @@ export async function fetchBotActivities(since?: string): Promise { console.log('Sorting activities...'); const sortStartTime = performance.now(); const sortedActivities = activities.sort((a, b) => new Date(b.timestamp).getTime() - new Date(a.timestamp).getTime()); - console.log(`Sorted ${activities.length} activities in ${((performance.now() - sortStartTime) / 1000).toFixed(2)}s`); + console.log(`Sorted ${String(activities.length)} activities in ${((performance.now() - sortStartTime) / 1000).toFixed(2)}s`); const totalTime = (performance.now() - startTime) / 1000; console.log(`Total execution time: ${totalTime.toFixed(2)}s`); From 65f2f6d8376ad7b29f46d1a29db26c17c03cabe0 Mon Sep 17 00:00:00 2001 From: openhands Date: Tue, 3 Dec 2024 02:42:46 +0000 Subject: [PATCH 3/7] Fix remaining type error in error message --- scripts/src/github-api.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/src/github-api.ts b/scripts/src/github-api.ts index af5d559..0d62690 100644 --- a/scripts/src/github-api.ts +++ b/scripts/src/github-api.ts @@ -14,7 +14,7 @@ async function fetchWithAuth(url: string): Promise { if (!response.ok) { const errorBody = await response.text(); - throw new Error(`GitHub API error: ${response.status.toString()} ${response.statusText}\n${errorBody}`); + throw new Error(`GitHub API error: ${String(response.status)} ${String(response.statusText)}\n${String(errorBody)}`); } // Parse Link header for pagination From fed7eaab8973ebf4ae3dace99f3c7cbe944b2d43 Mon Sep 17 00:00:00 2001 From: openhands Date: Tue, 3 Dec 2024 02:43:46 +0000 Subject: [PATCH 4/7] Fix remaining type error in error message --- scripts/src/github-api.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/src/github-api.ts b/scripts/src/github-api.ts index 0d62690..ab8b54b 100644 --- a/scripts/src/github-api.ts +++ b/scripts/src/github-api.ts @@ -262,7 +262,7 @@ export async function fetchBotActivities(since?: string): Promise { return sortedActivities; } catch (error) { - console.error('Error fetching bot activities:', error); + console.error('Error fetching bot activities:', String(error)); const totalTime = (performance.now() - startTime) / 1000; console.log(`Total execution time: ${totalTime.toFixed(2)}s (failed)`); throw error; From 8a978784d185fa66216c116bfb47026ecbbadb3c Mon Sep 17 00:00:00 2001 From: openhands Date: Tue, 3 Dec 2024 02:44:55 +0000 Subject: [PATCH 5/7] Fix error message type --- scripts/src/github-api.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/src/github-api.ts b/scripts/src/github-api.ts index ab8b54b..c81ea90 100644 --- a/scripts/src/github-api.ts +++ b/scripts/src/github-api.ts @@ -262,7 +262,8 @@ export async function fetchBotActivities(since?: string): Promise { return sortedActivities; } catch (error) { - console.error('Error fetching bot activities:', String(error)); + const errorMessage = error instanceof Error ? error.message : 'Unknown error'; + console.error('Error fetching bot activities:', errorMessage); const totalTime = (performance.now() - startTime) / 1000; console.log(`Total execution time: ${totalTime.toFixed(2)}s (failed)`); throw error; From 96b9d75538ada03b7fdd020d9a3f34129d15118d Mon Sep 17 00:00:00 2001 From: openhands Date: Tue, 3 Dec 2024 02:46:09 +0000 Subject: [PATCH 6/7] Fix error message type --- scripts/src/github-api.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/src/github-api.ts b/scripts/src/github-api.ts index c81ea90..5ad58f5 100644 --- a/scripts/src/github-api.ts +++ b/scripts/src/github-api.ts @@ -266,6 +266,6 @@ export async function fetchBotActivities(since?: string): Promise { console.error('Error fetching bot activities:', errorMessage); const totalTime = (performance.now() - startTime) / 1000; console.log(`Total execution time: ${totalTime.toFixed(2)}s (failed)`); - throw error; + throw new Error(errorMessage); } } \ No newline at end of file From fb4794ac2d1b3950a4c4991b656a14d6227646ca Mon Sep 17 00:00:00 2001 From: openhands Date: Tue, 3 Dec 2024 02:51:32 +0000 Subject: [PATCH 7/7] Fix error message type --- scripts/src/github-api.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/src/github-api.ts b/scripts/src/github-api.ts index 5ad58f5..394f576 100644 --- a/scripts/src/github-api.ts +++ b/scripts/src/github-api.ts @@ -170,7 +170,7 @@ async function main() { try { await fetchBotActivities(); } catch (error) { - process.stderr.write(`${error}\n`); + process.stderr.write(String(error) + '\n'); process.exit(1); } } @@ -262,10 +262,10 @@ export async function fetchBotActivities(since?: string): Promise { return sortedActivities; } catch (error) { - const errorMessage = error instanceof Error ? error.message : 'Unknown error'; - console.error('Error fetching bot activities:', errorMessage); + const errorMessage = error instanceof Error ? error.message : String(error); + process.stderr.write('Error fetching bot activities: ' + errorMessage + '\n'); const totalTime = (performance.now() - startTime) / 1000; - console.log(`Total execution time: ${totalTime.toFixed(2)}s (failed)`); + process.stderr.write('Total execution time: ' + totalTime.toFixed(2) + 's (failed)\n'); throw new Error(errorMessage); } } \ No newline at end of file