diff --git a/torchci/components/JobLinks.tsx b/torchci/components/JobLinks.tsx index d5002c96ce..ce4adb6a50 100644 --- a/torchci/components/JobLinks.tsx +++ b/torchci/components/JobLinks.tsx @@ -61,11 +61,11 @@ export default function JobLinks({ job }: { job: JobData }) { {authenticated && } - {authenticated && job.failureLine && ( + {authenticated && job.failureLines && ( )} @@ -98,7 +98,7 @@ This test was disabled because it is failing on main branch ([recent examples]($ } function DisableTest({ job, label }: { job: JobData; label: string }) { - const hasFailureClassification = job.failureLine != null; + const hasFailureClassification = job.failureLines != null; const swrKey = hasFailureClassification ? `/api/issue/${label}` : null; const { data } = useSWR(swrKey, fetcher, { // Set a 60s cache for the request, so that lots of tooltip hovers don't @@ -114,7 +114,7 @@ function DisableTest({ job, label }: { job: JobData; label: string }) { return null; } - const testName = getTestName(job.failureLine!); + const testName = job.failureLines ? getTestName(job.failureLines[0]) : null; // - The failure classification is not a python unittest or pytest failure. if (testName === null) { return null; diff --git a/torchci/components/LogViewer.tsx b/torchci/components/LogViewer.tsx index 9688cf35f9..2b0fd40ba0 100644 --- a/torchci/components/LogViewer.tsx +++ b/torchci/components/LogViewer.tsx @@ -194,13 +194,22 @@ function Log({ url, line }: { url: string; line: number | null }) { export default function LogViewer({ job, logRating = LogAnnotation.NULL, - showAnnotationToggle = false, + showAnnotationToggle = process.env.DEBUG_LOG_CLASSIFIER === "true", + maxNumFailureLines = process.env.DEBUG_LOG_CLASSIFIER === "true" ? 2 : 1, }: { job: JobData; logRating?: LogAnnotation; showAnnotationToggle?: boolean; + maxNumFailureLines?: number; }) { - const [showLogViewer, setShowLogViewer] = useState(false); + // const numFailureLines = + // Math.min(job.failureLines?.length || 0, maxNumFailureLines); + + // we will replace this with the code above once we support having multiple failure lines in rockset + const numFailureLines = maxNumFailureLines; + const [showLogViewer, setShowLogViewer] = useState( + Array.from({ length: numFailureLines }, () => false) + ); useEffect(() => { document.addEventListener("copy", (e) => { @@ -213,28 +222,39 @@ export default function LogViewer({ }); }); - if (!job.failureLine && !isFailure(job.conclusion)) { + if (!job.failureLines && !isFailure(job.conclusion)) { return null; } - function handleClick() { - setShowLogViewer(!showLogViewer); - } + const toggleLogViewer = (index: number) => { + // Make a copy of the current array state + const updatedShowLogViewer = [...showLogViewer]; + // Toggle the boolean value at the given index + updatedShowLogViewer[index] = !updatedShowLogViewer[index]; + + // Update the state + setShowLogViewer(updatedShowLogViewer); + }; return (
- - {showLogViewer && } + {showLogViewer.map((show, index) => ( +
+ + {show && } +
+ ))} {showAnnotationToggle && (
diff --git a/torchci/lib/drciUtils.ts b/torchci/lib/drciUtils.ts index 7e4a5dda5a..6202389451 100644 --- a/torchci/lib/drciUtils.ts +++ b/torchci/lib/drciUtils.ts @@ -311,7 +311,7 @@ export async function hasSimilarFailures( head_sha: record.sha as string, head_branch: record.branch as string, failure_captures: record.failureCaptures as string[], - failure_line: record.failureLine, + failure_lines: record.failureLines ? record.failureLines : undefined, }; // Only count different jobs with the same failure @@ -333,9 +333,9 @@ export function isInfraFlakyJob(job: RecentWorkflowsData): boolean { // the workflow summary tab return ( job.conclusion === "failure" && - (job.failure_line === null || - job.failure_line === undefined || - job.failure_line === "") && + (job.failure_lines === null || + job.failure_lines === undefined || + job.failure_lines[0] === "") && (job.runnerName === null || job.runnerName === undefined || job.runnerName === "") diff --git a/torchci/lib/searchUtils.ts b/torchci/lib/searchUtils.ts index e2cf742699..8307118cde 100644 --- a/torchci/lib/searchUtils.ts +++ b/torchci/lib/searchUtils.ts @@ -95,10 +95,11 @@ export async function searchSimilarFailures( time: data.completed_at, conclusion: data.conclusion, htmlUrl: data.html_url, - failureLine: data.torchci_classification.line, - failureLineNumber: data.torchci_classification.line_num, + failureLines: [data.torchci_classification.line], + failureLineNumbers: [data.torchci_classification.line_num], failureCaptures: data.torchci_classification.captures, }); }); + return { jobs: jobs }; } diff --git a/torchci/lib/types.ts b/torchci/lib/types.ts index 176a4d94ce..4cd1dcba15 100644 --- a/torchci/lib/types.ts +++ b/torchci/lib/types.ts @@ -21,8 +21,8 @@ export interface JobData extends BasicJobData { logUrl?: string; durationS?: number; queueTimeS?: number; - failureLine?: string; - failureLineNumber?: number; + failureLines?: string[]; + failureLineNumbers?: number[]; failureCaptures?: string[]; repo?: string; failureAnnotation?: string; @@ -41,7 +41,7 @@ export interface RecentWorkflowsData extends BasicJobData { head_branch?: string | null; pr_number?: number; failure_captures: string[]; - failure_line?: string | null; + failure_lines?: string[] | null; } export interface Artifact { diff --git a/torchci/pages/api/drci/drci.ts b/torchci/pages/api/drci/drci.ts index 1b017d988d..13e62811f8 100644 --- a/torchci/pages/api/drci/drci.ts +++ b/torchci/pages/api/drci/drci.ts @@ -520,8 +520,8 @@ function isFlaky(job: RecentWorkflowsData, flakyRules: FlakyRule[]): boolean { failureCapture.match(captureRegex) ); const matchFailureLine: boolean = - job.failure_line != null && - job.failure_line.match(captureRegex) != null; + job.failure_lines != null && + job.failure_lines[0].match(captureRegex) != null; // Accept both failure captures array and failure line string to make sure // that nothing is missing diff --git a/torchci/pages/metrics.tsx b/torchci/pages/metrics.tsx index 02ae77c63a..68697b5f43 100644 --- a/torchci/pages/metrics.tsx +++ b/torchci/pages/metrics.tsx @@ -869,7 +869,6 @@ export default function Page() { additionalOptions={{ yAxis: { scale: true } }} /> - = PARSE_DATETIME_ISO8601(: startTime) - AND m._event_time < PARSE_DATETIME_ISO8601(: stopTime) - GROUP BY - m.skip_mandatory_checks, - m.failed_checks, - m.ignore_current, - m.is_failed, - m.pr_num, - m.merge_commit_sha, - m.ignore_current_checks, - m.pending_checks -), --- A legit force merge needs to satisfy one of the two conditions below: --- 1. skip_mandatory_checks is true (-f) and failed_checks_count > 0 (with failures) or pending_checks_count > 0 (impatience). --- Under this condition, if a force merge (-f) is done when there is no failure and all jobs have finished, it's arguably --- just a regular merge in disguise. --- 2. ignore_current is true (-i) and is_failed is false (indicating a successful merge) and ignored_checks_count > 0 (has failures). --- As -i still waits for all remaining jobs to finish, this shouldn't be counted toward force merge due to impatience. --- --- If none applies, the merge should be counted as a regular merge regardless of the use of -f or -i. We could track that --- (regular merges masquerading as force merges) to understand how devs use (or abuse) these flags, but that's arguably a --- different case altogether. -merges_identifying_force_merges AS ( - SELECT - IF( - ( - skip_mandatory_checks = true - AND ( - failed_checks_count > 0 - OR pending_checks_count > 0 +-- gets percentage of total force merges, force merges with failures, and force merges without failures (impatient) +-- specifically this query tracks the force merges kpi on HUD +WITH + issue_comments AS( + SELECT + issue_comment.user.login, + issue_comment.author_association, + issue_comment.body, + issue_comment.issue_url, + issue_comment.html_url, + issue_comment.created_at, + issue_comment._event_time, + CAST( + SUBSTR( + issue_comment.issue_url, + LENGTH( + 'https://api.github.com/repos/pytorch/pytorch/issues/' + ) + 1 + ) as INT + ) as pr_num + FROM + commons.issue_comment + WHERE + ( + issue_comment.body LIKE '%pytorchbot merge%' + OR issue_comment.body LIKE '%pytorchmergebot merge%' + ) -- AND _event_time >= PARSE_DATETIME_ISO8601(:startTime) + -- AND _event_time < PARSE_DATETIME_ISO8601(:stopTime) + AND issue_comment.user.login NOT LIKE '%pytorch-bot%' + AND issue_comment.user.login NOT LIKE '%facebook-github-bot%' + AND issue_comment.user.login NOT LIKE '%pytorchmergebot%' + AND issue_comment.issue_url LIKE '%https://api.github.com/repos/pytorch/pytorch/issues/%' + ), + all_merges AS ( + SELECT + DISTINCT m.skip_mandatory_checks, + LENGTH(m.failed_checks) AS failed_checks_count, + LENGTH(m.ignore_current_checks) as ignored_checks_count, + m.ignore_current, + m.is_failed, + m.pr_num, + m.merge_commit_sha, + max(c._event_time) as time, + FROM + commons.merges m + inner join issue_comments c on m.pr_num = c.pr_num + WHERE + m.owner = 'pytorch' + AND m.project = 'pytorch' + AND m.merge_commit_sha != '' -- only consider successful merges + AND m._event_time >= PARSE_DATETIME_ISO8601(:startTime) + AND m._event_time < PARSE_DATETIME_ISO8601(:stopTime) -- AND m.pr_num in + GROUP BY + m.skip_mandatory_checks, + m.failed_checks, + m.ignore_current, + m.is_failed, + m.pr_num, + m.merge_commit_sha, + -- and m.pr_num = 104137 + m.ignore_current_checks + ), + force_merges_with_failed_checks AS ( + SELECT + IF( + (skip_mandatory_checks = true) + OR ( + ignore_current = true + AND is_failed = false + ), + 1, + 0 + ) AS force_merge, + failed_checks_count, + pr_num, + merge_commit_sha, + ignore_current, + ignored_checks_count, + time, + FROM + all_merges + ), + results as ( + SELECT + pr_num, + merge_commit_sha, + force_merge, + IF( + force_merge = 1 + AND ( + failed_checks_count > 0 + OR ignored_checks_count > 0 + ), + 1, + 0 + ) AS force_merge_with_failures, + CAST(time as DATE) as date + FROM + force_merges_with_failed_checks + ORDER BY + date DESC + ), + stats_per_day as ( + select + count(*) as total, + sum(force_merge) as total_force_merge_cnt, + sum(force_merge_with_failures) as with_failures_cnt, + sum(force_merge) - sum(force_merge_with_failures) as impatience_cnt, + date, + from + results + GROUP BY + date + ORDER BY + date DESC + ), + weekly_counts as ( + select + FORMAT_TIMESTAMP('%Y-%m-%d', DATE_TRUNC(: granularity, date)) AS granularity_bucket, + SUM(with_failures_cnt) with_failures_cnt, + SUM(impatience_cnt) as impatience_cnt, + SUM(total) as total, + SUM(total_force_merge_cnt) as total_force_merge_cnt + from + stats_per_day + group by + granularity_bucket + ), + stats_per_week as ( + SELECT + granularity_bucket, + with_failures_cnt * 100 / total as with_failures_percent, + impatience_cnt * 100 / total as impatience_percent, + total_force_merge_cnt * 100 / total as force_merge_percent, + from + weekly_counts + ), + final_table as ( + ( + select + granularity_bucket, + with_failures_percent as metric, + 'From Failures' as name + + from + stats_per_week + ) + UNION ALL + ( + select + granularity_bucket, + impatience_percent as metric, + 'From Impatience' as name + from + stats_per_week + ) + UNION ALL + ( + select + granularity_bucket, + force_merge_percent as metric, + 'All Force Merges' as name + from + stats_per_week ) - ) - OR ( - ignore_current = true - AND is_failed = false - AND ignored_checks_count > 0 -- if no checks were ignored, it's not a force merge - ), - 1, - 0 - ) AS force_merge, - failed_checks_count, - pr_num, - merge_commit_sha, - ignore_current, - ignored_checks_count, - time, - FROM - all_merges -), -results AS ( - SELECT - pr_num, - merge_commit_sha, - force_merge, - IF( - force_merge = 1 - AND ( - failed_checks_count > 0 - OR ignored_checks_count > 0 - ), - 1, - 0 - ) AS force_merge_with_failures, - CAST(time as DATE) AS date - FROM - merges_identifying_force_merges - ORDER BY - date DESC -), -bucketed_counts AS ( - SELECT - IF( - : one_bucket, - 'Overall', - FORMAT_TIMESTAMP( - '%Y-%m-%d', - DATE_TRUNC(: granularity, date) - ) - ) AS granularity_bucket, - SUM(force_merge_with_failures) AS with_failures_cnt, - SUM(force_merge) - SUM(force_merge_with_failures) AS impatience_cnt, - COUNT(*) AS total, - SUM(force_merge) AS total_force_merge_cnt - FROM - results - GROUP BY - granularity_bucket -), -rolling_raw_stats AS ( - -- Average over the past buckets - SELECT - granularity_bucket, - SUM(with_failures_cnt) OVER( - ORDER BY - granularity_bucket ROWS 1 PRECEDING - ) AS with_failures_cnt, - SUM(impatience_cnt) OVER( - ORDER BY - granularity_bucket ROWS 1 PRECEDING - ) AS impatience_cnt, - SUM(total_force_merge_cnt) OVER( - ORDER BY - granularity_bucket ROWS 1 PRECEDING - ) AS total_force_merge_cnt, - SUM(total) OVER( - ORDER BY - granularity_bucket ROWS 1 PRECEDING - ) AS total, - FROM - bucketed_counts -), -stats_per_bucket AS ( - SELECT - granularity_bucket, - with_failures_cnt * 100.0 / total AS with_failures_percent, - impatience_cnt * 100.0 / total AS impatience_percent, - total_force_merge_cnt * 100.0 / total AS force_merge_percent, - FROM - rolling_raw_stats -), -final_table AS ( - ( - SELECT - granularity_bucket, - with_failures_percent AS metric, - 'From Failures' AS name - FROM - stats_per_bucket - ) - UNION ALL - ( - SELECT - granularity_bucket, - impatience_percent AS metric, - 'From Impatience' AS name - FROM - stats_per_bucket - ) - UNION ALL - ( - SELECT - granularity_bucket, - force_merge_percent AS metric, - 'All Force Merges' AS name - FROM - stats_per_bucket ) -), -filtered_result AS ( - SELECT +select * - FROM +from final_table - WHERE - TRIM(: merge_type) = '' - OR name LIKE CONCAT('%', : merge_type, '%') -) -SELECT - * -FROM - filtered_result -ORDER BY - granularity_bucket DESC, - name \ No newline at end of file diff --git a/torchci/rockset/commons/annotated_flaky_jobs.lambda.json b/torchci/rockset/commons/annotated_flaky_jobs.lambda.json index 4a7b1e89c4..0fcf53c8a9 100644 --- a/torchci/rockset/commons/annotated_flaky_jobs.lambda.json +++ b/torchci/rockset/commons/annotated_flaky_jobs.lambda.json @@ -19,7 +19,7 @@ { "name": "stopTime", "type": "string", - "value": "2022-10-19T00:06:32.839Z" + "value": "2023-09-19T00:06:32.839Z" } ], "description": "" diff --git a/torchci/rockset/commons/hud_query.lambda.json b/torchci/rockset/commons/hud_query.lambda.json index db73e65807..778a6841d1 100644 --- a/torchci/rockset/commons/hud_query.lambda.json +++ b/torchci/rockset/commons/hud_query.lambda.json @@ -13,4 +13,4 @@ } ], "description": "" -} +} \ No newline at end of file diff --git a/torchci/rockset/commons/weekly_force_merge_stats.lambda.json b/torchci/rockset/commons/weekly_force_merge_stats.lambda.json index 1ddf685d74..4d6d770d07 100644 --- a/torchci/rockset/commons/weekly_force_merge_stats.lambda.json +++ b/torchci/rockset/commons/weekly_force_merge_stats.lambda.json @@ -6,26 +6,16 @@ "type": "string", "value": "week" }, - { - "name": "merge_type", - "type": "string", - "value": " " - }, - { - "name": "one_bucket", - "type": "bool", - "value": "False" - }, { "name": "startTime", "type": "string", - "value": "2023-04-27T00:00:00.000Z" + "value": "2023-03-12T11:00:00.000Z" }, { "name": "stopTime", "type": "string", - "value": "2024-06-01T00:00:00.000Z" + "value": "2023-07-27T00:00:00.000Z" } ], - "description": "Force merge KPI stats for HUD" + "description": "" } \ No newline at end of file diff --git a/torchci/rockset/prodVersions.json b/torchci/rockset/prodVersions.json index a9309a7d61..663a199467 100644 --- a/torchci/rockset/prodVersions.json +++ b/torchci/rockset/prodVersions.json @@ -1,8 +1,8 @@ { "commons": { - "annotated_flaky_jobs": "a907d13c9f290bc1", - "hud_query": "ae178387db09b145", - "commit_jobs_query": "4cea84282504c9cd", + "annotated_flaky_jobs": "bd991c8c9782f339", + "hud_query": "69f0bc9a618c82b1", + "commit_jobs_query": "8457359bb5183034", "disabled_non_flaky_tests": "f909abf9eec15b56", "commit_failed_jobs": "985d62570a63388d", "filter_forced_merge_pr": "a28350c863e36239", @@ -12,7 +12,7 @@ "individual_test_stats_per_workflow_per_oncall": "559b6735965f1eb2", "individual_test_times_per_oncall_per_workflow": "6b63c3dde3032bea", "flaky_workflows_jobs": "3ac657ca40327f94", - "failed_workflow_jobs": "6ec4fd3f36a72071", + "failed_workflow_jobs": "a91753fbbf82d470", "get_workflow_jobs": "6ed2029b19691a4b", "slow_tests": "ef8d035d23aa8ab6", "test_time_per_file": "1c8d6289623181d8", @@ -21,7 +21,7 @@ "test_time_and_price_per_oncall": "7af6d14035a19439", "test_times_per_workflow_type": "3ab0de839b95d22c", "issue_query": "e4d338de89980044", - "failure_samples_query": "18e7e696b4949f05", + "failure_samples_query": "7940a636284d0752", "num_commits_master": "e4a864147cf3bf44", "recent_pr_workflows_query": "0a22b6a523f96bd7", "reverted_prs_with_reason": "751f01cba16364f0", @@ -29,7 +29,7 @@ "test_insights_overview": "42dbd5232f45fd53", "test_insights_latest_runs": "1871833a91cb8b1b", "master_commit_red_jobs": "4869b467679a616a", - "weekly_force_merge_stats": "d2264131599bcf6e" + "weekly_force_merge_stats": "48bbcbff20f3f5b5" }, "pytorch_dev_infra_kpis": { "monthly_contribution_stats": "c1a8751a22f6b6ce", diff --git a/torchci/scripts/test_check_alerts.py b/torchci/scripts/test_check_alerts.py index 4f003fad94..0d690cd73e 100644 --- a/torchci/scripts/test_check_alerts.py +++ b/torchci/scripts/test_check_alerts.py @@ -27,10 +27,10 @@ "htmlUrl": "https://github.com/pytorch/pytorch/runs/7819529276?check_suite_focus=true", "logUrl": "https://ossci-raw-job-status.s3.amazonaws.com/log/7819529276", "durationS": 14876, - "failureLine": "##[error]The action has timed out.", + "failureLines": ["##[error]The action has timed out."], "failureContext": "", "failureCaptures": ["##[error]The action has timed out."], - "failureLineNumber": 83818, + "failureLineNumbers": [83818], "repo": "pytorch/pytorch", }, { @@ -40,10 +40,10 @@ "htmlUrl": "https://github.com/pytorch/pytorch/runs/7818399623?check_suite_focus=true", "logUrl": "https://ossci-raw-job-status.s3.amazonaws.com/log/7818399623", "durationS": 14882, - "failureLine": "##[error]The action has timed out.", + "failureLines": ["##[error]The action has timed out."], "failureContext": "", "failureCaptures": ["##[error]The action has timed out."], - "failureLineNumber": 72821, + "failureLineNumbers": [72821], "repo": "pytorch/pytorch", }, ] @@ -55,10 +55,10 @@ "htmlUrl": "https://github.com/pytorch/pytorch/runs/4364234624?check_suite_focus=true", "logUrl": "https://ossci-raw-job-status.s3.amazonaws.com/log/4364234624", "durationS": 14342, - "failureLine": "##[error]An unique error here.", + "failureLines": ["##[error]An unique error here."], "failureContext": "", "failureCaptures": ["##[error]An unique error here."], - "failureLineNumber": 12345, + "failureLineNumbers": [12345], "repo": "pytorch/pytorch", }, ] diff --git a/torchci/test/drciUtils.test.ts b/torchci/test/drciUtils.test.ts index 133b977d39..abe08541db 100644 --- a/torchci/test/drciUtils.test.ts +++ b/torchci/test/drciUtils.test.ts @@ -39,8 +39,8 @@ describe("Test various utils used by Dr.CI", () => { time: mockEndDate, conclusion: "failure", htmlUrl: "Anything goes", - failureLine: "ERROR", - failureLineNumber: 0, + failureLines: ["ERROR"], + failureLineNumbers: [0], failureCaptures: ["ERROR"], }; const mock = jest.spyOn(searchUtils, "searchSimilarFailures"); @@ -214,8 +214,8 @@ describe("Test various utils used by Dr.CI", () => { time: "2023-08-01T00:00:00Z", conclusion: "failure", htmlUrl: "Anything goes", - failureLine: "ERROR", - failureLineNumber: 0, + failureLines: ["ERROR"], + failureLineNumbers: [0], failureCaptures: ["ERROR"], }; mock.mockImplementation(() => Promise.resolve({ jobs: [mockJobData] })); @@ -310,7 +310,7 @@ describe("Test various utils used by Dr.CI", () => { name: "A", html_url: "A", head_sha: "A", - failure_line: "ERROR", + failure_lines: ["ERROR"], failure_captures: ["ERROR"], conclusion: "failure", completed_at: "2023-08-01T00:00:00Z", @@ -324,7 +324,7 @@ describe("Test various utils used by Dr.CI", () => { name: "A", html_url: "A", head_sha: "A", - failure_line: "", + failure_lines: [""], failure_captures: [], conclusion: "failure", completed_at: "2023-08-01T00:00:00Z", @@ -338,7 +338,7 @@ describe("Test various utils used by Dr.CI", () => { name: "A", html_url: "A", head_sha: "A", - failure_line: "", + failure_lines: [""], failure_captures: [], conclusion: "failure", completed_at: "2023-08-01T00:00:00Z",