-
- {showLogViewer &&
}
+ {showLogViewer.map((show, index) => (
+
+
+ {show && }
+
+ ))}
{showAnnotationToggle && (
diff --git a/torchci/lib/drciUtils.ts b/torchci/lib/drciUtils.ts
index 7e4a5dda5a..6202389451 100644
--- a/torchci/lib/drciUtils.ts
+++ b/torchci/lib/drciUtils.ts
@@ -311,7 +311,7 @@ export async function hasSimilarFailures(
head_sha: record.sha as string,
head_branch: record.branch as string,
failure_captures: record.failureCaptures as string[],
- failure_line: record.failureLine,
+ failure_lines: record.failureLines ? record.failureLines : undefined,
};
// Only count different jobs with the same failure
@@ -333,9 +333,9 @@ export function isInfraFlakyJob(job: RecentWorkflowsData): boolean {
// the workflow summary tab
return (
job.conclusion === "failure" &&
- (job.failure_line === null ||
- job.failure_line === undefined ||
- job.failure_line === "") &&
+ (job.failure_lines === null ||
+ job.failure_lines === undefined ||
+ job.failure_lines[0] === "") &&
(job.runnerName === null ||
job.runnerName === undefined ||
job.runnerName === "")
diff --git a/torchci/lib/searchUtils.ts b/torchci/lib/searchUtils.ts
index e2cf742699..8307118cde 100644
--- a/torchci/lib/searchUtils.ts
+++ b/torchci/lib/searchUtils.ts
@@ -95,10 +95,11 @@ export async function searchSimilarFailures(
time: data.completed_at,
conclusion: data.conclusion,
htmlUrl: data.html_url,
- failureLine: data.torchci_classification.line,
- failureLineNumber: data.torchci_classification.line_num,
+ failureLines: [data.torchci_classification.line],
+ failureLineNumbers: [data.torchci_classification.line_num],
failureCaptures: data.torchci_classification.captures,
});
});
+
return { jobs: jobs };
}
diff --git a/torchci/lib/types.ts b/torchci/lib/types.ts
index 176a4d94ce..4cd1dcba15 100644
--- a/torchci/lib/types.ts
+++ b/torchci/lib/types.ts
@@ -21,8 +21,8 @@ export interface JobData extends BasicJobData {
logUrl?: string;
durationS?: number;
queueTimeS?: number;
- failureLine?: string;
- failureLineNumber?: number;
+ failureLines?: string[];
+ failureLineNumbers?: number[];
failureCaptures?: string[];
repo?: string;
failureAnnotation?: string;
@@ -41,7 +41,7 @@ export interface RecentWorkflowsData extends BasicJobData {
head_branch?: string | null;
pr_number?: number;
failure_captures: string[];
- failure_line?: string | null;
+ failure_lines?: string[] | null;
}
export interface Artifact {
diff --git a/torchci/pages/api/drci/drci.ts b/torchci/pages/api/drci/drci.ts
index 1b017d988d..13e62811f8 100644
--- a/torchci/pages/api/drci/drci.ts
+++ b/torchci/pages/api/drci/drci.ts
@@ -520,8 +520,8 @@ function isFlaky(job: RecentWorkflowsData, flakyRules: FlakyRule[]): boolean {
failureCapture.match(captureRegex)
);
const matchFailureLine: boolean =
- job.failure_line != null &&
- job.failure_line.match(captureRegex) != null;
+ job.failure_lines != null &&
+ job.failure_lines[0].match(captureRegex) != null;
// Accept both failure captures array and failure line string to make sure
// that nothing is missing
diff --git a/torchci/pages/metrics.tsx b/torchci/pages/metrics.tsx
index 02ae77c63a..68697b5f43 100644
--- a/torchci/pages/metrics.tsx
+++ b/torchci/pages/metrics.tsx
@@ -869,7 +869,6 @@ export default function Page() {
additionalOptions={{ yAxis: { scale: true } }}
/>
-
= PARSE_DATETIME_ISO8601(: startTime)
- AND m._event_time < PARSE_DATETIME_ISO8601(: stopTime)
- GROUP BY
- m.skip_mandatory_checks,
- m.failed_checks,
- m.ignore_current,
- m.is_failed,
- m.pr_num,
- m.merge_commit_sha,
- m.ignore_current_checks,
- m.pending_checks
-),
--- A legit force merge needs to satisfy one of the two conditions below:
--- 1. skip_mandatory_checks is true (-f) and failed_checks_count > 0 (with failures) or pending_checks_count > 0 (impatience).
--- Under this condition, if a force merge (-f) is done when there is no failure and all jobs have finished, it's arguably
--- just a regular merge in disguise.
--- 2. ignore_current is true (-i) and is_failed is false (indicating a successful merge) and ignored_checks_count > 0 (has failures).
--- As -i still waits for all remaining jobs to finish, this shouldn't be counted toward force merge due to impatience.
---
--- If none applies, the merge should be counted as a regular merge regardless of the use of -f or -i. We could track that
--- (regular merges masquerading as force merges) to understand how devs use (or abuse) these flags, but that's arguably a
--- different case altogether.
-merges_identifying_force_merges AS (
- SELECT
- IF(
- (
- skip_mandatory_checks = true
- AND (
- failed_checks_count > 0
- OR pending_checks_count > 0
+-- gets percentage of total force merges, force merges with failures, and force merges without failures (impatient)
+-- specifically this query tracks the force merges kpi on HUD
+WITH
+ issue_comments AS(
+ SELECT
+ issue_comment.user.login,
+ issue_comment.author_association,
+ issue_comment.body,
+ issue_comment.issue_url,
+ issue_comment.html_url,
+ issue_comment.created_at,
+ issue_comment._event_time,
+ CAST(
+ SUBSTR(
+ issue_comment.issue_url,
+ LENGTH(
+ 'https://api.github.com/repos/pytorch/pytorch/issues/'
+ ) + 1
+ ) as INT
+ ) as pr_num
+ FROM
+ commons.issue_comment
+ WHERE
+ (
+ issue_comment.body LIKE '%pytorchbot merge%'
+ OR issue_comment.body LIKE '%pytorchmergebot merge%'
+ ) -- AND _event_time >= PARSE_DATETIME_ISO8601(:startTime)
+ -- AND _event_time < PARSE_DATETIME_ISO8601(:stopTime)
+ AND issue_comment.user.login NOT LIKE '%pytorch-bot%'
+ AND issue_comment.user.login NOT LIKE '%facebook-github-bot%'
+ AND issue_comment.user.login NOT LIKE '%pytorchmergebot%'
+ AND issue_comment.issue_url LIKE '%https://api.github.com/repos/pytorch/pytorch/issues/%'
+ ),
+ all_merges AS (
+ SELECT
+ DISTINCT m.skip_mandatory_checks,
+ LENGTH(m.failed_checks) AS failed_checks_count,
+ LENGTH(m.ignore_current_checks) as ignored_checks_count,
+ m.ignore_current,
+ m.is_failed,
+ m.pr_num,
+ m.merge_commit_sha,
+ max(c._event_time) as time,
+ FROM
+ commons.merges m
+ inner join issue_comments c on m.pr_num = c.pr_num
+ WHERE
+ m.owner = 'pytorch'
+ AND m.project = 'pytorch'
+ AND m.merge_commit_sha != '' -- only consider successful merges
+ AND m._event_time >= PARSE_DATETIME_ISO8601(:startTime)
+ AND m._event_time < PARSE_DATETIME_ISO8601(:stopTime) -- AND m.pr_num in
+ GROUP BY
+ m.skip_mandatory_checks,
+ m.failed_checks,
+ m.ignore_current,
+ m.is_failed,
+ m.pr_num,
+ m.merge_commit_sha,
+ -- and m.pr_num = 104137
+ m.ignore_current_checks
+ ),
+ force_merges_with_failed_checks AS (
+ SELECT
+ IF(
+ (skip_mandatory_checks = true)
+ OR (
+ ignore_current = true
+ AND is_failed = false
+ ),
+ 1,
+ 0
+ ) AS force_merge,
+ failed_checks_count,
+ pr_num,
+ merge_commit_sha,
+ ignore_current,
+ ignored_checks_count,
+ time,
+ FROM
+ all_merges
+ ),
+ results as (
+ SELECT
+ pr_num,
+ merge_commit_sha,
+ force_merge,
+ IF(
+ force_merge = 1
+ AND (
+ failed_checks_count > 0
+ OR ignored_checks_count > 0
+ ),
+ 1,
+ 0
+ ) AS force_merge_with_failures,
+ CAST(time as DATE) as date
+ FROM
+ force_merges_with_failed_checks
+ ORDER BY
+ date DESC
+ ),
+ stats_per_day as (
+ select
+ count(*) as total,
+ sum(force_merge) as total_force_merge_cnt,
+ sum(force_merge_with_failures) as with_failures_cnt,
+ sum(force_merge) - sum(force_merge_with_failures) as impatience_cnt,
+ date,
+ from
+ results
+ GROUP BY
+ date
+ ORDER BY
+ date DESC
+ ),
+ weekly_counts as (
+ select
+ FORMAT_TIMESTAMP('%Y-%m-%d', DATE_TRUNC(: granularity, date)) AS granularity_bucket,
+ SUM(with_failures_cnt) with_failures_cnt,
+ SUM(impatience_cnt) as impatience_cnt,
+ SUM(total) as total,
+ SUM(total_force_merge_cnt) as total_force_merge_cnt
+ from
+ stats_per_day
+ group by
+ granularity_bucket
+ ),
+ stats_per_week as (
+ SELECT
+ granularity_bucket,
+ with_failures_cnt * 100 / total as with_failures_percent,
+ impatience_cnt * 100 / total as impatience_percent,
+ total_force_merge_cnt * 100 / total as force_merge_percent,
+ from
+ weekly_counts
+ ),
+ final_table as (
+ (
+ select
+ granularity_bucket,
+ with_failures_percent as metric,
+ 'From Failures' as name
+
+ from
+ stats_per_week
+ )
+ UNION ALL
+ (
+ select
+ granularity_bucket,
+ impatience_percent as metric,
+ 'From Impatience' as name
+ from
+ stats_per_week
+ )
+ UNION ALL
+ (
+ select
+ granularity_bucket,
+ force_merge_percent as metric,
+ 'All Force Merges' as name
+ from
+ stats_per_week
)
- )
- OR (
- ignore_current = true
- AND is_failed = false
- AND ignored_checks_count > 0 -- if no checks were ignored, it's not a force merge
- ),
- 1,
- 0
- ) AS force_merge,
- failed_checks_count,
- pr_num,
- merge_commit_sha,
- ignore_current,
- ignored_checks_count,
- time,
- FROM
- all_merges
-),
-results AS (
- SELECT
- pr_num,
- merge_commit_sha,
- force_merge,
- IF(
- force_merge = 1
- AND (
- failed_checks_count > 0
- OR ignored_checks_count > 0
- ),
- 1,
- 0
- ) AS force_merge_with_failures,
- CAST(time as DATE) AS date
- FROM
- merges_identifying_force_merges
- ORDER BY
- date DESC
-),
-bucketed_counts AS (
- SELECT
- IF(
- : one_bucket,
- 'Overall',
- FORMAT_TIMESTAMP(
- '%Y-%m-%d',
- DATE_TRUNC(: granularity, date)
- )
- ) AS granularity_bucket,
- SUM(force_merge_with_failures) AS with_failures_cnt,
- SUM(force_merge) - SUM(force_merge_with_failures) AS impatience_cnt,
- COUNT(*) AS total,
- SUM(force_merge) AS total_force_merge_cnt
- FROM
- results
- GROUP BY
- granularity_bucket
-),
-rolling_raw_stats AS (
- -- Average over the past buckets
- SELECT
- granularity_bucket,
- SUM(with_failures_cnt) OVER(
- ORDER BY
- granularity_bucket ROWS 1 PRECEDING
- ) AS with_failures_cnt,
- SUM(impatience_cnt) OVER(
- ORDER BY
- granularity_bucket ROWS 1 PRECEDING
- ) AS impatience_cnt,
- SUM(total_force_merge_cnt) OVER(
- ORDER BY
- granularity_bucket ROWS 1 PRECEDING
- ) AS total_force_merge_cnt,
- SUM(total) OVER(
- ORDER BY
- granularity_bucket ROWS 1 PRECEDING
- ) AS total,
- FROM
- bucketed_counts
-),
-stats_per_bucket AS (
- SELECT
- granularity_bucket,
- with_failures_cnt * 100.0 / total AS with_failures_percent,
- impatience_cnt * 100.0 / total AS impatience_percent,
- total_force_merge_cnt * 100.0 / total AS force_merge_percent,
- FROM
- rolling_raw_stats
-),
-final_table AS (
- (
- SELECT
- granularity_bucket,
- with_failures_percent AS metric,
- 'From Failures' AS name
- FROM
- stats_per_bucket
- )
- UNION ALL
- (
- SELECT
- granularity_bucket,
- impatience_percent AS metric,
- 'From Impatience' AS name
- FROM
- stats_per_bucket
- )
- UNION ALL
- (
- SELECT
- granularity_bucket,
- force_merge_percent AS metric,
- 'All Force Merges' AS name
- FROM
- stats_per_bucket
)
-),
-filtered_result AS (
- SELECT
+select
*
- FROM
+from
final_table
- WHERE
- TRIM(: merge_type) = ''
- OR name LIKE CONCAT('%', : merge_type, '%')
-)
-SELECT
- *
-FROM
- filtered_result
-ORDER BY
- granularity_bucket DESC,
- name
\ No newline at end of file
diff --git a/torchci/rockset/commons/annotated_flaky_jobs.lambda.json b/torchci/rockset/commons/annotated_flaky_jobs.lambda.json
index 4a7b1e89c4..0fcf53c8a9 100644
--- a/torchci/rockset/commons/annotated_flaky_jobs.lambda.json
+++ b/torchci/rockset/commons/annotated_flaky_jobs.lambda.json
@@ -19,7 +19,7 @@
{
"name": "stopTime",
"type": "string",
- "value": "2022-10-19T00:06:32.839Z"
+ "value": "2023-09-19T00:06:32.839Z"
}
],
"description": ""
diff --git a/torchci/rockset/commons/hud_query.lambda.json b/torchci/rockset/commons/hud_query.lambda.json
index db73e65807..778a6841d1 100644
--- a/torchci/rockset/commons/hud_query.lambda.json
+++ b/torchci/rockset/commons/hud_query.lambda.json
@@ -13,4 +13,4 @@
}
],
"description": ""
-}
+}
\ No newline at end of file
diff --git a/torchci/rockset/commons/weekly_force_merge_stats.lambda.json b/torchci/rockset/commons/weekly_force_merge_stats.lambda.json
index 1ddf685d74..4d6d770d07 100644
--- a/torchci/rockset/commons/weekly_force_merge_stats.lambda.json
+++ b/torchci/rockset/commons/weekly_force_merge_stats.lambda.json
@@ -6,26 +6,16 @@
"type": "string",
"value": "week"
},
- {
- "name": "merge_type",
- "type": "string",
- "value": " "
- },
- {
- "name": "one_bucket",
- "type": "bool",
- "value": "False"
- },
{
"name": "startTime",
"type": "string",
- "value": "2023-04-27T00:00:00.000Z"
+ "value": "2023-03-12T11:00:00.000Z"
},
{
"name": "stopTime",
"type": "string",
- "value": "2024-06-01T00:00:00.000Z"
+ "value": "2023-07-27T00:00:00.000Z"
}
],
- "description": "Force merge KPI stats for HUD"
+ "description": ""
}
\ No newline at end of file
diff --git a/torchci/rockset/prodVersions.json b/torchci/rockset/prodVersions.json
index a9309a7d61..663a199467 100644
--- a/torchci/rockset/prodVersions.json
+++ b/torchci/rockset/prodVersions.json
@@ -1,8 +1,8 @@
{
"commons": {
- "annotated_flaky_jobs": "a907d13c9f290bc1",
- "hud_query": "ae178387db09b145",
- "commit_jobs_query": "4cea84282504c9cd",
+ "annotated_flaky_jobs": "bd991c8c9782f339",
+ "hud_query": "69f0bc9a618c82b1",
+ "commit_jobs_query": "8457359bb5183034",
"disabled_non_flaky_tests": "f909abf9eec15b56",
"commit_failed_jobs": "985d62570a63388d",
"filter_forced_merge_pr": "a28350c863e36239",
@@ -12,7 +12,7 @@
"individual_test_stats_per_workflow_per_oncall": "559b6735965f1eb2",
"individual_test_times_per_oncall_per_workflow": "6b63c3dde3032bea",
"flaky_workflows_jobs": "3ac657ca40327f94",
- "failed_workflow_jobs": "6ec4fd3f36a72071",
+ "failed_workflow_jobs": "a91753fbbf82d470",
"get_workflow_jobs": "6ed2029b19691a4b",
"slow_tests": "ef8d035d23aa8ab6",
"test_time_per_file": "1c8d6289623181d8",
@@ -21,7 +21,7 @@
"test_time_and_price_per_oncall": "7af6d14035a19439",
"test_times_per_workflow_type": "3ab0de839b95d22c",
"issue_query": "e4d338de89980044",
- "failure_samples_query": "18e7e696b4949f05",
+ "failure_samples_query": "7940a636284d0752",
"num_commits_master": "e4a864147cf3bf44",
"recent_pr_workflows_query": "0a22b6a523f96bd7",
"reverted_prs_with_reason": "751f01cba16364f0",
@@ -29,7 +29,7 @@
"test_insights_overview": "42dbd5232f45fd53",
"test_insights_latest_runs": "1871833a91cb8b1b",
"master_commit_red_jobs": "4869b467679a616a",
- "weekly_force_merge_stats": "d2264131599bcf6e"
+ "weekly_force_merge_stats": "48bbcbff20f3f5b5"
},
"pytorch_dev_infra_kpis": {
"monthly_contribution_stats": "c1a8751a22f6b6ce",
diff --git a/torchci/scripts/test_check_alerts.py b/torchci/scripts/test_check_alerts.py
index 4f003fad94..0d690cd73e 100644
--- a/torchci/scripts/test_check_alerts.py
+++ b/torchci/scripts/test_check_alerts.py
@@ -27,10 +27,10 @@
"htmlUrl": "https://github.com/pytorch/pytorch/runs/7819529276?check_suite_focus=true",
"logUrl": "https://ossci-raw-job-status.s3.amazonaws.com/log/7819529276",
"durationS": 14876,
- "failureLine": "##[error]The action has timed out.",
+ "failureLines": ["##[error]The action has timed out."],
"failureContext": "",
"failureCaptures": ["##[error]The action has timed out."],
- "failureLineNumber": 83818,
+ "failureLineNumbers": [83818],
"repo": "pytorch/pytorch",
},
{
@@ -40,10 +40,10 @@
"htmlUrl": "https://github.com/pytorch/pytorch/runs/7818399623?check_suite_focus=true",
"logUrl": "https://ossci-raw-job-status.s3.amazonaws.com/log/7818399623",
"durationS": 14882,
- "failureLine": "##[error]The action has timed out.",
+ "failureLines": ["##[error]The action has timed out."],
"failureContext": "",
"failureCaptures": ["##[error]The action has timed out."],
- "failureLineNumber": 72821,
+ "failureLineNumbers": [72821],
"repo": "pytorch/pytorch",
},
]
@@ -55,10 +55,10 @@
"htmlUrl": "https://github.com/pytorch/pytorch/runs/4364234624?check_suite_focus=true",
"logUrl": "https://ossci-raw-job-status.s3.amazonaws.com/log/4364234624",
"durationS": 14342,
- "failureLine": "##[error]An unique error here.",
+ "failureLines": ["##[error]An unique error here."],
"failureContext": "",
"failureCaptures": ["##[error]An unique error here."],
- "failureLineNumber": 12345,
+ "failureLineNumbers": [12345],
"repo": "pytorch/pytorch",
},
]
diff --git a/torchci/test/drciUtils.test.ts b/torchci/test/drciUtils.test.ts
index 133b977d39..abe08541db 100644
--- a/torchci/test/drciUtils.test.ts
+++ b/torchci/test/drciUtils.test.ts
@@ -39,8 +39,8 @@ describe("Test various utils used by Dr.CI", () => {
time: mockEndDate,
conclusion: "failure",
htmlUrl: "Anything goes",
- failureLine: "ERROR",
- failureLineNumber: 0,
+ failureLines: ["ERROR"],
+ failureLineNumbers: [0],
failureCaptures: ["ERROR"],
};
const mock = jest.spyOn(searchUtils, "searchSimilarFailures");
@@ -214,8 +214,8 @@ describe("Test various utils used by Dr.CI", () => {
time: "2023-08-01T00:00:00Z",
conclusion: "failure",
htmlUrl: "Anything goes",
- failureLine: "ERROR",
- failureLineNumber: 0,
+ failureLines: ["ERROR"],
+ failureLineNumbers: [0],
failureCaptures: ["ERROR"],
};
mock.mockImplementation(() => Promise.resolve({ jobs: [mockJobData] }));
@@ -310,7 +310,7 @@ describe("Test various utils used by Dr.CI", () => {
name: "A",
html_url: "A",
head_sha: "A",
- failure_line: "ERROR",
+ failure_lines: ["ERROR"],
failure_captures: ["ERROR"],
conclusion: "failure",
completed_at: "2023-08-01T00:00:00Z",
@@ -324,7 +324,7 @@ describe("Test various utils used by Dr.CI", () => {
name: "A",
html_url: "A",
head_sha: "A",
- failure_line: "",
+ failure_lines: [""],
failure_captures: [],
conclusion: "failure",
completed_at: "2023-08-01T00:00:00Z",
@@ -338,7 +338,7 @@ describe("Test various utils used by Dr.CI", () => {
name: "A",
html_url: "A",
head_sha: "A",
- failure_line: "",
+ failure_lines: [""],
failure_captures: [],
conclusion: "failure",
completed_at: "2023-08-01T00:00:00Z",