Add config for failed builds channel

lib/action.ml

@@ -58,7 +58,7 @@ module Action (Github_api : Api.Github) (Slack_api : Api.Slack) = struct
     n.commits
     |> List.filter (fun c ->
            let skip = Github.is_merge_commit_to_ignore ~cfg ~branch c in
-           if skip then log#info "main branch merge, ignoring %s: %s" c.id (first_line c.message);
+           if skip then log#info "main branch merge, ignoring %s: %s" c.id (Util.first_line c.message);
            not skip)
     |> List.concat_map (fun commit ->
            let rules = List.filter (filter_by_branch ~distinct:commit.distinct) rules in
@@ -168,7 +168,13 @@ module Action (Github_api : Api.Github) (Slack_api : Api.Slack) = struct
           | Error e -> action_error e
           | Ok commit -> Lwt.return @@ partition_commit cfg commit.files)
       in
-      Lwt.return (direct_message @ chans)
+      match Util.Build.is_failed_build n, cfg.status_rules.failed_builds_channel with
+      | true, Some failed_builds_channel ->
+        (* if we have a failed build and a failed builds channel, we send one notification there too,
+           but we don't notify the same channel twice *)
+        let chans = failed_builds_channel :: chans |> List.sort_uniq String.compare in
+        Lwt.return (direct_message @ chans)
+      | _ -> Lwt.return (direct_message @ chans)
     in
     let%lwt recipients =
       if Context.is_pipeline_allowed ctx repo.url ~pipeline then begin
@@ -265,7 +271,18 @@ module Action (Github_api : Api.Github) (Slack_api : Api.Slack) = struct
       Lwt.return notifs
     | Status n ->
       let%lwt channels = partition_status ctx n in
-      let notifs = List.map (generate_status_notification cfg n) channels in
+      let%lwt slack_user_id =
+        match Util.Build.is_failed_build n with
+        | false -> Lwt.return_none
+        | true ->
+          let email = n.commit.commit.author.email in
+          (match%lwt Slack_api.lookup_user ~ctx ~cfg ~email () with
+          | Ok (res : Slack_t.lookup_user_res) -> Lwt.return_some res.user.id
+          | Error e ->
+            log#warn "couldn't match commit email %s to slack profile: %s" email e;
+            Lwt.return_some email)
+      in
+      let notifs = List.map (generate_status_notification ?slack_user_id ~ctx cfg n) channels in
       Lwt.return notifs
 
   let send_notifications (ctx : Context.t) notifications =

lib/api_remote.ml

@@ -1,6 +1,7 @@
 open Printf
 open Devkit
 open Common
+open Util
 
 module Github : Api.Github = struct
   let commits_url ~(repo : Github_t.repository) ~sha =

lib/common.ml

@@ -40,32 +40,3 @@ module Re2 = struct
   let wrap s = create_exn s
   let unwrap = Re2.to_string
 end
-
-open Devkit
-
-let fmt_error ?exn fmt =
-  Printf.ksprintf
-    (fun s ->
-      match exn with
-      | Some exn -> Error (s ^ " :  exn " ^ Exn.str exn)
-      | None -> Error s)
-    fmt
-
-let first_line s =
-  match String.split_on_char '\n' s with
-  | x :: _ -> x
-  | [] -> s
-
-let decode_string_pad s = Stre.rstrip ~chars:"= \n\r\t" s |> Base64.decode_exn ~pad:false
-
-let http_request ?headers ?body meth path =
-  let setup h =
-    Curl.set_followlocation h true;
-    Curl.set_maxredirs h 1
-  in
-  match%lwt Web.http_request_lwt ~setup ~ua:"monorobot" ~verbose:true ?headers ?body meth path with
-  | `Ok s -> Lwt.return @@ Ok s
-  | `Error e -> Lwt.return @@ Error e
-
-let sign_string_sha256 ~key ~basestring =
-  Cstruct.of_string basestring |> Nocrypto.Hash.SHA256.hmac ~key:(Cstruct.of_string key) |> Hex.of_cstruct |> Hex.show

lib/config.atd

@@ -6,6 +6,7 @@ type project_owners_rule <ocaml from="Rule"> = abstract
 (* This type of rule is used for CI build notifications. *)
 type status_rules = {
   ?allowed_pipelines : string list nullable; (* keep only status events with a title matching this list *)
+  ?failed_builds_channel: string nullable; (* channel to post failed builds notifications to *)
   rules: status_rule list;
 }
 
@@ -33,7 +34,7 @@ type project_owners = {
 type config = {
   prefix_rules : prefix_rules;
   label_rules : label_rules;
-  ~status_rules <ocaml default="{allowed_pipelines = Some []; rules = []}"> : status_rules;
+  ~status_rules <ocaml default="{allowed_pipelines = Some []; rules = []; failed_builds_channel = None}"> : status_rules;
   ~project_owners <ocaml default="{rules = []}"> : project_owners;
   ~ignored_users <ocaml default="[]">: string list; (* list of ignored users *)
   ?main_branch_name : string nullable; (* the name of the main branch; used to filter out notifications about merges of main branch into other branches *)

lib/context.ml

@@ -83,6 +83,7 @@ let is_pipeline_allowed ctx repo_url ~pipeline =
   | _ -> true
 
 let refresh_secrets ctx =
+  let open Util in
   let path = ctx.secrets_filepath in
   match Config_j.secrets_of_string (Std.input_file path) with
   | exception exn -> fmt_error ~exn "failed to read secrets from file %s" path
@@ -104,7 +105,7 @@ let refresh_state ctx =
       log#info "loading saved state from file %s" path;
       (* todo: extract state related parts to state.ml *)
       match State_j.state_of_string (Std.input_file path) with
-      | exception exn -> fmt_error ~exn "failed to read state from file %s" path
+      | exception exn -> Util.fmt_error ~exn "failed to read state from file %s" path
       | state -> Ok { ctx with state = { State.state } }
     end
     else Ok ctx

lib/github.ml

@@ -49,7 +49,7 @@ let is_merge_commit_to_ignore ~(cfg : Config_t.config) ~branch commit =
       Merge remote-tracking branch 'origin/develop' into feature_branch
       Merge remote-tracking branch 'origin/develop' (the default message pattern generated by GitHub "Update with merge commit" button)
     *)
-    let title = Common.first_line commit.message in
+    let title = Util.first_line commit.message in
     begin
       match Re2.find_submatches_exn merge_commit_re title with
       | [| Some _; Some incoming_branch; receiving_branch |] ->

lib/slack.ml

@@ -1,5 +1,6 @@
 open Printf
 open Common
+open Util
 open Mrkdwn
 open Github_j
 open Slack_j
@@ -240,9 +241,10 @@ let generate_push_notification notification channel =
 
 let buildkite_description_re = Re2.create_exn {|^Build #(\d+)(.*)|}
 
-let generate_status_notification (cfg : Config_t.config) (notification : status_notification) channel =
+let generate_status_notification ~(ctx : Context.t) ?slack_user_id (cfg : Config_t.config)
+  (notification : status_notification) channel =
   let { commit; state; description; target_url; context; repository; _ } = notification in
-  let ({ commit : inner_commit; sha; author; html_url; _ } : status_commit) = commit in
+  let ({ commit : inner_commit; sha; html_url; _ } : status_commit) = commit in
   let ({ message; _ } : inner_commit) = commit in
   let is_buildkite = String.starts_with context ~prefix:"buildkite" in
   let color_info = if state = Success then "good" else "danger" in
@@ -265,17 +267,12 @@ let generate_status_notification (cfg : Config_t.config) (notification : status_
   in
   let commit_info =
     [
-      (* if we have a DM notification, we don't need to repeat the commit message and author because
-         the user receiving the message is already the author of that commit. Users handles start with U *)
-      (match Devkit.Stre.starts_with channel "U" with
-      | true -> sprintf "*Commit*: `<%s|%s>`" html_url (git_short_sha_hash sha)
-      | false ->
-        sprintf "*Commit*: `<%s|%s>` %s - %s" html_url (git_short_sha_hash sha) (first_line message)
-          ((* If the author's email is not associated with a github account the author will be missing.
-               Using the information from the commit instead, which should be equivalent. *)
-           Option.map_default
-             (fun { login; _ } -> login)
-             commit.author.name author));
+      (let mention =
+         match slack_user_id with
+         | None -> ""
+         | Some id -> sprintf "<@%s>" id
+       in
+       sprintf "*Commit*: `<%s|%s>` %s" html_url (git_short_sha_hash sha) mention);
     ]
   in
   let branches_info =
@@ -323,7 +320,24 @@ let generate_status_notification (cfg : Config_t.config) (notification : status_
         | None -> default_summary
         | Some pipeline_url -> sprintf "<%s|[%s]>: %s for \"%s\"" pipeline_url context build_desc commit_message)
   in
-  let msg = String.concat "\n" @@ List.concat [ commit_info; branches_info ] in
+  let failed_builds_info =
+    let is_failed_builds_channel =
+      Option.map_default (String.equal channel) false cfg.status_rules.failed_builds_channel
+    in
+    match Util.Build.is_failed_build notification && is_failed_builds_channel with
+    | false -> []
+    | true ->
+      let repo_state = State.find_or_add_repo ctx.state repository.url in
+      let pipeline = notification.context in
+      let slack_step_link (s, l) =
+        let step = Devkit.Stre.drop_prefix s (pipeline ^ "/") in
+        Printf.sprintf "<%s|%s> " l step
+      in
+      (match Build.new_failed_steps notification repo_state pipeline with
+      | [] -> []
+      | steps -> [ sprintf "*Steps broken*: %s" (String.concat ", " (List.map slack_step_link steps)) ])
+  in
+  let msg = String.concat "\n" @@ List.concat [ commit_info; branches_info; failed_builds_info ] in
   let attachment =
     { empty_attachments with mrkdwn_in = Some [ "fields"; "text" ]; color = Some color_info; text = Some msg }
   in
@@ -362,5 +376,5 @@ let validate_signature ?(version = "v0") ?signing_key ~headers body =
   | None -> Error "unable to find header X-Slack-Request-Timestamp"
   | Some timestamp ->
     let basestring = Printf.sprintf "%s:%s:%s" version timestamp body in
-    let expected_signature = Printf.sprintf "%s=%s" version (Common.sign_string_sha256 ~key ~basestring) in
+    let expected_signature = Printf.sprintf "%s=%s" version (Util.sign_string_sha256 ~key ~basestring) in
     if String.equal expected_signature signature then Ok () else Error "signatures don't match"

lib/state.ml

@@ -111,4 +111,4 @@ let save { state; _ } path =
   try
     Files.save_as path (fun oc -> output_string oc data);
     Ok ()
-  with exn -> fmt_error ~exn "failed to save state to file %s" path
+  with exn -> Util.fmt_error ~exn "failed to save state to file %s" path

lib/util.ml

@@ -0,0 +1,59 @@
+open Devkit
+
+let fmt_error ?exn fmt =
+  Printf.ksprintf
+    (fun s ->
+      match exn with
+      | Some exn -> Error (s ^ " :  exn " ^ Exn.str exn)
+      | None -> Error s)
+    fmt
+
+let first_line s =
+  match String.split_on_char '\n' s with
+  | x :: _ -> x
+  | [] -> s
+
+let decode_string_pad s = Stre.rstrip ~chars:"= \n\r\t" s |> Base64.decode_exn ~pad:false
+
+let http_request ?headers ?body meth path =
+  let setup h =
+    Curl.set_followlocation h true;
+    Curl.set_maxredirs h 1
+  in
+  match%lwt Web.http_request_lwt ~setup ~ua:"monorobot" ~verbose:true ?headers ?body meth path with
+  | `Ok s -> Lwt.return @@ Ok s
+  | `Error e -> Lwt.return @@ Error e
+
+let sign_string_sha256 ~key ~basestring =
+  Cstruct.of_string basestring |> Nocrypto.Hash.SHA256.hmac ~key:(Cstruct.of_string key) |> Hex.of_cstruct |> Hex.show
+
+module Build = struct
+  let buildkite_is_failed_re = Re2.create_exn {|^Build #\d+ failed|}
+
+  let is_failed_build (n : Github_t.status_notification) =
+    n.state = Failure && Re2.matches buildkite_is_failed_re (Option.default "" n.description)
+
+  let new_failed_steps (n : Github_t.status_notification) (repo_state : State_t.repo_state) pipeline =
+    let to_failed_steps branch step statuses acc =
+      (* check if step of an allowed pipeline *)
+      match step with
+      | step when step = pipeline -> acc
+      | step when not @@ Devkit.Stre.starts_with step pipeline -> acc
+      | _ ->
+      match Common.StringMap.find_opt branch statuses with
+      | Some (s : State_t.build_status) when s.status = Failure ->
+        (match s.current_failed_commit, s.original_failed_commit with
+        | Some _, _ ->
+          (* if we have a value for current_failed_commit, this step was already failed and notified *)
+          acc
+        | None, Some { build_link = Some build_link; sha; _ } when sha = n.commit.sha ->
+          (* we need to check the value of the commit sha to avoid false positives *)
+          (step, build_link) :: acc
+        | _ -> acc)
+      | _ -> acc
+    in
+    match n.state = Failure, n.branches with
+    | false, _ -> []
+    | true, [ branch ] -> Common.StringMap.fold (to_failed_steps branch.name) repo_state.pipeline_statuses []
+    | true, _ -> []
+end

mock_states/status.commit1-02-failed.json

@@ -1,15 +1,16 @@
 {
   "pipeline_statuses": {
-    "buildkite/pipeline2": {
-      "master": {
+    "buildkite/pipeline2/failed-step": {
+      "author/patches/js-storage": {
         "status": "failure",
         "original_failed_commit": {
           "sha": "7e0a933e9c71b4ca107680ca958ca1888d5e479b",
           "author": "[email protected]",
+          "url": "https://github.com/ahrefs/monorepo/commit/7e0a933e9c71b4ca107680ca958ca1888d5e479b",
           "commit_message": "c1 message",
           "build_link": [
             "Some",
-            "https://buildkite.com/ahrefs/monorepo/builds/181732"
+            "https://buildkite.com/ahrefs/monorepo/builds/181732#0192347d-e4ee-4072-9da4-f441eeb65ed4"
           ],
           "last_updated": "2024-06-02T04:57:47+00:00"
         }

mock_states/status.failure_test.json

@@ -0,0 +1,42 @@
+{
+  "pipeline_statuses": {
+    "buildkite/pipeline2": {
+      "master": {
+        "status": "failure",
+        "original_failed_commit": {
+          "sha": "0d95302addd66c1816bce1b1d495ed1c93ccd478",
+          "author": "[email protected]",
+          "url": "https://github.com/ahrefs/monorepo/commit/0d95302addd66c1816bce1b1d495ed1c93ccd478",
+          "commit_message": "Update README.md",
+          "build_link": [
+            "Some",
+            "https://buildkite.com/org/pipeline2/builds/2#0192341c-4f46-4bfc-82ab-48415b146f40"
+          ],
+          "last_updated": "2024-06-02T04:57:47+00:00"
+        }
+      }
+    },
+    "buildkite/pipeline2/failed-step": {
+      "master": {
+        "status": "failure",
+        "original_failed_commit": {
+          "sha": "0d95302addd66c1816bce1b1d495ed1c93ccd478",
+          "author": "[email protected]",
+          "url": "https://github.com/ahrefs/monorepo/commit/0d95302addd66c1816bce1b1d495ed1c93ccd478",
+          "commit_message": "Update README.md",
+          "build_link": [
+            "Some",
+            "https://buildkite.com/org/pipeline2/builds/2#0192341c-4f46-4bfc-82ab-48415b146f40"
+          ],
+          "last_updated": "2024-06-02T04:57:47+00:00"
+        }
+      }
+    }
+  },
+  "pipeline_commits": {
+    "buildkite/pipeline2": {
+      "s1": ["0d95302addd66c1816bce1b1d495ed1c93ccd478"],
+      "s2": []
+    }
+  }
+}