Setup matrix workflow to spread runners (#2)

* Setup matrix workflow to spread runners * does name help? * id was wrong * update deps * remove name * incorrect name * try cat * output is no json * write in json
nflverse · Sep 6, 2024 · 039b124 · 039b124
1 parent d5d68e4
commit 039b124
Show file tree

Hide file tree

Showing 4 changed files with 101 additions and 23 deletions.
diff --git a/.github/workflows/update_ngs.yaml b/.github/workflows/update_ngs.yaml
@@ -4,22 +4,96 @@ on:
     - cron:  '0 7 * 1,2,9-12 *'
   workflow_dispatch:
     inputs:
+      season_rebuild:
+        description: 'Rebuild Season (9999 defaults to latest season)'
+        required: false
+        default: 9999
+        type: number
       full_rebuild:
-        description: 'Full Rebuild'
+        description: 'Full Rebuild (overwrites above season)'
         required: true
         default: false
         type: boolean
 
-name: update_ngs
+name: Update NGS Data
 
 jobs:
-  update:
-    name: update data
+  ngs_setup:
     runs-on: ubuntu-latest
+    name: ngs_setup
     env:
-      GITHUB_PAT: ${{ secrets.NFLVERSE_GH_TOKEN }}
       GH_TOKEN: ${{ secrets.NFLVERSE_GH_TOKEN }}
-      NFLVERSE_REBUILD: ${{ inputs.full_rebuild }}
+      FULL_REBUILD: ${{ inputs.full_rebuild || false }}
+      SEASON_REBUILD: ${{ inputs.season_rebuild || 9999 }}
+    outputs:
+      seasons: ${{ steps.query_seasons.outputs.seasons }}
+    steps:
+      - uses: r-lib/actions/setup-r@v2
+        with:
+          use-public-rspm: true
+          extra-repositories: 'https://nflverse.r-universe.dev'
+      - uses: r-lib/actions/setup-r-dependencies@v2
+        with:
+          packages: nflreadr, jsonlite
+      - id: query_seasons
+        name: Query Seasons
+        run: |
+          if [ $FULL_REBUILD == true ]
+          then
+            echo "seasons=$(Rscript -e 'jsonlite::toJSON(seq(2016, nflreadr::most_recent_season()))')" >> "$GITHUB_OUTPUT"
+          elif [ $SEASON_REBUILD == 9999 ]
+          then
+            echo "seasons=$(Rscript -e 'jsonlite::toJSON(nflreadr::most_recent_season())')" >> "$GITHUB_OUTPUT"
+          else
+            echo "seasons=${{ env.SEASON_REBUILD }}" > "$GITHUB_OUTPUT"
+          fi
+
+  update_ngs_seasons:
+    needs: ngs_setup
+    name: Update ${{ matrix.season }} ${{ matrix.type }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        season: ${{ fromJson(needs.ngs_setup.outputs.seasons) }}
+        type: ["passing", "rushing", "receiving"]
+    env:
+      GH_TOKEN: ${{ secrets.NFLVERSE_GH_TOKEN }}
+      NFLVERSE_UPDATE_SEASON: ${{ matrix.season }}
+      NFLVERSE_UPDATE_TYPE: ${{ matrix.type }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: r-lib/actions/setup-r@v2
+        with:
+          use-public-rspm: true
+          extra-repositories: 'https://nflverse.r-universe.dev'
+      - uses: r-lib/actions/setup-r-dependencies@v2
+        with:
+          packages: |
+            dplyr,
+            purrr,
+            cli,
+            httr,
+            janitor,
+            rvest,
+            rlang,
+            glue,
+            nflreadr
+          extra-packages: nflverse/nflverse-data
+      - name: Release ${{ matrix.type }}
+        run: Rscript -e 'source("R/update_ngs.R")'
+
+  combine_stats:
+    needs: [ngs_setup, update_ngs_seasons]
+    name: Build ${{ matrix.type }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        type: ["combine"]
+    env:
+      GH_TOKEN: ${{ secrets.NFLVERSE_GH_TOKEN }}
+      NFLVERSE_UPDATE_TYPE: ${{ matrix.type }}
     steps:
       - uses: actions/checkout@v4
       - uses: r-lib/actions/setup-r@v2
@@ -28,8 +102,11 @@ jobs:
           extra-repositories: 'https://nflverse.r-universe.dev'
       - uses: r-lib/actions/setup-r-dependencies@v2
         with:
-          cache-version: 1
-          extra-packages: |
-            nflverse/nflverse-data
-      - name: Run ngs update script
+          packages: |
+            cli,
+            glue,
+            nflreadr,
+            purrr
+          extra-packages: nflverse/nflverse-data
+      - name: Release ${{ matrix.type }}
         run: Rscript -e 'source("R/update_ngs.R")'
diff --git a/DESCRIPTION b/DESCRIPTION
@@ -10,16 +10,15 @@ Authors@R: c(
 Description: A collection of NGS scraping functions for nflverse
 License: GPL (>= 3)
 Imports:
-    arrow,
     dplyr,
     purrr,
-    readr,
     cli,
     httr,
     janitor,
-    callr,
     rvest,
-    qs
+    rlang,
+    glue,
+    nflreadr
 Encoding: UTF-8
 LazyData: true
 Roxygen: list(markdown = TRUE)

diff --git a/R/ngs_functions.R b/R/ngs_functions.R
@@ -79,13 +79,14 @@ load_week_ngs <- function(season, week, type, session) {
  out
 }
 
-save_ngs_data <- function(seasons) {
+save_ngs_data <- function(seasons, type = c("passing", "rushing", "receiving")) {
   most_recent <- nflreadr::most_recent_season()
+  type <- rlang::arg_match(type)
   if (!all(seasons %in% 2016:most_recent)) {
     cli::cli_abort("Please pass valid seasons between 2016 and {most_recent}")
   }
   session <- rvest::session("https://nextgenstats.nfl.com/stats/top-plays/fastest-ball-carriers")
-  todo <- expand.grid(season = seasons, type = c("passing", "rushing", "receiving"))
+  todo <- expand.grid(season = seasons, type = type)
   file_dir <- file.path(tempdir(), "ngs")
   if (!dir.exists(file_dir)) dir.create(file_dir)
   purrr::walk2(todo$season, todo$type, save_ngs_type, session, file_dir = file_dir)
@@ -97,7 +98,7 @@ save_ngs_data <- function(seasons) {
   cli::cli_alert_success("{.field DONE}")
 }
 
-save_ngs_type <- function(season, type = c("passing", "rushing", "receiving"), session, file_dir) {
+save_ngs_type <- function(season, type, session, file_dir) {
   max_week <- nflreadr::load_schedules(seasons = season) |>
     dplyr::filter(!is.na(result)) |>
     dplyr::pull(week) |>

diff --git a/R/update_ngs.R b/R/update_ngs.R
@@ -1,10 +1,11 @@
+season <- Sys.getenv("NFLVERSE_UPDATE_SEASON", unset = NA_character_) |> as.integer()
+type <- Sys.getenv("NFLVERSE_UPDATE_TYPE", unset = NA_character_)
+type <- rlang::arg_match0(type, c("passing", "rushing", "receiving", "combine"))
+
 source("R/ngs_functions.R")
 
-if(Sys.getenv("NFLVERSE_REBUILD", "false") == "true"){
-  seasons_to_update <- seq(2016, nflreadr::most_recent_season())
+if (type == "combine"){
+  purrr::walk(c("passing", "rushing", "receiving"), combine_ngs_data)
 } else {
-  seasons_to_update <- nflreadr::most_recent_season()
+  save_ngs_data(season, type)
 }
-save_ngs_data(seasons_to_update)
-
-purrr::walk(c("passing", "rushing", "receiving"), combine_ngs_data)