From fdf25503d083f887470a6f47eb34fd1d71d3e9b3 Mon Sep 17 00:00:00 2001 From: Jesse McGinnis Date: Thu, 30 May 2024 08:29:29 -0400 Subject: [PATCH 1/4] Simply cue to only schema checks and cleanup makefile --- .gitignore | 5 -- Makefile | 124 +++++++++++------------------- README.md | 12 +-- dev.yml | 4 +- schema/data/attributes_schema.cue | 4 +- schema/dist/attributes_schema.cue | 27 +++++++ schema/dist/categories_schema.cue | 31 ++++++++ schema/dist/mappings_schema.cue | 25 ++++++ schema/schema.cue | 80 ------------------- 9 files changed, 138 insertions(+), 174 deletions(-) create mode 100644 schema/dist/attributes_schema.cue create mode 100644 schema/dist/categories_schema.cue create mode 100644 schema/dist/mappings_schema.cue delete mode 100644 schema/schema.cue diff --git a/.gitignore b/.gitignore index ec322ef33..b116ee014 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,3 @@ -# Ignore generated schema data files to validate against -schema/attributes_data.cue -schema/categories_data.cue -schema/mappings_data.cue - # Ignore docs site /_site /docs/_data/unstable diff --git a/Makefile b/Makefile index 9aa497455..297efbd6b 100644 --- a/Makefile +++ b/Makefile @@ -7,11 +7,11 @@ ifeq ($(VERBOSE),1) V= VPIPE= - VARG=--verbose + VERBOSE_ARG=--verbose else V=@ VPIPE= >/dev/null - VARG= + VERBOSE_ARG= endif FMT = printf "\e[%sm>> %-21s\e[0;1m →\e[1;32m %s\e[0m\n" @@ -36,21 +36,17 @@ GENERATED_DOCS_PATH = docs/_data/unstable DOCS_GENERATED_SENTINEL = tmp/.docs_generated_sentinel # DIST -GENERATED_DIST_PATH = dist +DIST_PATH = dist DIST_GENERATED_SENTINEL = tmp/.dist_generated_sentinel -CATEGORIES_JSON = $(GENERATED_DIST_PATH)/en/categories.json -ATTRIBUTES_JSON = $(GENERATED_DIST_PATH)/en/attributes.json -MAPPINGS_JSON = $(GENERATED_DIST_PATH)/en/mappings.json +TAXONOMY_JSON = $(DIST_PATH)/en/taxonomy.json +CATEGORIES_JSON = $(DIST_PATH)/en/categories.json +ATTRIBUTES_JSON = $(DIST_PATH)/en/attributes.json +MAPPINGS_JSON = $(DIST_PATH)/en/mappings.json # APP files to run application DEV_DB = storage/development.sqlite3 TEST_DB = storage/test.sqlite3 -# CUE imports needed for schema validation -ATTRIBUTES_DATA_CUE = schema/attributes_data.cue -CATEGORIES_DATA_CUE = schema/categories_data.cue -MAPPINGS_DATA_CUE = schema/mappings_data.cue - ############################################################################### # INPUTS @@ -69,57 +65,43 @@ default: build .PHONY: default build: $(DIST_GENERATED_SENTINEL) \ - $(DOCS_GENERATED_SENTINEL) \ - $(CATEGORIES_DATA_CUE) \ - $(ATTRIBUTES_DATA_CUE) \ - $(MAPPINGS_DATA_CUE) + $(DOCS_GENERATED_SENTINEL) .PHONY: build $(DOCS_GENERATED_SENTINEL): $(DEV_DB) $(CATEGORIES_DATA) $(ATTRIBUTES_DATA) $(VALUES_DATA) @$(GENERATE) "Building Docs" "$(GENERATED_DOCS_PATH)/*" - $(V)./bin/generate_docs $(VARG) + $(V)./bin/generate_docs $(VERBOSE_ARG) $(V)touch $@ $(DIST_GENERATED_SENTINEL): $(DEV_DB) $(CATEGORIES_DATA) $(ATTRIBUTES_DATA) $(VALUES_DATA) $(MAPPINGS_DATA) - @$(GENERATE) "Building Distribution" "$(GENERATED_DIST_PATH)/*.[json|txt]" - $(V)bin/generate_dist --locales $(LOCALES) $(VARG) + @$(GENERATE) "Building Distribution" "$(DIST_PATH)/*.[json|txt]" + $(V)bin/generate_dist --locales $(LOCALES) $(VERBOSE_ARG) $(V)touch $@ # # RELEASE -release: build +release: $(DIST_GENERATED_SENTINEL) @$(RUN_CMD) "Preparing release" - $(V)bin/generate_release $(VARG) + $(V)bin/generate_release $(VERBOSE_ARG) .PHONY: release # # CLEAN clean: - @$(NUKE) "Cleaning dev db" $(DEV_DB) - $(V)rm -f $(DEV_DB)* - @$(NUKE) "Cleaning test db" $(TEST_DB) - $(V)rm -f $(TEST_DB)* - @$(NUKE) "Cleaning generated docs" $(GENERATED_DOCS_PATH) - $(V)rm -f $(DOCS_GENERATED_SENTINEL) + @$(NUKE) "Cleaning sentinels" "$(DIST_GENERATED_SENTINEL) $(DOCS_GENERATED_SENTINEL)" + $(V)rm -f $(DIST_GENERATED_SENTINEL) $(DOCS_GENERATED_SENTINEL) + @$(NUKE) "Cleaning local dbs" "$(DEV_DB) $(TEST_DB)" + $(V)rm -f $(DEV_DB)* $(TEST_DB)* + @$(NUKE) "Cleaning unstable docs" $(GENERATED_DOCS_PATH) $(V)rm -rf $(GENERATED_DOCS_PATH) - @$(NUKE) "Cleaning attribute data cuefile" $(ATTRIBUTES_DATA_CUE) - $(V)rm -f $(ATTRIBUTES_DATA_CUE) - @$(NUKE) "Cleaning category data cuefile" $(CATEGORIES_DATA_CUE) - $(V)rm -f $(CATEGORIES_DATA_CUE) - @$(NUKE) "Cleaning mapping data cuefile" $(MAPPINGS_DATA_CUE) - $(V)rm -f $(MAPPINGS_DATA_CUE) - @$(NUKE) "Cleaning generated distribution files" $(GENERATED_DIST_PATH) - $(V)rm -f $(DIST_GENERATED_SENTINEL) - $(V)rm -rf $(GENERATED_DIST_PATH)/*.json - $(V)rm -rf $(GENERATED_DIST_PATH)/*.txt .PHONY: clean # # COMMANDS -docs: $(DOCS_GENERATED_SENTINEL) +run_docs: $(DOCS_GENERATED_SENTINEL) @$(RUN_CMD) "Running docs server" - $(V)bundle exec jekyll serve --source docs --destination _site $(VARG) -.PHONY: docs + $(V)bundle exec jekyll serve --source docs --destination _site $(VERBOSE_ARG) +.PHONY: run_docs console: @$(RUN_CMD) "Running console with dependencies" @@ -128,11 +110,11 @@ console: # # DATABASE SETUP -seed: vet_data_schemas +seed: vet_schema_data @$(GENERATE) "Seeding Database" $(DEV_DB) $(V)bin/rails db:drop $(V)bin/rails db:schema:load - $(V)bin/seed $(VARG) + $(V)bin/seed $(VERBOSE_ARG) .PHONY: seed $(DEV_DB): @@ -140,57 +122,39 @@ $(DEV_DB): # # TESTS -test: vet_data_schemas vet_dist_schemas - @$(RUN_CMD) "Running All Tests" - $(V)bin/rails test $(filter-out $@,$(MAKECMDGOALS)) +test: test_unit test_integration vet_schema .PHONY: test -unit_tests: +test_unit: @$(RUN_CMD) "Running Unit Tests" $(V)bin/rails unit $(filter-out $@,$(MAKECMDGOALS)) -.PHONY: unit_tests +.PHONY: test_unit -integration_tests: +test_integration: @$(RUN_CMD) "Running Integration Tests" $(V)bin/rails integration $(filter-out $@,$(MAKECMDGOALS)) -.PHONY: integration_tests - -vet_dist_schemas: $(CATEGORIES_DATA_CUE) $(ATTRIBUTES_DATA_CUE) $(MAPPINGS_DATA_CUE) - @$(RUN_CMD) "Validating data/*.json schema" - $(V)cd schema && cue vet - $(V)echo "Done!" -.PHONY: vet_dist_schemas +.PHONY: test_integration -vet_data_schemas: vet_data_attributes_schema vet_data_categories_schema vet_data_values_schema -.PHONY: vet_data_schemas +vet_schema: vet_schema_data vet_schema_dist +.PHONY: .vet_schema -vet_data_attributes_schema: +vet_schema_data: @$(RUN_CMD) "Validating $(ATTRIBUTES_DATA_PATH) schema" $(V)cue vet schema/data/attributes_schema.cue $(ATTRIBUTES_DATA_PATH) - $(V)echo "Done!" -.PHONY: vet_data_attributes_schema - -vet_data_categories_schema: @$(RUN_CMD) "Validating $(CATEGORIES_DATA_PATH) schema" $(V)cue vet schema/data/categories_schema.cue -d '#schema' $(CATEGORIES_DATA_PATH) - $(V)echo "Done!" -.PHONY: vet_data_categories_schema - -vet_data_values_schema: @$(RUN_CMD) "Validating $(VALUES_DATA_PATH) schema" $(V)cue vet schema/data/values_schema.cue -d '#schema' $(VALUES_DATA_PATH) - $(V)echo "Done!" -.PHONY: vet_data_values_schema - -# TODO: These two targets can be done together -$(CATEGORIES_DATA_CUE): $(DIST_GENERATED_SENTINEL) - @$(GENERATE) "Importing $(CATEGORIES_JSON)" $(CATEGORIES_DATA_CUE) - $(V)cue import $(CATEGORIES_JSON) -p product_taxonomy -f -o $(CATEGORIES_DATA_CUE) - -$(ATTRIBUTES_DATA_CUE): $(DIST_GENERATED_SENTINEL) - @$(GENERATE) "Importing $(ATTRIBUTES_JSON)" $(ATTRIBUTES_DATA_CUE) - $(V)cue import $(ATTRIBUTES_JSON) -p product_taxonomy -f -o $(ATTRIBUTES_DATA_CUE) - -$(MAPPINGS_DATA_CUE): $(DIST_GENERATED_SENTINEL) - @$(GENERATE) "Importing $(MAPPINGS_JSON)" $(MAPPINGS_DATA_CUE) - $(V)cue import $(MAPPINGS_JSON) -p product_taxonomy -f -o $(MAPPINGS_DATA_CUE) +.PHONY: vet_schema_data + +vet_schema_dist: + @$(RUN_CMD) "Validating $(ATTRIBUTES_JSON) schema" + $(V)cue vet schema/dist/attributes_schema.cue $(ATTRIBUTES_JSON) + @$(RUN_CMD) "Validating $(CATEGORIES_JSON) schema" + $(V)cue vet schema/dist/categories_schema.cue $(CATEGORIES_JSON) + @$(RUN_CMD) "Validating $(TAXONOMY_JSON) schema" + $(V)cue vet schema/dist/attributes_schema.cue $(TAXONOMY_JSON) + $(V)cue vet schema/dist/categories_schema.cue $(TAXONOMY_JSON) + @$(RUN_CMD) "Validating $(MAPPINGS_JSON) schema" + $(V)cue vet schema/dist/mappings_schema.cue $(MAPPINGS_JSON) +.PHONY: vet_schema_dist diff --git a/README.md b/README.md index 7775a4289..b26ff78be 100644 --- a/README.md +++ b/README.md @@ -85,12 +85,12 @@ For everyone else you'll need to: Here are the commands you'll use most often: ```sh -make [build] # build the dist and documentation files -make clean # remove sentinels and generated files -make seed # parse data/ into local db -make console # irb with dependencies loaded -make test # run ruby tests and cue schema verification -make docs # http://localhost:4000 interactive view of dist/ +make [build] # build the dist and documentation files +make clean # remove sentinels and generated files +make seed # parse data/ into local db +make console # irb with dependencies loaded +make test # run ruby tests and cue schema verification +make run_docs # http://localhost:4000 interactive view of dist/ ``` If you want to add a new distribution format, you'll need to do 3 things: diff --git a/dev.yml b/dev.yml index 06f926d4d..e4c6ed676 100644 --- a/dev.yml +++ b/dev.yml @@ -31,7 +31,7 @@ commands: run: make console release: run: make release - docs: - run: make docs + run_docs: + run: make run_docs test: run: make test diff --git a/schema/data/attributes_schema.cue b/schema/data/attributes_schema.cue index 284dc8001..659cdb5a6 100644 --- a/schema/data/attributes_schema.cue +++ b/schema/data/attributes_schema.cue @@ -1,10 +1,12 @@ +#reference_regex: "^.+__.+$" + base_attributes!: [ ...{ id!: int name!: string friendly_id!: string handle!: string - values!: [_, ...string] // at least one; all strings + values!: [_, ...string & =~#reference_regex] // at least one; all strings }, ] diff --git a/schema/dist/attributes_schema.cue b/schema/dist/attributes_schema.cue new file mode 100644 index 000000000..cfd53e9aa --- /dev/null +++ b/schema/dist/attributes_schema.cue @@ -0,0 +1,27 @@ +// This file defines and enforces the shape of the data for dist/attributes.json and dist/taxonomy.json +// Data validations are handled by the application test-suite + +#attribute_gid_regex: "^gid://shopify/TaxonomyAttribute/\\d+$" +#value_gid_regex: "^gid://shopify/TaxonomyValue/\\d+$" + +version!: string & =~"^\\d+.\\d+.\\d+$" +attributes!: [ + ...{ + id!: string & =~#attribute_gid_regex + name!: string + handle!: string + extended_attributes!: [ + ...{ + name!: string + handle!: string + }, + ] + values!: [ + ...{ + id!: string & =~#value_gid_regex + name!: string + handle!: string + }, + ] + }, +] diff --git a/schema/dist/categories_schema.cue b/schema/dist/categories_schema.cue new file mode 100644 index 000000000..9ff23846d --- /dev/null +++ b/schema/dist/categories_schema.cue @@ -0,0 +1,31 @@ +// This file defines and enforces the shape of the data for dist/categories.json and dist/taxonomy.json +// Data validations are handled by the application test-suite + +#attribute_gid_regex: "^gid://shopify/TaxonomyAttribute/\\d+$" +#category_gid_regex: "^gid://shopify/TaxonomyCategory/[a-zA-Z]{2}(-\\d+)*$" + +_category_reference: { + id!: string & =~#category_gid_regex + name!: string +} + +version!: string & =~"^\\d+.\\d+.\\d+$" +verticals!: [...{ + name!: string + prefix!: string & =~"^[a-zA-Z]{2}$" + categories!: [...{ + id!: string & =~#category_gid_regex + name!: string + level!: int & >=0 + full_name!: string + parent_id: null | string & =~#category_gid_regex + attributes!: [...{ + id!: string & =~#attribute_gid_regex + name!: string + handle!: string + extended!: bool + }] + children!: [..._category_reference] + ancestors!: [..._category_reference] + }] +}] diff --git a/schema/dist/mappings_schema.cue b/schema/dist/mappings_schema.cue new file mode 100644 index 000000000..46f5c7854 --- /dev/null +++ b/schema/dist/mappings_schema.cue @@ -0,0 +1,25 @@ +// This file defines and enforces the shape of the data for dist/mappings.json +// Data validations are handled by the application test-suite + +#attribute_gid_regex: "^gid://shopify/TaxonomyAttribute/\\d+$" +#value_gid_regex: "^gid://shopify/TaxonomyValue/\\d+$" + +version!: string & =~"^\\d+.\\d+.\\d+$" +mappings!: [ + ...{ + input_taxonomy!: string + output_taxonomy!: string + rules!: [ + ...{ + input!: { + product_category_id: string + attributes?: [...{ + name: string & =~#attribute_gid_regex + value: string & =~#value_gid_regex | null + }] + } + output!: [string]: _ + }, + ] + }, +] diff --git a/schema/schema.cue b/schema/schema.cue deleted file mode 100644 index ff144b75f..000000000 --- a/schema/schema.cue +++ /dev/null @@ -1,80 +0,0 @@ -package product_taxonomy - -#attribute_gid_regex: "^gid://shopify/TaxonomyAttribute/\\d+$" -#value_gid_regex: "^gid://shopify/TaxonomyValue/\\d+$" - -#category_gid_regex: "^gid://shopify/TaxonomyCategory/[a-zA-Z]{2}(-\\d+)*$" - -// This file defines and enforces the shape of the data for dist/attributes.json and dist/categories.json -// Data validations are handled by the application test-suite - -// Present in categories.json / attributes.json / taxonomy.json. If this is specified in multiple files this enforces their equality. -version!: string & =~"^\\d+.\\d+.\\d+$" - -// Present in attributes.json / taxonomy.json -attributes!: [ - ...{ - id!: string & =~#attribute_gid_regex - name!: string - handle!: string - extended_attributes!: [ - ...{ - name!: string - handle!: string - }, - ] - values!: [ - ...{ - id!: string & =~#value_gid_regex - name!: string - handle!: string - }, - ] - }, -] - -_category_reference: { - id!: string & =~#category_gid_regex - name!: string -} - -// Present in categories.json / taxonomy.json -verticals!: [...{ - name!: string - prefix!: string & =~"^[a-zA-Z]{2}$" - categories!: [...{ - id!: string & =~#category_gid_regex - name!: string - level!: int & >=0 - full_name!: string - parent_id: null | string & =~#category_gid_regex - attributes!: [...{ - id!: string & =~#attribute_gid_regex - name!: string - handle!: string - extended!: bool - }] - children!: [..._category_reference] - ancestors!: [..._category_reference] - }] -}] - -// Present in mappings.json / mappings_data.cue -mappings!: [ - ...{ - input_taxonomy!: string - output_taxonomy!: string - rules!: [ - ...{ - input!: { - product_category_id: string - attributes?: [...{ - name: string & =~#attribute_gid_regex - value: string & =~#value_gid_regex | null - }] - } - output!: [string]: _ - }, - ] - }, -] From f70ba87c01af213ad812a022a69c4695c47a0742 Mon Sep 17 00:00:00 2001 From: Jesse McGinnis Date: Thu, 30 May 2024 08:50:05 -0400 Subject: [PATCH 2/4] Make GitHub actions use cached bundler --- .github/workflows/deploy_github_pages.yml | 47 +++----- .github/workflows/ensure_dist_committed.yml | 23 +--- .github/workflows/generate_dist.yml | 25 ++-- .github/workflows/generate_full_dist.yml | 123 ++++++++------------ .github/workflows/tests.yml | 29 ++--- Makefile | 16 +-- bin/generate_docs | 5 +- 7 files changed, 97 insertions(+), 171 deletions(-) diff --git a/.github/workflows/deploy_github_pages.yml b/.github/workflows/deploy_github_pages.yml index 4888b504a..a6d7c4415 100644 --- a/.github/workflows/deploy_github_pages.yml +++ b/.github/workflows/deploy_github_pages.yml @@ -1,10 +1,10 @@ -name: Deploy GitHub page +name: Deploy docs site on: + workflow_dispatch: push: branches: - main - workflow_dispatch: permissions: contents: read @@ -19,25 +19,15 @@ jobs: generate_docs: runs-on: ubuntu-latest steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Ruby - uses: ruby/setup-ruby@v1 - - - name: Install Cue - uses: cue-lang/setup-cue@v1.0.0 + - uses: actions/checkout@v4 + - uses: ruby/setup-ruby@v1 + with: + bundler-cache: true + - uses: cue-lang/setup-cue@v1.0.0 with: version: 'v0.7.0' - - - name: Install bundler and gems - run: | - gem install bundler - bundle install --jobs 4 --retry 3 - - name: Generate dist & docs run: make VERBOSE=1 - - name: Upload data uses: actions/upload-artifact@v4 with: @@ -48,29 +38,19 @@ jobs: runs-on: ubuntu-latest needs: generate_docs steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Download data - uses: actions/download-artifact@v4 + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 with: name: data path: docs/_data - - - name: Setup Pages - uses: actions/configure-pages@v4 - - - name: Build with Jekyll - uses: actions/jekyll-build-pages@v1 + - uses: actions/configure-pages@v4 + - uses: actions/jekyll-build-pages@v1 with: source: docs destination: _site - - name: Copy search indexes run: sudo bin/copy_docs_search_indexes - - - name: Upload artifact - uses: actions/upload-pages-artifact@v3 + - uses: actions/upload-pages-artifact@v3 deploy: environment: @@ -79,6 +59,5 @@ jobs: runs-on: ubuntu-latest needs: build steps: - - name: Deploy to GitHub Pages + - uses: actions/deploy-pages@v4 id: deployment - uses: actions/deploy-pages@v4 diff --git a/.github/workflows/ensure_dist_committed.yml b/.github/workflows/ensure_dist_committed.yml index 206fb5469..423247d28 100644 --- a/.github/workflows/ensure_dist_committed.yml +++ b/.github/workflows/ensure_dist_committed.yml @@ -1,10 +1,10 @@ name: Ensure distribution files committed are up-to-date on: + workflow_dispatch: pull_request: branches: - main - workflow_dispatch: permissions: issues: write @@ -13,30 +13,19 @@ jobs: ensure_dist_committed: runs-on: ubuntu-latest steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Ruby - uses: ruby/setup-ruby@v1 - - - name: Install Cue - uses: cue-lang/setup-cue@v1.0.0 + - uses: actions/checkout@v4 + - uses: ruby/setup-ruby@v1 + with: + bundler-cache: true + - uses: cue-lang/setup-cue@v1.0.0 with: version: 'v0.7.0' - - - name: Install bundler and gems - run: | - gem install bundler - bundle install --jobs 4 --retry 3 - - name: Generate distribution files run: make VERBOSE=1 - - name: Ensure there are no uncommitted distrubtion file changes id: ensure-dist-committed run: | git diff --exit-code || (echo "Please commit the changes made by the build process" && exit 1) - - name: Inform PR author of failure and path to fix if: ${{ failure() && steps.ensure-dist-committed.outcome == 'failure' }} run: | diff --git a/.github/workflows/generate_dist.yml b/.github/workflows/generate_dist.yml index 2228aaa33..7f3e61bbe 100644 --- a/.github/workflows/generate_dist.yml +++ b/.github/workflows/generate_dist.yml @@ -1,9 +1,9 @@ -name: Generate and commit distribution files +name: Generate and commit distribution files to requested PR on: + workflow_dispatch: issue_comment: types: [created, edited] - workflow_dispatch: concurrency: group: "dist" @@ -17,31 +17,20 @@ jobs: runs-on: ubuntu-latest if: ${{ github.event.issue.pull_request && github.event.comment.body == '/generate_dist' }} steps: - - name: Checkout code - uses: actions/checkout@v4 - + - uses: actions/checkout@v4 - name: Checkout PR run: gh pr checkout $ISSUE env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} ISSUE: ${{ github.event.issue.html_url }} - - - name: Set up Ruby - uses: ruby/setup-ruby@v1 - - - name: Install Cue - uses: cue-lang/setup-cue@v1.0.0 + - uses: ruby/setup-ruby@v1 + with: + bundler-cache: true + - uses: cue-lang/setup-cue@v1.0.0 with: version: 'v0.7.0' - - - name: Install bundler and gems - run: | - gem install bundler - bundle install --jobs 4 --retry 3 - - name: Generate distribution files run: make VERBOSE=1 - - name: Commit distribution files run: | git config --local user.name "GitHub Action" diff --git a/.github/workflows/generate_full_dist.yml b/.github/workflows/generate_full_dist.yml index 498917253..90cf6fc79 100644 --- a/.github/workflows/generate_full_dist.yml +++ b/.github/workflows/generate_full_dist.yml @@ -1,4 +1,4 @@ -name: Generate and Commit All Distribution Files +name: Generate distribution files for all locales on: workflow_dispatch: @@ -20,15 +20,13 @@ jobs: outputs: LOCALES: ${{ steps.retrieve_locales.outputs.locales }} steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Retrieve Locales - id: retrieve_locales - run: | - locales=$(ls data/localizations/categories/ | grep -Eo '^[a-z]{2,3}(-[A-Z]{2})?' | uniq | xargs) - locales_json=$(echo -n "["; for locale in $locales; do echo -n "\"$locale\","; done | sed 's/,$//'; echo -n "]") - echo "locales=$locales_json" >> $GITHUB_OUTPUT + - uses: actions/checkout@v4 + - name: Retrieve Locales + id: retrieve_locales + run: | + locales=$(ls data/localizations/categories/ | grep -Eo '^[a-z]{2,3}(-[A-Z]{2})?' | uniq | xargs) + locales_json=$(echo -n "["; for locale in $locales; do echo -n "\"$locale\","; done | sed 's/,$//'; echo -n "]") + echo "locales=$locales_json" >> $GITHUB_OUTPUT generate_distribution_files: needs: generate_locales @@ -37,70 +35,51 @@ jobs: matrix: locale: ${{ fromJson(needs.generate_locales.outputs.LOCALES) }} steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Ruby - uses: ruby/setup-ruby@v1 - - - name: Install Cue - uses: cue-lang/setup-cue@v1.0.0 - with: - version: 'v0.7.0' - - - name: Install Bundler and Gems - run: | - gem install bundler - bundle install --jobs 4 --retry 3 - - - name: Generate Distribution Files - run: make VERBOSE=1 LOCALES=${{ matrix.locale }} - - - name: Upload distribution files - uses: actions/upload-artifact@v4 - with: - name: ${{ matrix.locale }} - path: dist/${{ matrix.locale }} + - uses: actions/checkout@v4 + - uses: ruby/setup-ruby@v1 + with: + bundler-cache: true + - uses: cue-lang/setup-cue@v1.0.0 + with: + version: 'v0.7.0' + - name: Generate local-specific distribution files + run: make VERBOSE=1 LOCALES=${{ matrix.locale }} + - uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.locale }} + path: dist/${{ matrix.locale }} create_pull_request: needs: generate_distribution_files runs-on: ubuntu-latest steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set LOCALES environment variable - run: echo "LOCALES=${{ needs.generate_locales.outputs.LOCALES }}" >> $GITHUB_ENV - - - name: Download artifacts - uses: actions/download-artifact@v4 - with: - path: dist - - name: Display structure of downloaded files - run: ls -R dist - - - name: Set up short SHA - id: short_sha - run: echo "sha=$(git rev-parse --short ${{ github.sha }})" >> $GITHUB_OUTPUT - - - name: Get last commit author username - id: last_committer - run: | - LAST_COMMIT_SHA=$(git log -1 --pretty=format:'%H') - API_URL="https://api.github.com/repos/${{ github.repository }}/commits/${LAST_COMMIT_SHA}" - AUTHOR_LOGIN=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" $API_URL | jq -r '.author.login') - echo "username=$AUTHOR_LOGIN" >> $GITHUB_OUTPUT - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Commit changes and create PR - uses: peter-evans/create-pull-request@v6 - with: - commit-message: "🤖 Update localization files for ${{ github.sha }}" - branch: "localization-dist" - branch-suffix: short-commit-hash - title: "🤖 Update localization files for ${{ steps.short_sha.outputs.sha }}" - body: | - Update `dist/` localization files. Triggered by ${{ steps.short_sha.outputs.sha }}. - cc @${{ steps.last_committer.outputs.username }} - labels: automated-pr, localization - reviewers: ${{ steps.last_committer.outputs.username }} + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + path: dist + - name: Display structure of downloaded files + run: ls -R dist + - name: Set up short SHA + id: short_sha + run: echo "sha=$(git rev-parse --short ${{ github.sha }})" >> $GITHUB_OUTPUT + - name: Get last commit author username + id: last_committer + run: | + LAST_COMMIT_SHA=$(git log -1 --pretty=format:'%H') + API_URL="https://api.github.com/repos/${{ github.repository }}/commits/${LAST_COMMIT_SHA}" + AUTHOR_LOGIN=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" $API_URL | jq -r '.author.login') + echo "username=$AUTHOR_LOGIN" >> $GITHUB_OUTPUT + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Commit changes and create PR + uses: peter-evans/create-pull-request@v6 + with: + commit-message: "🤖 Update localization files for ${{ github.sha }}" + branch: "localization-dist" + branch-suffix: short-commit-hash + title: "🤖 Update localization files for ${{ steps.short_sha.outputs.sha }}" + body: | + Update `dist/` localization files. Triggered by ${{ steps.short_sha.outputs.sha }}. + cc @${{ steps.last_committer.outputs.username }} + labels: automated-pr, localization + reviewers: ${{ steps.last_committer.outputs.username }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e3179deed..d2be34913 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,34 +1,27 @@ name: Verify output and test on: + workflow_dispatch: push: branches: - main pull_request: branches: - main - workflow_dispatch: jobs: test: + strategy: + fail-fast: true + matrix: + test: [test_unit, test_integration, vet_schema] runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v2 - - - name: Set up Ruby - uses: ruby/setup-ruby@v1 - - - name: Install bundler and gems - run: | - gem install bundler - bundle install --jobs 4 --retry 3 - - - name: Install Cue - uses: cue-lang/setup-cue@v1.0.0 + - uses: actions/checkout@v4 + - uses: ruby/setup-ruby@v1 + with: + bundler-cache: true + - uses: cue-lang/setup-cue@v1.0.0 with: version: 'v0.7.0' - - - name: Run tests - run: make test VERBOSE=1 + - run: make ${{ matrix.test }} VERBOSE=1 diff --git a/Makefile b/Makefile index 297efbd6b..3c821a71c 100644 --- a/Makefile +++ b/Makefile @@ -51,10 +51,6 @@ TEST_DB = storage/test.sqlite3 # INPUTS LOCALES ?= en -CATEGORIES_DATA = $(shell find $(CATEGORIES_DATA_PATH)) -ATTRIBUTES_DATA = $(shell find $(ATTRIBUTES_DATA_PATH)) -VALUES_DATA = $(shell find $(VALUES_DATA_PATH)) -MAPPINGS_DATA = $(shell find $(MAPPINGS_DATA_PATH)) ############################################################################### # COMMANDS @@ -68,16 +64,16 @@ build: $(DIST_GENERATED_SENTINEL) \ $(DOCS_GENERATED_SENTINEL) .PHONY: build -$(DOCS_GENERATED_SENTINEL): $(DEV_DB) $(CATEGORIES_DATA) $(ATTRIBUTES_DATA) $(VALUES_DATA) - @$(GENERATE) "Building Docs" "$(GENERATED_DOCS_PATH)/*" - $(V)./bin/generate_docs $(VERBOSE_ARG) - $(V)touch $@ - -$(DIST_GENERATED_SENTINEL): $(DEV_DB) $(CATEGORIES_DATA) $(ATTRIBUTES_DATA) $(VALUES_DATA) $(MAPPINGS_DATA) +$(DIST_GENERATED_SENTINEL): $(DEV_DB) @$(GENERATE) "Building Distribution" "$(DIST_PATH)/*.[json|txt]" $(V)bin/generate_dist --locales $(LOCALES) $(VERBOSE_ARG) $(V)touch $@ +$(DOCS_GENERATED_SENTINEL): $(DIST_GENERATED_SENTINEL) + @$(GENERATE) "Building Docs" "$(GENERATED_DOCS_PATH)/*" + $(V)./bin/generate_docs $(VERBOSE_ARG) + $(V)touch $@ + # # RELEASE release: $(DIST_GENERATED_SENTINEL) diff --git a/bin/generate_docs b/bin/generate_docs index 29c5534af..66aa452df 100755 --- a/bin/generate_docs +++ b/bin/generate_docs @@ -16,8 +16,9 @@ cli.options.force = true if cli.options.version == UNSTABLE # we want unstable t puts("Generating `#{cli.options.version}` documentation files...") cli.options_status -category_data = cli.parse_json("dist/en/categories.json").fetch("verticals") -attribute_data = cli.parse_json("dist/en/attributes.json").fetch("attributes") +taxonomy_data = cli.parse_json("dist/en/taxonomy.json") +category_data = taxonomy_data.fetch("verticals") +attribute_data = taxonomy_data.fetch("attributes") DATA_TARGET = "docs/_data/#{cli.options.version}" cli.write_file("#{DATA_TARGET}/sibling_groups.yml") do |file| From 59f1ae5e24ae5139c9b0c4ccc45415c0f8c2a33b Mon Sep 17 00:00:00 2001 From: Jesse McGinnis Date: Thu, 30 May 2024 08:53:17 -0400 Subject: [PATCH 3/4] Remove generating multiple PRs, since this is only triggered by main --- .github/workflows/generate_full_dist.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/generate_full_dist.yml b/.github/workflows/generate_full_dist.yml index 90cf6fc79..9b7f81c79 100644 --- a/.github/workflows/generate_full_dist.yml +++ b/.github/workflows/generate_full_dist.yml @@ -57,8 +57,6 @@ jobs: - uses: actions/download-artifact@v4 with: path: dist - - name: Display structure of downloaded files - run: ls -R dist - name: Set up short SHA id: short_sha run: echo "sha=$(git rev-parse --short ${{ github.sha }})" >> $GITHUB_OUTPUT @@ -76,8 +74,7 @@ jobs: with: commit-message: "🤖 Update localization files for ${{ github.sha }}" branch: "localization-dist" - branch-suffix: short-commit-hash - title: "🤖 Update localization files for ${{ steps.short_sha.outputs.sha }}" + title: "🤖 Update localization files" body: | Update `dist/` localization files. Triggered by ${{ steps.short_sha.outputs.sha }}. cc @${{ steps.last_committer.outputs.username }} From 84b20f0ba613cddad3bcafba92005640ddea4188 Mon Sep 17 00:00:00 2001 From: Jesse McGinnis Date: Thu, 30 May 2024 09:17:19 -0400 Subject: [PATCH 4/4] Rename locale distribution action --- .../{generate_full_dist.yml => generate_dist_locales.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{generate_full_dist.yml => generate_dist_locales.yml} (100%) diff --git a/.github/workflows/generate_full_dist.yml b/.github/workflows/generate_dist_locales.yml similarity index 100% rename from .github/workflows/generate_full_dist.yml rename to .github/workflows/generate_dist_locales.yml