From 589d52fd2dfec6f3edf447bb5ce7ee05c0e4deb3 Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Fri, 13 Sep 2024 08:45:59 +0100 Subject: [PATCH 01/14] Use spdx3-validate Also print validation setup (URLs and tool versions) Signed-off-by: Arthit Suriyawongkul --- .github/workflows/validate_examples.yml | 2 +- bin/check-examples.sh | 29 +++++++++++++++++++++++-- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/.github/workflows/validate_examples.yml b/.github/workflows/validate_examples.yml index 56e611ff77..2f7407ed7b 100644 --- a/.github/workflows/validate_examples.yml +++ b/.github/workflows/validate_examples.yml @@ -15,7 +15,7 @@ jobs: - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 #v4.1.7 - name: Install Python dependencies run: | - python3 -m pip install pyshacl==0.26.0 check-jsonschema==0.29.2 + python3 -m pip install check-jsonschema==0.29.2 pyshacl==0.26.0 spdx3-validate==0.0.2 - name: Install dependencies run: | sudo apt install -y gawk diff --git a/bin/check-examples.sh b/bin/check-examples.sh index 5f74bdeef5..a270965037 100755 --- a/bin/check-examples.sh +++ b/bin/check-examples.sh @@ -13,7 +13,19 @@ SCHEMA_URL="https://spdx.org/schema/${SPDX_VERSION}/spdx-json-schema.json" RDF_URL="https://spdx.org/rdf/${SPDX_VERSION}/spdx-model.ttl" CONTEXT_URL="https://spdx.org/rdf/${SPDX_VERSION}/spdx-context.jsonld" +# print validation setup +echo "Checking examples" +echo "SPDX version: $SPDX_VERSION" +echo "Schema: $SCHEMA_URL" +echo "RDF: $RDF_URL" +echo "Context: $CONTEXT_URL" +echo "$(check-jsonschema --version)" +echo -n "$(pyshacl --version)" +echo "spdx3-validate version: $(spdx3-validate --version)" +echo "" + check_schema() { + echo "Checking schema (check-jsonschema): $1" check-jsonschema \ -v \ --schemafile $SCHEMA_URL \ @@ -21,18 +33,27 @@ check_schema() { } check_model() { + echo "Checking model (pyschacl): $1" pyshacl \ -s $RDF_URL \ -e $RDF_URL \ "$1" } +validate() { + echo "Validating (spdx3-validate): $1" + spdx3-validate -j $1 +} + # Check examples in JSON files in examples/jsonld/ if [ "$(ls $THIS_DIR/../examples/jsonld/*.json 2>/dev/null)" ]; then for f in $THIS_DIR/../examples/jsonld/*.json; do - echo "Checking $f" check_schema $f + echo "" check_model $f + echo "" + validate $f + echo "" done fi @@ -43,7 +64,7 @@ for f in $THIS_DIR/../docs/annexes/*.md; do if ! grep -q '^```json' $f; then continue fi - echo "Checking $f" + echo "Extract snippets from $f" DEST=$TEMP/$(basename $f) mkdir -p $DEST @@ -81,6 +102,7 @@ HEREDOC HEREDOC fi check_schema $doc + echo "" cat $doc >> $DEST/combined.json echo "," >> $DEST/combined.json done @@ -88,4 +110,7 @@ HEREDOC echo "{}]" >> $DEST/combined.json check_model $DEST/combined.json + echo "" + validate $DEST/combined.json + echo "" done From 4bfe5103ca26eb23523bf73f80bc5bc02bcbd089 Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Fri, 13 Sep 2024 08:52:39 +0100 Subject: [PATCH 02/14] Update example dir to trigger validation Signed-off-by: Arthit Suriyawongkul --- examples/jsonld/package_sbom.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/jsonld/package_sbom.json b/examples/jsonld/package_sbom.json index 7af7e9a68f..6f0018d3c9 100644 --- a/examples/jsonld/package_sbom.json +++ b/examples/jsonld/package_sbom.json @@ -61,7 +61,7 @@ "spdxId": "http://spdx.example.com/Package1", "creationInfo": "_:creationinfo", "name": "my-package", - "software_packageVersion": "1.0", + "software_packageVersion": "1.0.0", "software_downloadLocation": "http://dl.example.com/my-package_1.0.0.tar", "builtTime": "2024-03-06T00:00:00Z", "originatedBy": [ From bb8b348ce64dce59852d8b2db0491af5e9d71e5f Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Fri, 13 Sep 2024 09:04:04 +0100 Subject: [PATCH 03/14] Print resolved URLs Can be useful for debug during version updates Signed-off-by: Arthit Suriyawongkul --- bin/check-examples.sh | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/bin/check-examples.sh b/bin/check-examples.sh index a270965037..94c8208f1c 100755 --- a/bin/check-examples.sh +++ b/bin/check-examples.sh @@ -15,10 +15,13 @@ CONTEXT_URL="https://spdx.org/rdf/${SPDX_VERSION}/spdx-context.jsonld" # print validation setup echo "Checking examples" -echo "SPDX version: $SPDX_VERSION" -echo "Schema: $SCHEMA_URL" -echo "RDF: $RDF_URL" -echo "Context: $CONTEXT_URL" +echo "SPDX version : $SPDX_VERSION" +echo "Schema : $SCHEMA_URL" +echo "Schema resolved : $(curl -I "$SCHEMA_URL" 2>/dev/null | grep -i "location:" | awk '{print $2}')" +echo "RDF : $RDF_URL" +echo "RDF resolved : $(curl -I "$RDF_URL" 2>/dev/null | grep -i "location:" | awk '{print $2}')" +echo "Context : $CONTEXT_URL" +echo "Context resolved : $(curl -I "$CONTEXT_URL" 2>/dev/null | grep -i "location:" | awk '{print $2}')" echo "$(check-jsonschema --version)" echo -n "$(pyshacl --version)" echo "spdx3-validate version: $(spdx3-validate --version)" From 75da3656d408442311b69d8499d8cb44da7dc15f Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Sat, 14 Sep 2024 21:45:59 +0100 Subject: [PATCH 04/14] Use full option names More quick clues for code readers Signed-off-by: Arthit Suriyawongkul --- bin/check-examples.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/check-examples.sh b/bin/check-examples.sh index 94c8208f1c..04143b4fc1 100755 --- a/bin/check-examples.sh +++ b/bin/check-examples.sh @@ -30,7 +30,7 @@ echo "" check_schema() { echo "Checking schema (check-jsonschema): $1" check-jsonschema \ - -v \ + --verbose \ --schemafile $SCHEMA_URL \ "$1" } @@ -38,14 +38,14 @@ check_schema() { check_model() { echo "Checking model (pyschacl): $1" pyshacl \ - -s $RDF_URL \ - -e $RDF_URL \ + --shacl $RDF_URL \ + --ont-graph $RDF_URL \ "$1" } validate() { echo "Validating (spdx3-validate): $1" - spdx3-validate -j $1 + spdx3-validate --json $1 } # Check examples in JSON files in examples/jsonld/ From dc0b047f96d1788eaadce3ab09da9e8f58ece557 Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Mon, 7 Oct 2024 17:42:27 +0700 Subject: [PATCH 05/14] Make locations variables Signed-off-by: Arthit Suriyawongkul --- .github/workflows/validate_examples.yml | 5 ++-- bin/check-examples.sh | 35 ++++++++++++++----------- 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/.github/workflows/validate_examples.yml b/.github/workflows/validate_examples.yml index 2f7407ed7b..1161ca8453 100644 --- a/.github/workflows/validate_examples.yml +++ b/.github/workflows/validate_examples.yml @@ -1,3 +1,4 @@ +# SPDX-License-Identifier: MIT on: pull_request: paths: @@ -12,10 +13,10 @@ jobs: validate-examples: runs-on: ubuntu-latest steps: - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 #v4.1.7 + - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 #v4.2.0 - name: Install Python dependencies run: | - python3 -m pip install check-jsonschema==0.29.2 pyshacl==0.26.0 spdx3-validate==0.0.2 + python3 -m pip install check-jsonschema==0.29.3 pyshacl==0.26.0 spdx3-validate==0.0.4 - name: Install dependencies run: | sudo apt install -y gawk diff --git a/bin/check-examples.sh b/bin/check-examples.sh index 04143b4fc1..b0ace9c99c 100755 --- a/bin/check-examples.sh +++ b/bin/check-examples.sh @@ -8,13 +8,18 @@ set -e THIS_DIR="$(dirname "$0")" +MD_DIR=docs/annexes +JSON_DIR=examples/jsonld + SPDX_VERSION="3.0.1" SCHEMA_URL="https://spdx.org/schema/${SPDX_VERSION}/spdx-json-schema.json" RDF_URL="https://spdx.org/rdf/${SPDX_VERSION}/spdx-model.ttl" CONTEXT_URL="https://spdx.org/rdf/${SPDX_VERSION}/spdx-context.jsonld" # print validation setup -echo "Checking examples" +echo "Checking examples in" +echo "Snippets : $MD_DIR" +echo "Files : $JSON_DIR" echo "SPDX version : $SPDX_VERSION" echo "Schema : $SCHEMA_URL" echo "Schema resolved : $(curl -I "$SCHEMA_URL" 2>/dev/null | grep -i "location:" | awk '{print $2}')" @@ -43,27 +48,26 @@ check_model() { "$1" } -validate() { - echo "Validating (spdx3-validate): $1" +check_spdx() { + echo "SPDX 3 Validating (spdx3-validate): $1" spdx3-validate --json $1 } # Check examples in JSON files in examples/jsonld/ -if [ "$(ls $THIS_DIR/../examples/jsonld/*.json 2>/dev/null)" ]; then - for f in $THIS_DIR/../examples/jsonld/*.json; do +if [ "$(ls $THIS_DIR/../$JSON_DIR/*.json 2>/dev/null)" ]; then + for f in $THIS_DIR/../$JSON_DIR/*.json; do check_schema $f echo "" check_model $f echo "" - validate $f + check_spdx $f echo "" done fi -TEMP=$(mktemp -d) - # Check examples in inline code snippets in Markdown files in docs/annexes/ -for f in $THIS_DIR/../docs/annexes/*.md; do +TEMP=$(mktemp -d) +for f in $THIS_DIR/../$MD_DIR/*.md; do if ! grep -q '^```json' $f; then continue fi @@ -75,7 +79,8 @@ for f in $THIS_DIR/../docs/annexes/*.md; do cat $f | awk -v DEST="$DEST" 'BEGIN{flag=0} /^```json/, $0=="```" { if (/^---$/){flag++} else if ($0 !~ /^```.*/ ) print $0 > DEST "/doc-" flag ".spdx.json"}' # Combine all JSON code snippets into a single file, with SPDX context and creation info. - echo "[" > $DEST/combined.json + COMBINED_JSON = $DEST/__combined.jso + echo "[" > $COMBINED_JSON for doc in $DEST/*.spdx.json; do if ! grep -q '@context' $doc; then @@ -106,14 +111,14 @@ HEREDOC fi check_schema $doc echo "" - cat $doc >> $DEST/combined.json - echo "," >> $DEST/combined.json + cat $doc >> $COMBINED_JSON + echo "," >> $COMBINED_JSON done - echo "{}]" >> $DEST/combined.json + echo "{}]" >> $COMBINED_JSON - check_model $DEST/combined.json + check_model $COMBINED_JSON echo "" - validate $DEST/combined.json + check_spdx $COMBINED_JSON echo "" done From 0ece60c59fd59ea2398fcdc486d69e0431708ad5 Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Mon, 7 Oct 2024 18:02:55 +0700 Subject: [PATCH 06/14] Add SPDX-FileCopyrightText Signed-off-by: Arthit Suriyawongkul --- .github/workflows/validate_examples.yml | 1 - bin/check-examples.sh | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validate_examples.yml b/.github/workflows/validate_examples.yml index 1161ca8453..0ebca23423 100644 --- a/.github/workflows/validate_examples.yml +++ b/.github/workflows/validate_examples.yml @@ -1,4 +1,3 @@ -# SPDX-License-Identifier: MIT on: pull_request: paths: diff --git a/bin/check-examples.sh b/bin/check-examples.sh index b0ace9c99c..7adeb7a870 100755 --- a/bin/check-examples.sh +++ b/bin/check-examples.sh @@ -4,6 +4,7 @@ # documentation # # SPDX-License-Identifier: MIT +# SPDX-FileCopyrightText: Copyright 2024 The SPDX Contributors set -e From 7911faa19e2d8d8548e9753b4a7bacef8546fe63 Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Tue, 8 Oct 2024 03:12:35 +0700 Subject: [PATCH 07/14] Update checkout action to v4.2.1 Signed-off-by: Arthit Suriyawongkul --- .github/workflows/validate_examples.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validate_examples.yml b/.github/workflows/validate_examples.yml index 0ebca23423..f72d1a3b22 100644 --- a/.github/workflows/validate_examples.yml +++ b/.github/workflows/validate_examples.yml @@ -12,7 +12,7 @@ jobs: validate-examples: runs-on: ubuntu-latest steps: - - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 #v4.2.0 + - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 #v4.2.1 - name: Install Python dependencies run: | python3 -m pip install check-jsonschema==0.29.3 pyshacl==0.26.0 spdx3-validate==0.0.4 From 05b7251e92d14b9db2bd324265b5ad3cb6a0916b Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Fri, 25 Oct 2024 12:50:38 +0100 Subject: [PATCH 08/14] Update PySHACL Signed-off-by: Arthit Suriyawongkul --- .github/workflows/validate_examples.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validate_examples.yml b/.github/workflows/validate_examples.yml index 83e6f9cd69..f4b184ba81 100644 --- a/.github/workflows/validate_examples.yml +++ b/.github/workflows/validate_examples.yml @@ -21,7 +21,7 @@ jobs: cache: "pip" - name: Install Python dependencies run: | - python3 -m pip install check-jsonschema==0.29.4 pyshacl==0.27.0 spdx3-validate==0.0.4 + python3 -m pip install check-jsonschema==0.29.4 pyshacl==0.28.1 spdx3-validate==0.0.4 python3 -m pip install check-jsonschema==0.29.4 pyshacl==0.27.0 - name: Install dependencies run: | From fc06c37bcda4a6a1240f282f289406e750a87fe9 Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Mon, 25 Nov 2024 16:14:21 +0000 Subject: [PATCH 09/14] Update spdx3-validate to 0.0.5 Signed-off-by: Arthit Suriyawongkul --- .github/workflows/validate_examples.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validate_examples.yml b/.github/workflows/validate_examples.yml index 8a154be90f..9575dbde14 100644 --- a/.github/workflows/validate_examples.yml +++ b/.github/workflows/validate_examples.yml @@ -21,7 +21,7 @@ jobs: cache: "pip" - name: Install Python dependencies run: | - python3 -m pip install check-jsonschema==0.29.4 pyshacl==0.29.0 spdx3-validate==0.0.4 + python3 -m pip install check-jsonschema==0.29.4 pyshacl==0.29.0 spdx3-validate==0.0.5 - name: Install dependencies run: | sudo apt install -y gawk From b74252c792e6919b33b989e577a0fd4dbae9405b Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Fri, 6 Dec 2024 06:18:09 +0000 Subject: [PATCH 10/14] Skip pyshacl Temporary skip pyshacl to see the results of spdx3-validate Signed-off-by: Arthit Suriyawongkul --- .github/workflows/validate_examples.yml | 2 +- bin/check-examples.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/validate_examples.yml b/.github/workflows/validate_examples.yml index 9575dbde14..9884444ee6 100644 --- a/.github/workflows/validate_examples.yml +++ b/.github/workflows/validate_examples.yml @@ -21,7 +21,7 @@ jobs: cache: "pip" - name: Install Python dependencies run: | - python3 -m pip install check-jsonschema==0.29.4 pyshacl==0.29.0 spdx3-validate==0.0.5 + python3 -m pip install check-jsonschema==0.30.0 pyshacl==0.29.0 spdx3-validate==0.0.5 - name: Install dependencies run: | sudo apt install -y gawk diff --git a/bin/check-examples.sh b/bin/check-examples.sh index 7adeb7a870..3ce03365fe 100755 --- a/bin/check-examples.sh +++ b/bin/check-examples.sh @@ -118,8 +118,8 @@ HEREDOC echo "{}]" >> $COMBINED_JSON - check_model $COMBINED_JSON - echo "" + # check_model $COMBINED_JSON + # echo "" check_spdx $COMBINED_JSON echo "" done From 957c722bda6179f2383eb1bbaa7a275568b2e31c Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Fri, 6 Dec 2024 06:20:29 +0000 Subject: [PATCH 11/14] Skip pyshacl Signed-off-by: Arthit Suriyawongkul --- bin/check-examples.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/check-examples.sh b/bin/check-examples.sh index 3ce03365fe..361bf8fe3d 100755 --- a/bin/check-examples.sh +++ b/bin/check-examples.sh @@ -59,8 +59,8 @@ if [ "$(ls $THIS_DIR/../$JSON_DIR/*.json 2>/dev/null)" ]; then for f in $THIS_DIR/../$JSON_DIR/*.json; do check_schema $f echo "" - check_model $f - echo "" + # check_model $f + # echo "" check_spdx $f echo "" done From cd2034ce2de11186542289917e57013dea23d02e Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Fri, 6 Dec 2024 06:45:50 +0000 Subject: [PATCH 12/14] Put back pyshacl Signed-off-by: Arthit Suriyawongkul --- bin/check-examples.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/check-examples.sh b/bin/check-examples.sh index 361bf8fe3d..7adeb7a870 100755 --- a/bin/check-examples.sh +++ b/bin/check-examples.sh @@ -59,8 +59,8 @@ if [ "$(ls $THIS_DIR/../$JSON_DIR/*.json 2>/dev/null)" ]; then for f in $THIS_DIR/../$JSON_DIR/*.json; do check_schema $f echo "" - # check_model $f - # echo "" + check_model $f + echo "" check_spdx $f echo "" done @@ -118,8 +118,8 @@ HEREDOC echo "{}]" >> $COMBINED_JSON - # check_model $COMBINED_JSON - # echo "" + check_model $COMBINED_JSON + echo "" check_spdx $COMBINED_JSON echo "" done From 047e70c33b91d4dedb1560a909bab7b7af36df86 Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Sun, 5 Jan 2025 09:34:31 +0000 Subject: [PATCH 13/14] Update pyshacl==0.29.1 Signed-off-by: Arthit Suriyawongkul --- .github/workflows/validate_examples.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validate_examples.yml b/.github/workflows/validate_examples.yml index 9884444ee6..9a18c0face 100644 --- a/.github/workflows/validate_examples.yml +++ b/.github/workflows/validate_examples.yml @@ -21,7 +21,7 @@ jobs: cache: "pip" - name: Install Python dependencies run: | - python3 -m pip install check-jsonschema==0.30.0 pyshacl==0.29.0 spdx3-validate==0.0.5 + python3 -m pip install check-jsonschema==0.30.0 pyshacl==0.29.1 spdx3-validate==0.0.5 - name: Install dependencies run: | sudo apt install -y gawk From fff347be3e9b38c6b4237d4952e0a3222af34b28 Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Wed, 15 Jan 2025 00:37:42 +0000 Subject: [PATCH 14/14] Update check-jsonschema==0.31.0 Signed-off-by: Arthit Suriyawongkul --- .github/workflows/validate_examples.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validate_examples.yml b/.github/workflows/validate_examples.yml index 9a18c0face..6de5e59941 100644 --- a/.github/workflows/validate_examples.yml +++ b/.github/workflows/validate_examples.yml @@ -21,7 +21,7 @@ jobs: cache: "pip" - name: Install Python dependencies run: | - python3 -m pip install check-jsonschema==0.30.0 pyshacl==0.29.1 spdx3-validate==0.0.5 + python3 -m pip install check-jsonschema==0.31.0 pyshacl==0.29.1 spdx3-validate==0.0.5 - name: Install dependencies run: | sudo apt install -y gawk