diff --git a/.github/workflows/kuksa_databroker-cli_build.yml b/.github/workflows/kuksa_databroker-cli_build.yml index 67aac170..ed2f73ed 100644 --- a/.github/workflows/kuksa_databroker-cli_build.yml +++ b/.github/workflows/kuksa_databroker-cli_build.yml @@ -49,6 +49,8 @@ jobs: ~/.cargo/registry/index/ ~/.cargo/registry/cache/ ~/.cargo/git/db/ + ~/.cargo/.crates.toml + ~/.cargo/.crates2.json target/ key: databroker-cli-lint-${{ hashFiles('**/Cargo.lock') }} - name: cargo fmt @@ -63,15 +65,20 @@ jobs: uses: eclipse-kuksa/kuksa-actions/.github/workflows/check_ghcr_push.yml@2 secrets: inherit - build-container: - name: Build multiarch container + build: + name: Build databroker-cli runs-on: ubuntu-latest - needs: check_ghcr_push - + env: + CARGO_TERM_COLOR: always + strategy: + matrix: + platform: + - name: amd64 + - name: arm64 + - name: riscv64 steps: - uses: actions/checkout@v4 - with: - submodules: recursive + - uses: actions/cache@v4 with: path: | @@ -79,20 +86,59 @@ jobs: ~/.cargo/registry/index/ ~/.cargo/registry/cache/ ~/.cargo/git/db/ - target/ - key: databroker-cli-release-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }} - + ~/.cargo/.crates.toml + ~/.cargo/.crates2.json + ~/.cache/pip/ + target-*/ + key: databroker-cli-release-${{ matrix.platform.name }}-${{ hashFiles('**/Cargo.lock') }} - name: Install build prerequisites working-directory: ${{github.workspace}}/ run: | - which cargo-license || cargo install cargo-license - which cross || cargo install cross - - name: Building - working-directory: ${{github.workspace}}/ + cargo install cross cargo-license cargo-cyclonedx + pip install "git+https://github.com/eclipse-kuksa/kuksa-common.git@6f3d7627760582d8ba83cc8a0f7449d00fffee84#subdirectory=sbom-tools" + - name: Build + working-directory: ${{github.workspace}} + env: + KUKSA_DATABROKERCLI_SBOM: y + run: | + ./scripts/build-databroker-cli.sh ${{ matrix.platform.name }} + + - name: "Archiving artifacts" + shell: bash + working-directory: ${{github.workspace}}/dist/${{ matrix.platform.name }} run: | - ./build-all-targets-cli.sh + tar -czf ../databroker-cli-${{ matrix.platform.name }}.tar.gz * - - name: Docker meta + - name: "Uploading artifacts" + uses: actions/upload-artifact@v4 + with: + name: databroker-cli-${{ matrix.platform.name }} + path: ${{github.workspace}}/dist/databroker-cli-${{ matrix.platform.name}}.tar.gz + if-no-files-found: error + + create-container: + name: Create multiarch container + runs-on: ubuntu-latest + + needs: [build, check_ghcr_push] + + steps: + - uses: actions/checkout@v4 + - name: Retrieve artifacts + uses: actions/download-artifact@v4 + with: + path: artifacts + pattern: databroker-cli* + merge-multiple: true + + - name: Unpack binaries + run: | + mkdir -p dist/amd64 dist/arm64 dist/riscv64 + tar xf artifacts/databroker-cli-arm64.tar.gz -C dist/arm64 + tar xf artifacts/databroker-cli-amd64.tar.gz -C dist/amd64 + tar xf artifacts/databroker-cli-riscv64.tar.gz -C dist/riscv64 + + - name: Set container metadata id: meta uses: docker/metadata-action@v5 with: @@ -107,7 +153,7 @@ jobs: type=semver,pattern={{major}}.{{minor}} type=semver,pattern={{major}} - - name: Set up Docker Buildx + - name: Setup Docker Buildx id: buildx uses: docker/setup-buildx-action@v3 @@ -128,7 +174,7 @@ jobs: linux/amd64 linux/arm64 linux/riscv64 - file: ./Dockerfile-cli + file: ./scripts/Dockerfile-cli context: . push: true tags: | @@ -148,7 +194,7 @@ jobs: linux/amd64 linux/arm64 linux/riscv64 - file: ./Dockerfile-cli + file: ./scripts/Dockerfile-cli context: . push: true tags: "ttl.sh/eclipse-kuksa/kuksa-databroker-cli-${{github.sha}}" @@ -161,40 +207,9 @@ jobs: uses: eclipse-kuksa/kuksa-actions/post-container-location@2 with: image: ttl.sh/eclipse-kuksa/kuksa-databroker-cli-${{github.sha}} - - name: Pack binaries with thirdparty - env: - AMD64_DIR: ${{ github.workspace }}/dist/amd64 - ARM64_DIR: ${{ github.workspace }}/dist/arm64 - RISCV64_DIR: ${{ github.workspace }}/dist/riscv64 - run: | - cd "$AMD64_DIR" - tar czf databroker-cli-amd64.tar.gz * - cd "$ARM64_DIR" - tar czf databroker-cli-arm64.tar.gz * - cd "$RISCV64_DIR" - tar czf databroker-cli-riscv64.tar.gz * - - - name: "Archiving AMD64 artifacts" - uses: actions/upload-artifact@v4 - with: - name: databroker-cli-amd64.tar.gz - path: ${{github.workspace}}/dist/amd64/databroker-cli-amd64.tar.gz - - - name: "Archiving ARM64 artifacts" - uses: actions/upload-artifact@v4 - with: - name: databroker-cli-arm64.tar.gz - path: ${{github.workspace}}/dist/arm64/databroker-cli-arm64.tar.gz - - - name: "Archiving RISCV64 artifacts" - uses: actions/upload-artifact@v4 - with: - name: databroker-cli-riscv64.tar.gz - path: ${{github.workspace}}/dist/riscv64/databroker-cli-riscv64.tar.gz - bom: - name: Check Bill of Material + name: License Compliance Check runs-on: ubuntu-latest steps: @@ -209,13 +224,22 @@ jobs: target/ key: databroker-cli-bom-${{ hashFiles('**/Cargo.lock') }} - - name: "Createbom: License check and Dash output generation" - working-directory: ${{github.workspace}}/createbom + # Follows the pattern from + # https://github.com/eclipse/dash-licenses?tab=readme-ov-file#example-rustcargo + - name: "Using cargo to create Dash input" + working-directory: ${{github.workspace}}/ + # target all is not really needed, and will also return i.e. wasm deps, however + # better safe than sorry, the alternative would be running this for each currently + # built target and combining the lists, but that would need adapting, when + # adding targets, or also when i.e. switching between MUSL/glibc. So this is safer run: | - which cargo-license || cargo install cargo-license - python3 createbom.py --dash ${{github.workspace}}/dash-databroker-cli ../databroker + cargo tree -e normal --prefix none --no-dedupe -p databroker-cli --target all --all-features > ${{github.workspace}}/cargodeps + cat ${{github.workspace}}/cargodeps | sort -u \ + | grep -v '^[[:space:]]*$' | grep -v kuksa | grep -v databroker \ + | sed -E 's|([^ ]+) v([^ ]+).*|crate/cratesio/-/\1/\2|' \ + > ${{github.workspace}}/dash-databrokercli-deps - name: Dash license check uses: eclipse-kuksa/kuksa-actions/check-dash@3 with: - dashinput: ${{github.workspace}}/dash-databroker-cli + dashinput: ${{github.workspace}}/dash-databrokercli-deps diff --git a/.github/workflows/kuksa_databroker_build.yml b/.github/workflows/kuksa_databroker_build.yml index d1e8c564..9d0dd872 100644 --- a/.github/workflows/kuksa_databroker_build.yml +++ b/.github/workflows/kuksa_databroker_build.yml @@ -44,6 +44,8 @@ jobs: ~/.cargo/registry/index/ ~/.cargo/registry/cache/ ~/.cargo/git/db/ + ~/.cargo/.crates.toml + ~/.cargo/.crates2.json target/ key: databroker-lint-${{ hashFiles('**/Cargo.lock') }} - name: Show toolchain information @@ -75,7 +77,10 @@ jobs: ~/.cargo/registry/index/ ~/.cargo/registry/cache/ ~/.cargo/git/db/ - target/ + ~/.cargo/.crates.toml + ~/.cargo/.crates2.json + ~/.cache/pip/ + target-*/ key: databroker-coverage-${{ hashFiles('**/Cargo.lock') }} - name: Install cargo-llvm-cov uses: taiki-e/install-action@cargo-llvm-cov @@ -91,28 +96,17 @@ jobs: build: name: Build runs-on: ubuntu-latest - needs: [bom] env: CARGO_TERM_COLOR: always strategy: matrix: platform: - name: amd64 - target: x86_64-unknown-linux-musl - name: arm64 - target: aarch64-unknown-linux-musl - name: riscv64 - target: riscv64gc-unknown-linux-gnu steps: - uses: actions/checkout@v4 - - name: Retrieve artifacts - uses: actions/download-artifact@v4 - with: - path: artifacts - pattern: Third* - merge-multiple: true - - name: Display structure of downloaded files - run: ls -R + - uses: actions/cache@v4 with: path: | @@ -120,32 +114,35 @@ jobs: ~/.cargo/registry/index/ ~/.cargo/registry/cache/ ~/.cargo/git/db/ + ~/.cargo/.crates.toml + ~/.cargo/.crates2.json target/ key: databroker-release-${{ matrix.platform.name }}-${{ hashFiles('**/Cargo.lock') }} - name: Install build prerequisites working-directory: ${{github.workspace}}/ run: | - which cross || cargo install cross + cargo install cross cargo-license cargo-cyclonedx + pip install "git+https://github.com/eclipse-kuksa/kuksa-common.git@6f3d7627760582d8ba83cc8a0f7449d00fffee84#subdirectory=sbom-tools" - name: Build - working-directory: ${{github.workspace}}/ + working-directory: ${{github.workspace}} + env: + KUKSA_DATABROKER_FEATURES: databroker/viss,databroker/tls + KUKSA_DATABROKER_SBOM: y run: | - cross build --target ${{ matrix.platform.target }} --features viss --bin databroker --release - mkdir -p "dist" - cp "target/${{ matrix.platform.target }}/release/databroker" "dist" - - name: Package dist files + ./scripts/build-databroker.sh ${{ matrix.platform.name }} + + - name: "Archiving artifacts" shell: bash - working-directory: ${{github.workspace}} + working-directory: ${{github.workspace}}/dist/${{ matrix.platform.name }} run: | - cd dist - tar xf ../artifacts/thirdparty.tar.gz - tar -czf databroker-${{ matrix.platform.name }}.tar.gz * + tar -czf ../databroker-${{ matrix.platform.name }}.tar.gz * - - name: Upload artifacts + - name: "Uploading artifacts" uses: actions/upload-artifact@v4 with: - name: databroker-${{ matrix.platform.name }}.tar.gz - path: dist/databroker-${{ matrix.platform.name }}.tar.gz - if-no-files-found: error + name: databroker-${{ matrix.platform.name }} + path: ${{github.workspace}}/dist/databroker-${{ matrix.platform.name}}.tar.gz + if-no-files-found: error check_ghcr_push: name: Check access rights @@ -156,7 +153,7 @@ jobs: name: Create multiarch container runs-on: ubuntu-latest - needs: [build, bom, check_ghcr_push] + needs: [build, check_ghcr_push] steps: - uses: actions/checkout@v4 @@ -164,26 +161,16 @@ jobs: uses: actions/download-artifact@v4 with: path: artifacts + pattern: databroker-* merge-multiple: true - name: Unpack binaries - env: - AMD64_DIR: ${{ github.workspace }}/target/x86_64-unknown-linux-musl/release - ARM64_DIR: ${{ github.workspace }}/target/aarch64-unknown-linux-musl/release - RISCV64_DIR: ${{ github.workspace }}/target/riscv64gc-unknown-linux-gnu/release run: | - cd artifacts - tar xf databroker-amd64.tar.gz - mkdir -p "$AMD64_DIR" - mv databroker "$AMD64_DIR" - tar xf databroker-arm64.tar.gz - mkdir -p "$ARM64_DIR" - mv databroker "$ARM64_DIR" - tar xf databroker-riscv64.tar.gz - mkdir -p "$RISCV64_DIR" - mv databroker "$RISCV64_DIR" - tar xf thirdparty.tar.gz - mv thirdparty ../databroker/ + mkdir -p dist/amd64 dist/arm64 dist/riscv64 + tar xf artifacts/databroker-arm64.tar.gz -C dist/arm64 + tar xf artifacts/databroker-amd64.tar.gz -C dist/amd64 + tar xf artifacts/databroker-riscv64.tar.gz -C dist/riscv64 + - name: Set container metadata id: meta uses: docker/metadata-action@v5 @@ -220,7 +207,7 @@ jobs: linux/amd64 linux/arm64 linux/riscv64 - file: ./Dockerfile + file: ./scripts/Dockerfile context: . push: true tags: | @@ -240,7 +227,7 @@ jobs: linux/amd64 linux/arm64 linux/riscv64 - file: ./Dockerfile + file: ./scripts/Dockerfile context: . push: true tags: "ttl.sh/eclipse-kuksa/kuksa-databroker-${{github.sha}}" @@ -258,6 +245,9 @@ jobs: name: Run integration test runs-on: ubuntu-latest needs: [create-container] + strategy: + matrix: + platform: ["arm64", "amd64", "riscv64"] steps: - name: Set up QEMU @@ -265,30 +255,17 @@ jobs: - uses: actions/checkout@v4 - - name: Run integration test on AMD64 container + - name: Run integration test on ${{ matrix.platform }} container env: DATABROKER_IMAGE: ttl.sh/eclipse-kuksa/kuksa-databroker-${{github.sha}} - CONTAINER_PLATFORM: linux/amd64 - run: | - ${{github.workspace}}/integration_test/run.sh - - - name: Run integration test on ARM64 container - env: - DATABROKER_IMAGE: ttl.sh/eclipse-kuksa/kuksa-databroker-${{github.sha}} - CONTAINER_PLATFORM: linux/arm64 - run: | - ${{github.workspace}}/integration_test/run.sh - - - name: Run integration test on RISCV64 container - env: - DATABROKER_IMAGE: ttl.sh/eclipse-kuksa/kuksa-databroker-${{github.sha}} - CONTAINER_PLATFORM: linux/riscv64 + CONTAINER_PLATFORM: linux/${{ matrix.platform }} run: | ${{github.workspace}}/integration_test/run.sh bom: - name: Check Bill of Material + name: License Compliance Check runs-on: ubuntu-latest + steps: - uses: actions/checkout@v4 - uses: actions/cache@v4 @@ -301,28 +278,22 @@ jobs: target/ key: databroker-bom-${{ hashFiles('**/Cargo.lock') }} - - name: Install prerequisites - working-directory: ${{github.workspace}}/createbom - run: | - which cargo-license || cargo install cargo-license - - name: License check and Dash output generation - working-directory: ${{github.workspace}}/createbom + # Follows the pattern from + # https://github.com/eclipse/dash-licenses?tab=readme-ov-file#example-rustcargo + - name: "Using cargo to create Dash input" + working-directory: ${{github.workspace}}/ + # target all is not really needed, and will also return i.e. wasm deps, however + # better safe than sorry, the alternative would be running this for each currently + # built target and combining the lists, but that would need adapting, when + # adding targets, or also when i.e. switching between MUSL/glibc. So this is safer run: | - python3 createbom.py --dash ${{github.workspace}}/dash-databroker ../databroker + cargo tree -e normal --prefix none --no-dedupe -p databroker --target all --all-features > ${{github.workspace}}/cargodeps + cat ${{github.workspace}}/cargodeps | sort -u \ + | grep -v '^[[:space:]]*$' | grep -v kuksa | grep -v databroker \ + | sed -E 's|([^ ]+) v([^ ]+).*|crate/cratesio/-/\1/\2|' \ + > ${{github.workspace}}/dash-databroker-deps + - name: Dash license check uses: eclipse-kuksa/kuksa-actions/check-dash@3 with: - dashinput: ${{github.workspace}}/dash-databroker - - name: Generate Bill of Materials - working-directory: ${{github.workspace}}/createbom - run: | - rm -r ../databroker/thirdparty - python3 createbom.py ../databroker - cd ../databroker - tar czf thirdparty.tar.gz thirdparty - - name: Upload Bill of Materials - uses: actions/upload-artifact@v4 - with: - name: Third party licenses - path: databroker/thirdparty.tar.gz - if-no-files-found: error + dashinput: ${{github.workspace}}/dash-databroker-deps diff --git a/.gitignore b/.gitignore index d866f6da..d22c11d0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ target/ +target-*/ dist/ __pycache__ databroker/thirdparty diff --git a/DEPENDENCIES b/DEPENDENCIES deleted file mode 100644 index 06df9e6f..00000000 --- a/DEPENDENCIES +++ /dev/null @@ -1,134 +0,0 @@ -crate/cratesio/-/aho-corasick/0.7.20, MIT AND Unlicense, approved, #4240 -crate/cratesio/-/ansi_term/0.12.1, MIT, approved, clearlydefined -crate/cratesio/-/anyhow/1.0.68, MIT OR Apache-2.0, approved, clearlydefined -crate/cratesio/-/arrayref/0.3.6, BSD-2-Clause, approved, clearlydefined -crate/cratesio/-/arrayvec/0.5.2, Apache-2.0 AND MIT, approved, clearlydefined -crate/cratesio/-/async-stream-impl/0.3.3, MIT, approved, clearlydefined -crate/cratesio/-/async-stream/0.3.3, MIT, approved, clearlydefined -crate/cratesio/-/async-trait/0.1.63, Apache-2.0 AND MIT AND Apache-2.0 AND MIT, approved, #6666 -crate/cratesio/-/base64/0.13.1, Apache-2.0 AND MIT AND Apache-2.0 AND MIT, approved, #6655 -crate/cratesio/-/bitflags/1.3.2, MIT OR Apache-2.0, approved, clearlydefined -crate/cratesio/-/blake2b_simd/0.5.11, MIT, approved, clearlydefined -crate/cratesio/-/bytes/1.3.0, MIT, approved, clearlydefined -crate/cratesio/-/cfg-if/1.0.0, Apache-2.0 AND MIT, approved, clearlydefined -crate/cratesio/-/clap/3.2.23, Apache-2.0 AND MIT, approved, #4243 -crate/cratesio/-/clap_lex/0.2.4, Apache-2.0 AND MIT AND Apache-2.0 AND MIT, approved, #4254 -crate/cratesio/-/constant_time_eq/0.1.5, CC0-1.0, approved, clearlydefined -crate/cratesio/-/crossbeam-utils/0.8.14, MIT OR Apache-2.0, approved, clearlydefined -crate/cratesio/-/dirs-sys/0.3.7, MIT OR Apache-2.0, approved, clearlydefined -crate/cratesio/-/dirs/1.0.5, Apache-2.0 AND MIT, approved, clearlydefined -crate/cratesio/-/dirs/4.0.0, MIT OR Apache-2.0, approved, clearlydefined -crate/cratesio/-/either/1.8.0, Apache-2.0 AND MIT AND Apache-2.0 AND MIT, approved, #4245 -crate/cratesio/-/fnv/1.0.7, Apache-2.0 AND MIT, approved, clearlydefined -crate/cratesio/-/futures-channel/0.3.25, Apache-2.0 AND MIT AND Apache-2.0 AND MIT AND BSD-2-Clause-Views, approved, #6671 -crate/cratesio/-/futures-core/0.3.25, MIT OR Apache-2.0, approved, clearlydefined -crate/cratesio/-/futures-sink/0.3.25, MIT OR Apache-2.0, approved, clearlydefined -crate/cratesio/-/futures-task/0.3.25, MIT OR Apache-2.0, approved, clearlydefined -crate/cratesio/-/futures-util/0.3.25, MIT OR Apache-2.0, approved, clearlydefined -crate/cratesio/-/getrandom/0.1.16, Apache-2.0 AND MIT, approved, clearlydefined -crate/cratesio/-/getrandom/0.2.8, MIT OR Apache-2.0, approved, clearlydefined -crate/cratesio/-/h2/0.3.15, MIT, approved, clearlydefined -crate/cratesio/-/hashbrown/0.12.3, MIT OR Apache-2.0, approved, clearlydefined -crate/cratesio/-/hermit-abi/0.2.6, MIT OR Apache-2.0, approved, clearlydefined -crate/cratesio/-/http-body/0.4.5, MIT, approved, clearlydefined -crate/cratesio/-/http/0.2.8, MIT OR Apache-2.0, approved, clearlydefined -crate/cratesio/-/httparse/1.8.0, Apache-2.0 AND MIT AND Apache-2.0 AND MIT, approved, #4256 -crate/cratesio/-/httpdate/1.0.2, MIT OR Apache-2.0, approved, clearlydefined -crate/cratesio/-/hyper-timeout/0.4.1, Apache-2.0 AND MIT, approved, clearlydefined -crate/cratesio/-/hyper/0.14.23, MIT, approved, clearlydefined -crate/cratesio/-/indexmap/1.9.2, Apache-2.0 OR MIT, approved, clearlydefined -crate/cratesio/-/itertools/0.10.5, Apache-2.0 AND MIT AND Apache-2.0 AND MIT, approved, #4247 -crate/cratesio/-/itoa/1.0.5, MIT OR Apache-2.0, approved, clearlydefined -crate/cratesio/-/lazy_static/1.4.0, Apache-2.0 AND MIT, approved, clearlydefined -crate/cratesio/-/libc/0.2.139, MIT OR Apache-2.0, approved, clearlydefined -crate/cratesio/-/linefeed/0.6.0, MIT OR Apache-2.0, approved, clearlydefined -crate/cratesio/-/log/0.4.17, MIT OR Apache-2.0, approved, clearlydefined -crate/cratesio/-/matchers/0.1.0, MIT, approved, clearlydefined -crate/cratesio/-/memchr/2.5.0, Unlicense OR MIT, approved, clearlydefined -crate/cratesio/-/memoffset/0.6.5, MIT, approved, clearlydefined -crate/cratesio/-/mio/0.8.5, MIT, approved, clearlydefined -crate/cratesio/-/mortal/0.2.3, MIT OR Apache-2.0, approved, clearlydefined -crate/cratesio/-/nix/0.23.2, MIT, approved, clearlydefined -crate/cratesio/-/nom/5.1.2, MIT, approved, clearlydefined -crate/cratesio/-/nu-ansi-term/0.46.0, MIT, approved, clearlydefined -crate/cratesio/-/num_cpus/1.15.0, MIT OR Apache-2.0, approved, clearlydefined -crate/cratesio/-/once_cell/1.17.0, MIT OR Apache-2.0, approved, clearlydefined -crate/cratesio/-/os_str_bytes/6.4.1, MIT OR Apache-2.0, approved, clearlydefined -crate/cratesio/-/overload/0.1.1, MIT, approved, clearlydefined -crate/cratesio/-/percent-encoding/2.2.0, MIT OR Apache-2.0, approved, clearlydefined -crate/cratesio/-/phf/0.11.1, MIT, approved, clearlydefined -crate/cratesio/-/phf_shared/0.11.1, MIT, approved, clearlydefined -crate/cratesio/-/pin-project-internal/1.0.12, Apache-2.0 OR MIT, approved, clearlydefined -crate/cratesio/-/pin-project-lite/0.2.9, Apache-2.0 OR MIT, approved, clearlydefined -crate/cratesio/-/pin-project/1.0.12, Apache-2.0 OR MIT, approved, clearlydefined -crate/cratesio/-/pin-utils/0.1.0, Apache-2.0 AND MIT, approved, clearlydefined -crate/cratesio/-/ppv-lite86/0.2.17, MIT OR Apache-2.0, approved, clearlydefined -crate/cratesio/-/proc-macro2/1.0.50, MIT OR Apache-2.0, approved, clearlydefined -crate/cratesio/-/prost-derive/0.9.0, Apache-2.0, approved, clearlydefined -crate/cratesio/-/prost-types/0.9.0, Apache-2.0, approved, clearlydefined -crate/cratesio/-/prost/0.9.0, Apache-2.0, approved, clearlydefined -crate/cratesio/-/quote/1.0.23, MIT OR Apache-2.0, approved, clearlydefined -crate/cratesio/-/rand/0.8.5, MIT OR Apache-2.0, approved, clearlydefined -crate/cratesio/-/rand_chacha/0.3.1, MIT OR Apache-2.0, approved, clearlydefined -crate/cratesio/-/rand_core/0.6.4, MIT OR Apache-2.0, approved, clearlydefined -crate/cratesio/-/redox_syscall/0.1.57, MIT, approved, clearlydefined -crate/cratesio/-/redox_syscall/0.2.16, MIT, approved, clearlydefined -crate/cratesio/-/redox_users/0.3.5, MIT, approved, clearlydefined -crate/cratesio/-/redox_users/0.4.3, MIT, approved, clearlydefined -crate/cratesio/-/regex-automata/0.1.10, MIT OR (MIT AND Unlicense), approved, clearlydefined -crate/cratesio/-/regex-syntax/0.6.28, Apache-2.0 AND MIT AND Apache-2.0 AND MIT AND Unicode-DFS-2016, approved, #4252 -crate/cratesio/-/regex/1.7.1, MIT OR Apache-2.0, approved, clearlydefined -crate/cratesio/-/rust-argon2/0.8.3, MIT OR Apache-2.0, approved, clearlydefined -crate/cratesio/-/ryu/1.0.12, Apache-2.0 AND BSL-1.0 AND CC-BY-SA-3.0, approved, #4267 -crate/cratesio/-/serde/1.0.152, MIT OR Apache-2.0, approved, clearlydefined -crate/cratesio/-/serde_derive/1.0.152, MIT OR Apache-2.0, approved, clearlydefined -crate/cratesio/-/serde_json/1.0.91, Apache-2.0 AND MIT, approved, #4264 -crate/cratesio/-/sharded-slab/0.1.4, MIT, approved, clearlydefined -crate/cratesio/-/signal-hook-registry/1.4.0, Apache-2.0 OR MIT, approved, clearlydefined -crate/cratesio/-/siphasher/0.3.10, Apache-2.0 AND MIT, approved, #6665 -crate/cratesio/-/slab/0.4.7, MIT, approved, clearlydefined -crate/cratesio/-/smallstr/0.2.0, MIT OR Apache-2.0, approved, clearlydefined -crate/cratesio/-/smallvec/1.10.0, MIT OR Apache-2.0, approved, clearlydefined -crate/cratesio/-/socket2/0.4.7, MIT OR Apache-2.0, approved, clearlydefined -crate/cratesio/-/sqlparser/0.16.0, Apache-2.0, approved, #6669 -crate/cratesio/-/syn/1.0.107, MIT OR Apache-2.0, approved, clearlydefined -crate/cratesio/-/terminfo/0.7.5, WTFPL AND X11-distribute-modifications-variant, approved, #6663 -crate/cratesio/-/textwrap/0.16.0, MIT, approved, #6657 -crate/cratesio/-/thiserror-impl/1.0.38, MIT OR Apache-2.0, approved, clearlydefined -crate/cratesio/-/thiserror/1.0.38, MIT OR Apache-2.0, approved, clearlydefined -crate/cratesio/-/thread_local/1.1.4, Apache-2.0 OR MIT, approved, clearlydefined -crate/cratesio/-/tinyvec/1.6.0, Zlib OR (Apache-2.0 OR MIT), approved, clearlydefined -crate/cratesio/-/tinyvec_macros/0.1.0, MIT OR (Apache-2.0 AND MIT) OR (MIT AND Zlib), approved, clearlydefined -crate/cratesio/-/tokio-io-timeout/1.2.0, MIT OR Apache-2.0, approved, clearlydefined -crate/cratesio/-/tokio-macros/1.8.2, MIT, approved, clearlydefined -crate/cratesio/-/tokio-stream/0.1.11, MIT, approved, clearlydefined -crate/cratesio/-/tokio-util/0.6.10, MIT, approved, clearlydefined -crate/cratesio/-/tokio-util/0.7.4, MIT, approved, clearlydefined -crate/cratesio/-/tokio/1.24.2, MIT, approved, #6659 -crate/cratesio/-/tonic/0.6.2, MIT AND Apache-2.0, approved, #6653 -crate/cratesio/-/tower-layer/0.3.2, MIT, approved, clearlydefined -crate/cratesio/-/tower-service/0.3.2, MIT, approved, clearlydefined -crate/cratesio/-/tower/0.4.13, MIT AND Apache-2.0, approved, #6661 -crate/cratesio/-/tracing-attributes/0.1.23, MIT, approved, clearlydefined -crate/cratesio/-/tracing-core/0.1.30, MIT, approved, clearlydefined -crate/cratesio/-/tracing-futures/0.2.5, MIT, approved, clearlydefined -crate/cratesio/-/tracing-subscriber/0.3.16, MIT AND BSD-3-Clause AND BSD-2-Clause AND LicenseRef-Public-Domain, approved, #6670 -crate/cratesio/-/tracing/0.1.37, MIT, approved, clearlydefined -crate/cratesio/-/try-lock/0.2.4, MIT, approved, clearlydefined -crate/cratesio/-/unicode-ident/1.0.6, , approved, #4138 -crate/cratesio/-/unicode-normalization/0.1.22, MIT OR Apache-2.0, approved, clearlydefined -crate/cratesio/-/unicode-width/0.1.10, MIT OR Apache-2.0, approved, clearlydefined -crate/cratesio/-/want/0.3.0, MIT, approved, clearlydefined -crate/cratesio/-/wasi/0.11.0+wasi-snapshot-preview1, (Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT), approved, #6667 -crate/cratesio/-/wasi/0.9.0+wasi-snapshot-preview1, (Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT), approved, #6654 -crate/cratesio/-/winapi-i686-pc-windows-gnu/0.4.0, Apache-2.0 AND MIT, approved, #6664 -crate/cratesio/-/winapi-x86_64-pc-windows-gnu/0.4.0, Apache-2.0 AND MIT, approved, #6658 -crate/cratesio/-/winapi/0.3.9, Apache-2.0 AND MIT, approved, clearlydefined -crate/cratesio/-/windows-sys/0.42.0, MIT OR Apache-2.0, approved, clearlydefined -crate/cratesio/-/windows_aarch64_gnullvm/0.42.1, MIT OR Apache-2.0, approved, clearlydefined -crate/cratesio/-/windows_aarch64_msvc/0.42.1, MIT OR Apache-2.0, approved, clearlydefined -crate/cratesio/-/windows_i686_gnu/0.42.1, MIT OR Apache-2.0, approved, clearlydefined -crate/cratesio/-/windows_i686_msvc/0.42.1, MIT OR Apache-2.0, approved, clearlydefined -crate/cratesio/-/windows_x86_64_gnu/0.42.1, MIT OR Apache-2.0, approved, clearlydefined -crate/cratesio/-/windows_x86_64_gnullvm/0.42.1, MIT OR Apache-2.0, approved, clearlydefined -crate/cratesio/-/windows_x86_64_msvc/0.42.1, MIT OR Apache-2.0, approved, clearlydefined diff --git a/build-all-targets-cli.sh b/build-all-targets-cli.sh deleted file mode 100755 index 784baaf5..00000000 --- a/build-all-targets-cli.sh +++ /dev/null @@ -1,76 +0,0 @@ -#!/bin/bash -# -# Copyright (c) 2023 Contributors to the Eclipse Foundation -# -# Building all currently supported targets for databroker-cli. -# Uses cross for cross-compiling. Needs to be executed -# before docker build, as docker collects the artifacts -# created by this script -# this needs the have cross, cargo-license and createbom dependencies installed -# -# SPDX-License-Identifier: Apache-2.0 - -# exit on error, to not waste any time -set -e - -SCRIPT_PATH=$(realpath "$0") -SCRIPT_DIR=$(dirname "$SCRIPT_PATH") - -CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse - -# We need to clean this folder in target, otherwise we get weird side -# effects building for different archs, complaining libc crate can not find -# GLIBC, i.e -# Compiling libc v0.2.149 -#error: failed to run custom build command for `libc v0.2.149` -# -#Caused by: -# process didn't exit successfully: `/target/release/build/libc-2dd22ab6b5fb9fd2/#build-script-build` (exit status: 1) -# --- stderr -# /target/release/build/libc-2dd22ab6b5fb9fd2/build-script-build: /lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.29' not found (required by /target/release/build/libc-2dd22ab6b5fb9fd2/build-script-build) -# -# It seems cross/cargo is reusing something from previous builds it shouldn't. -# the finished artifact resides in ../target/x86_64-unknown-linux-musl/release -# so deleting the temporary files in target/releae is no problem -cleanup_target_release_dir() { - echo "Clean up target dir..." - rm -rf "$SCRIPT_DIR/target/release" -} - -# Create thirdparty bom -rm -rf "$SCRIPT_DIR/databroker/thirdparty" || true -pushd createbom/ -python3 createbom.py ../databroker-cli -popd - -# Building AMD46 -echo "Building AMD64" -cleanup_target_release_dir -cross build --target x86_64-unknown-linux-musl --bin databroker-cli --release - -# Building ARM64 -echo "Building ARM64" -cleanup_target_release_dir -cross build --target aarch64-unknown-linux-musl --bin databroker-cli --release - -# Build RISCV64, this is a glibc based build, as musl is not -# yet supported -echo "Building RISCV64" -cleanup_target_release_dir -cross build --target riscv64gc-unknown-linux-gnu --bin databroker-cli --release - -# Prepare dist folders -echo "Prepare amd64 dist folder" -mkdir -p "$SCRIPT_DIR/dist/amd64" -cp "$SCRIPT_DIR/target/x86_64-unknown-linux-musl/release/databroker-cli" "$SCRIPT_DIR/dist/amd64" -cp -r "$SCRIPT_DIR/databroker-cli/thirdparty" "$SCRIPT_DIR/dist/amd64" - -echo "Prepare arm64 dist folder" -mkdir -p "$SCRIPT_DIR/dist/arm64" -cp "$SCRIPT_DIR/target/aarch64-unknown-linux-musl/release/databroker-cli" "$SCRIPT_DIR/dist/arm64" -cp -r "$SCRIPT_DIR/databroker-cli/thirdparty" "$SCRIPT_DIR/dist/arm64" - -echo "Prepare riscv64 dist folder" -mkdir -p "$SCRIPT_DIR/dist/riscv64" -cp "$SCRIPT_DIR/target/riscv64gc-unknown-linux-gnu/release/databroker-cli" "$SCRIPT_DIR/dist/riscv64" -cp -r "$SCRIPT_DIR/databroker-cli/thirdparty" "$SCRIPT_DIR/dist/riscv64" diff --git a/build-all-targets.sh b/build-all-targets.sh deleted file mode 100755 index e602ddd6..00000000 --- a/build-all-targets.sh +++ /dev/null @@ -1,85 +0,0 @@ -#!/bin/bash -# -# Copyright (c) 2023 Contributors to the Eclipse Foundation -# -# Building all currently supported targets. -# Uses cross for cross-compiling. Needs to be executed -# before docker build, as docker collects the artifacts -# created by this script -# this needs the have cross, cargo-license and createbom dependencies installed -# -# SPDX-License-Identifier: Apache-2.0 - -# exit on error, to not waste any time -set -e - -SCRIPT_PATH=$(realpath "$0") -SCRIPT_DIR=$(dirname "$SCRIPT_PATH") - -CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse - -# Check if a certai feature set was requested -if [ -z "$KUKSA_DATABROKER_FEATURES" ]; then - # If not set, assign a default value - KUKSA_DATABROKER_FEATURES="databroker/default" -fi - -echo "Building with features: $KUKSA_DATABROKER_FEATURES" - - -# We need to clean this folder in target, otherwise we get weird side -# effects building for different archs, complaining libc crate can not find -# GLIBC, i.e -# Compiling libc v0.2.149 -#error: failed to run custom build command for `libc v0.2.149` -# -#Caused by: -# process didn't exit successfully: `/target/release/build/libc-2dd22ab6b5fb9fd2/#build-script-build` (exit status: 1) -# --- stderr -# /target/release/build/libc-2dd22ab6b5fb9fd2/build-script-build: /lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.29' not found (required by /target/release/build/libc-2dd22ab6b5fb9fd2/build-script-build) -# -# It seems cross/cargo is reusing something from previous builds it shouldn't. -# the finished artifact resides in ../target/x86_64-unknown-linux-musl/release -# so deleting the temporary files in target/releae is no problem -cleanup_target_release_dir() { - echo "Clean up target dir..." - rm -rf "$SCRIPT_DIR/target/release" -} - -# Create thirdparty bom -rm -rf "$SCRIPT_DIR/databroker/thirdparty" || true -pushd createbom/ -python3 createbom.py ../databroker -popd - -# Building AMD46 -echo "Building AMD64" -cleanup_target_release_dir -cross build --target x86_64-unknown-linux-musl --features $KUKSA_DATABROKER_FEATURES --bin databroker --release - -# Building ARM64 -echo "Building ARM64" -cleanup_target_release_dir -cross build --target aarch64-unknown-linux-musl --features $KUKSA_DATABROKER_FEATURES --bin databroker --release - -# Build RISCV64, this is a glibc based build, as musl is not -# yet supported -echo "Building RISCV64" -cleanup_target_release_dir -cross build --target riscv64gc-unknown-linux-gnu --features $KUKSA_DATABROKER_FEATURES --bin databroker --release - -# Prepare dist folders -echo "Prepare amd64 dist folder" -mkdir -p "$SCRIPT_DIR/dist/amd64" -cp "$SCRIPT_DIR/target/x86_64-unknown-linux-musl/release/databroker" "$SCRIPT_DIR/dist/amd64" -cp -r "$SCRIPT_DIR/databroker/thirdparty" "$SCRIPT_DIR/dist/amd64" - -echo "Prepare arm64 dist folder" -mkdir -p "$SCRIPT_DIR/dist/arm64" -cp "$SCRIPT_DIR/target/aarch64-unknown-linux-musl/release/databroker" "$SCRIPT_DIR/dist/arm64" -cp -r "$SCRIPT_DIR/databroker/thirdparty" "$SCRIPT_DIR/dist/arm64" - -echo "Prepare riscv64 dist folder" -mkdir -p "$SCRIPT_DIR/dist/riscv64" -cp "$SCRIPT_DIR/target/riscv64gc-unknown-linux-gnu/release/databroker" "$SCRIPT_DIR/dist/riscv64" -cp -r "$SCRIPT_DIR/databroker/thirdparty" "$SCRIPT_DIR/dist/riscv64" diff --git a/createbom/README.md b/createbom/README.md deleted file mode 100644 index 874585ac..00000000 --- a/createbom/README.md +++ /dev/null @@ -1,20 +0,0 @@ -# BOM Generator - -Generates a BOM - -## Troubleshooting - -If you run it and you get errors like: - -``` -Could not find license file for 0BSD in adler -Error: BOM creation failed, unresolved licenses detected -``` - -The a possible reason is that the `Cargo.lock` in the repository has been updated and some components use licenses -not covered here. This can be solved by: - -* Find the corresponding license test. Check for instance [SPDX](https://spdx.org/licenses/) -* Verify that license is feasible for our use. -* Download or create a text file with the license text, do `gzip` and put it in `licensestore` folder. -* Add the identifier (in the example above `0BSD`) to `maplicensefile.py` diff --git a/createbom/bomutil/maplicensefile.py b/createbom/bomutil/maplicensefile.py deleted file mode 100644 index 044f587d..00000000 --- a/createbom/bomutil/maplicensefile.py +++ /dev/null @@ -1,40 +0,0 @@ -#! /usr/bin/env python -######################################################################## -# Copyright (c) 2022, 2023 Robert Bosch GmbH -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SPDX-License-Identifier: Apache-2.0 -######################################################################## - -"""Mapping of license identifiers of cargo license to the filenames of the actual license texts.""" - -MAP = { - "Apache-2.0": "Apache-2.0.txt.gz", - "BlueOak-1.0.0": "BlueOak-1.0.0.md.gz", - "MIT": "MIT.txt.gz", - "Unlicense": "Unlicense.txt.gz", - "BSL-1.0": "BSL-1.0.txt.gz", - "Unicode-DFS-2016": "Unicode-DFS-2016.txt.gz", - "BSD-2-Clause": "BSD-2-Clause.txt.gz", - "CC0-1.0": "CC0-1.0.txt.gz", - "WTFPL": "WTFPL.txt.gz", - "Zlib": "Zlib.txt.gz", - "ISC": "ISC.txt.gz", - "ring": "ring.LICENSE.txt.gz", - "rustls-webpki": "webpki.LICENSE.txt.gz", - # License text taken from https://spdx.org/licenses/0BSD.html - "0BSD": "0BSD.txt.gz", - # License test taken from https://spdx.org/licenses/BSD-3-Clause.html - "BSD-3-Clause": "BSD-3-Clause.txt.gz" -} diff --git a/createbom/bomutil/quirks.py b/createbom/bomutil/quirks.py deleted file mode 100644 index 70df6fa2..00000000 --- a/createbom/bomutil/quirks.py +++ /dev/null @@ -1,36 +0,0 @@ -#! /usr/bin/env python -######################################################################## -# Copyright (c) 2022, 2023 Robert Bosch GmbH -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SPDX-License-Identifier: Apache-2.0 -######################################################################## - -'''Hook for applying some sanitation to make further processing easier''' - -def apply_quirks(component): - ''' - Takes one component entry from cargo license and might return - a modified/extended entry. - Use sparingly. Comment what you are doing - Use narrow matching (name and complete license string) to catch - changes - ''' - if component["name"] in {"io-lifetimes", "linux-raw-sys", "rustix", "wasi"} \ - and component["license"] == "Apache-2.0 OR Apache-2.0 WITH LLVM-exception OR MIT": - # All licenses are "OR", we already ship Apache-2.0 and MIT. The LLVM exception - # does not apply to us, so lets keep it clean. - component["license"] = "Apache-2.0 OR MIT" - return component - return component diff --git a/createbom/createbom.py b/createbom/createbom.py deleted file mode 100755 index 4785195f..00000000 --- a/createbom/createbom.py +++ /dev/null @@ -1,201 +0,0 @@ -#!/usr/bin/env python3 -######################################################################## -# Copyright (c) 2022,2023 Robert Bosch GmbH -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SPDX-License-Identifier: Apache-2.0 -######################################################################## - -""" -This script will generate a list of all dependencies and licenses of a -Rust project. It will create a folder called thirdparty in that -project folder containing a list of dependencies and a copy -of each license used in dependencies -""" - -import argparse -import sys -import json -import re -import os -import gzip - -from subprocess import check_output, CalledProcessError - -from bomutil.maplicensefile import MAP as supported_licenses -from bomutil import quirks - - -class LicenseException(Exception): - pass - - -class RunCargoException(Exception): - pass - - -def extract_license_ids(license): - """Extract valid licenses for each dependency. We most of the time - do not care whether it is "AND" or "OR" currently, we currently assume we are - compatible to all "OR" variants, and thus include all""" - license_ids = [] - if license: - license_ids = re.split(r"\s*AND\s*|\s*OR\s*|\(|\)", license) - license_ids = list(filter(None, license_ids)) - - return license_ids - - -def extract_license_filenames(crate): - license_files = [] - crate = quirks.apply_quirks(crate) - - crate_name = crate["name"] - license_ids = extract_license_ids(crate["license"]) - license_file = crate["license_file"] - - if not license_ids and not license_file: - raise LicenseException( - f"Neither license nor license file specified for {crate_name}" - ) - - if license_file: - license_ids.append(crate_name) - - missing = [] - for license_id in license_ids: - if license_id in supported_licenses: - license_file = supported_licenses[license_id] - license_files.append(license_file) - else: - missing.append(license_id) - - if missing: - missing_licenses = ", ".join(missing) - raise LicenseException( - f"Could not find license file for {missing_licenses} in {crate_name}" - ) - - return license_files - - -def generate_bom(source_path, target_path, dashout): - try: - cargo_output = check_output( - [ - "cargo", - "license", - "--json", - "--avoid-build-deps", - "--current-dir", - source_path, - ] - ) - except CalledProcessError as e: - raise RunCargoException(f"Error running cargo license: {e}") - - crates = json.loads(cargo_output) - dashlist = [] - - # Cargo will also pick up our own dependencies. As they are not thirdparty - # creating a new list without our own packages - crates = [ - crate for crate in crates if not crate["name"].startswith("databroker") - and not crate["name"].startswith("kuksa") - ] - - license_files = set() - errors = [] - for crate in crates: - try: - print(f"License for {crate['name']} {crate['version']}: ", end="") - license_filenames = extract_license_filenames(crate) - for license_filename in license_filenames: - license_files.add(license_filename) - unpacked_filenames = [ - filename[:-3] if filename.endswith(".gz") else filename - for filename in license_filenames - ] - print(" ".join(unpacked_filenames)) - del crate["license_file"] - crate["license_files"] = unpacked_filenames - dashlist.append( - f"crate/cratesio/-/{crate['name']}/{crate['version']}") - except LicenseException as e: - errors.append(e) - - if errors: - for error in errors: - print(error) - raise LicenseException( - "BOM creation failed, unresolved licenses detected") - - # Exporting - os.mkdir(target_path) - - for license_file in license_files: - print(f"Copying {license_file[:-2]}") - with gzip.open("licensestore/" + license_file, "rb") as inf: - content = inf.read() - with open(os.path.join(target_path, license_file[:-3]), "wb") as outf: - outf.write(content) - - print("Writing thirdparty_components.txt") - with open( - os.path.join(target_path, "thirdparty_components.txt"), "w", encoding="utf-8" - ) as jsonout: - json.dump(crates, jsonout, indent=4) - - if dashout is not None: - print(f"Exporting dash output to {dashout}") - with open(dashout, 'w') as f: - for line in dashlist: - f.write(f"{line}\n") - - -def main(args=None): - parser = argparse.ArgumentParser() - parser.add_argument("dir", help="Rust project directory") - parser.add_argument("--dash", default=None, type=str, - help="if present, write an input file for dash PATH", metavar="PATH") - args = parser.parse_args(args) - - source_path = os.path.abspath(args.dir) - target_path = os.path.join(source_path, "thirdparty") - - if os.path.exists(target_path): - print( - f"Folder {target_path} already exists. Remove it before running this script." - ) - return -2 - - if args.dash is not None and os.path.exists(args.dash): - print( - f"Requested Dash output file {args.dash} exists. Remove it before running this script.") - return -3 - - print(f"Generating BOM for project in {source_path}") - try: - generate_bom(source_path, target_path, args.dash) - except LicenseException as e: - print(f"Error: {e}") - return -100 - except RunCargoException as e: - print(f"Error: {e}") - return -1 - - -if __name__ == "__main__": - - sys.exit(main(sys.argv[1:])) diff --git a/createbom/licensestore/0BSD.txt.gz b/createbom/licensestore/0BSD.txt.gz deleted file mode 100644 index d6b2a6dc..00000000 Binary files a/createbom/licensestore/0BSD.txt.gz and /dev/null differ diff --git a/createbom/licensestore/Apache-2.0.txt.gz b/createbom/licensestore/Apache-2.0.txt.gz deleted file mode 100644 index 43305c2e..00000000 Binary files a/createbom/licensestore/Apache-2.0.txt.gz and /dev/null differ diff --git a/createbom/licensestore/BSD-2-Clause.txt.gz b/createbom/licensestore/BSD-2-Clause.txt.gz deleted file mode 100644 index 55fd2bdc..00000000 Binary files a/createbom/licensestore/BSD-2-Clause.txt.gz and /dev/null differ diff --git a/createbom/licensestore/BSD-3-Clause.txt.gz b/createbom/licensestore/BSD-3-Clause.txt.gz deleted file mode 100644 index eb4e7256..00000000 Binary files a/createbom/licensestore/BSD-3-Clause.txt.gz and /dev/null differ diff --git a/createbom/licensestore/BSL-1.0.txt.gz b/createbom/licensestore/BSL-1.0.txt.gz deleted file mode 100644 index b19cc1c5..00000000 Binary files a/createbom/licensestore/BSL-1.0.txt.gz and /dev/null differ diff --git a/createbom/licensestore/BlueOak-1.0.0.md.gz b/createbom/licensestore/BlueOak-1.0.0.md.gz deleted file mode 100644 index 377124c2..00000000 Binary files a/createbom/licensestore/BlueOak-1.0.0.md.gz and /dev/null differ diff --git a/createbom/licensestore/CC0-1.0.txt.gz b/createbom/licensestore/CC0-1.0.txt.gz deleted file mode 100644 index 579708c8..00000000 Binary files a/createbom/licensestore/CC0-1.0.txt.gz and /dev/null differ diff --git a/createbom/licensestore/ISC.txt.gz b/createbom/licensestore/ISC.txt.gz deleted file mode 100644 index b14b8468..00000000 Binary files a/createbom/licensestore/ISC.txt.gz and /dev/null differ diff --git a/createbom/licensestore/MIT.txt.gz b/createbom/licensestore/MIT.txt.gz deleted file mode 100644 index 37744a62..00000000 Binary files a/createbom/licensestore/MIT.txt.gz and /dev/null differ diff --git a/createbom/licensestore/Unicode-DFS-2016.txt.gz b/createbom/licensestore/Unicode-DFS-2016.txt.gz deleted file mode 100644 index 7a014010..00000000 Binary files a/createbom/licensestore/Unicode-DFS-2016.txt.gz and /dev/null differ diff --git a/createbom/licensestore/Unlicense.txt.gz b/createbom/licensestore/Unlicense.txt.gz deleted file mode 100644 index 2d5f62c1..00000000 Binary files a/createbom/licensestore/Unlicense.txt.gz and /dev/null differ diff --git a/createbom/licensestore/WTFPL.txt.gz b/createbom/licensestore/WTFPL.txt.gz deleted file mode 100644 index 9e57dae5..00000000 Binary files a/createbom/licensestore/WTFPL.txt.gz and /dev/null differ diff --git a/createbom/licensestore/Zlib.txt.gz b/createbom/licensestore/Zlib.txt.gz deleted file mode 100644 index f9c19c9e..00000000 Binary files a/createbom/licensestore/Zlib.txt.gz and /dev/null differ diff --git a/createbom/licensestore/ring.LICENSE.txt.gz b/createbom/licensestore/ring.LICENSE.txt.gz deleted file mode 100644 index 651640e6..00000000 Binary files a/createbom/licensestore/ring.LICENSE.txt.gz and /dev/null differ diff --git a/createbom/licensestore/webpki.LICENSE.txt.gz b/createbom/licensestore/webpki.LICENSE.txt.gz deleted file mode 100644 index 5da686a8..00000000 Binary files a/createbom/licensestore/webpki.LICENSE.txt.gz and /dev/null differ diff --git a/Dockerfile b/scripts/Dockerfile similarity index 71% rename from Dockerfile rename to scripts/Dockerfile index bc405613..83272581 100644 --- a/Dockerfile +++ b/scripts/Dockerfile @@ -11,9 +11,12 @@ # * SPDX-License-Identifier: Apache-2.0 # ********************************************************************************/ -# This is expected to be executed in the kuksa.val top-level directory -# You need to run build-all-targets.sh first, as this docker file jsut -# collects the artifacts +# You need to run build-databroker.sh first, for all platforms you want to build +# the container for as this docker file just collects the artifacts, i.e. +# $ KUKSA_DATABROKER_SBOM=y ./scripts/build-databroker.sh arm64 amd64 riscv64 +# +# This Dockerfile is intended to run inside the root directory, i.e. +# $ docker buildx build --platform linux/amd64 -f scripts/Dockerfile . # Different targets need different base images, so prepare aliases here @@ -21,24 +24,19 @@ # AMD is a statically linked MUSL build FROM scratch AS target-amd64 ENV BUILDTARGET="x86_64-unknown-linux-musl" -COPY ./target/x86_64-unknown-linux-musl/release/databroker /app/databroker - # ARM64 is a statically linked GRPC build FROM scratch AS target-arm64 ENV BUILDTARGET="aarch64-unknown-linux-musl" -COPY ./target/aarch64-unknown-linux-musl/release/databroker /app/databroker # RISCV is a glibc build. Rust toolchain not supported for MUSL # Normally we prefer "distroless" base images, i.e.: # FROM gcr.io/distroless/base-debian12:debug as target-riscv64 -# However, distorless has no RISCV support yet, +# However, distroless has no RISCV support yet, # (Nov 2023). Using debian unstable for now FROM riscv64/debian:sid-slim as target-riscv64 - ENV BUILDTARGET="riscv64gc-unknown-linux-gnu" -COPY ./target/riscv64gc-unknown-linux-gnu/release/databroker /app/databroker # Now adding generic parts FROM target-$TARGETARCH as target @@ -46,8 +44,11 @@ ARG TARGETARCH # Before running this file thirdparty files must have been created -# by build-all-targets.sh or corresponding command in buildaction -COPY ./databroker/thirdparty/ /app/thirdparty +# by build-databroker.sh or corresponding command in buildaction +COPY ./dist/$TARGETARCH/databroker /app/databroker +COPY ./dist/$TARGETARCH/sbom.json /app/sbom.json +COPY ./dist/$TARGETARCH/thirdparty-licenses/ /app/thirdparty-licenses + COPY ./data/vss-core/vss_release_3.1.1.json vss_release_3.1.1.json COPY ./data/vss-core/vss_release_4.0.json vss_release_4.0.json diff --git a/Dockerfile-cli b/scripts/Dockerfile-cli similarity index 75% rename from Dockerfile-cli rename to scripts/Dockerfile-cli index 8b5b3919..53e5eb3c 100644 --- a/Dockerfile-cli +++ b/scripts/Dockerfile-cli @@ -11,21 +11,20 @@ # * SPDX-License-Identifier: Apache-2.0 # ********************************************************************************/ -# This is expected to be executed in the kuksa.val top-level directory -# You need to run build-all-targets-cli.sh first, as this docker file jsut -# collects the artifacts +# You need to run build-databroker-cli.sh first, for all platforms you want to build +# the container for as this docker file just collects the artifacts, i.e. +# $ KUKSA_DATABROKERCLI_SBOM=y ./scripts/build-databroker-cli.sh arm64 amd64 riscv64 +# +# This Dockerfile is intended to run inside the root directory, i.e. +# $ docker buildx build --platform linux/amd64 -f scripts/Dockerfile-cli . # AMD is a statically linked MUSL build FROM scratch AS target-amd64 ENV BUILDTARGET="x86_64-unknown-linux-musl" -COPY ./target/x86_64-unknown-linux-musl/release/databroker-cli /app/databroker-cli - # ARM64 is a statically linked GRPC build FROM scratch AS target-arm64 ENV BUILDTARGET="aarch64-unknown-linux-musl" -COPY ./target/aarch64-unknown-linux-musl/release/databroker-cli /app/databroker-cli - # RISCV is a glibc build. Rust toolchain not supported for MUSL # Normally we prefer "distroless" base images, i.e.: @@ -34,7 +33,6 @@ COPY ./target/aarch64-unknown-linux-musl/release/databroker-cli /app/databroker- # (Nov 2023). Using debian unstable for now FROM riscv64/debian:sid-slim as target-riscv64 ENV BUILDTARGET="riscv64gc-unknown-linux-gnu" -COPY ./target/riscv64gc-unknown-linux-gnu/release/databroker-cli /app/databroker-cli # Databroker-cli is an interactive cli, thus it can only work correctly # if we have some terminfo configurations available. We will transplant @@ -50,7 +48,9 @@ RUN apk update && apk add ncurses-terminfo-base FROM target-$TARGETARCH as target ARG TARGETARCH -COPY ./dist/$TARGETARCH/thirdparty/ /app/thirdparty +COPY ./dist/$TARGETARCH/databroker-cli /app/databroker-cli +COPY ./dist/$TARGETARCH/sbom.json /app/sbom.json +COPY ./dist/$TARGETARCH/thirdparty-licenses/ /app/thirdparty-licenses # Copy terminfo database COPY --from=terminfo-donor /etc/terminfo /etc/terminfo diff --git a/scripts/build-databroker-cli.sh b/scripts/build-databroker-cli.sh new file mode 100755 index 00000000..e9e09c97 --- /dev/null +++ b/scripts/build-databroker-cli.sh @@ -0,0 +1,149 @@ +#!/bin/bash +# +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# Building all currently supported targets for databroker-cli. +# Uses cross for cross-compiling. Needs to be executed +# before docker build, as docker collects the artifacts +# created by this script +# this needs the have cross, cargo-license and kuksa sbom helper +# installed +# +# SPDX-License-Identifier: Apache-2.0 + +# This script will build databroker-cli for different architectures uing cross +# (https://github.com/cross-rs/cross) +# Artifacts will be out in the directory ./dist in the form required for the +# Dockerfile-cli +# +# You run it like +# +# ./build-databroker-cli.sh plattforms +# +# where platform can be one or more of +# +# arm64, amd64, riscv64, i.e. the following are valid commandlines +# +# ./build-databroker-cli.sh amd64 +# ./build-databroker-cli.sh amd64 arm64 riscv64 +# +# you can enable features that will be passed to cargo +# by setting the environment variable KUKSA_DATABROKERCLI_FEATURES +# +# If you want generate an SBOM and assemble a list of licenses set +# KUKSA_DATABROKERCLI_SBOM to "y(es)" or "true", +# +# KUKSA_DATABROKERCLI_SBOM=y +# +# This will generate a Cyclone DX SBOM and collect license. For +# this to work it expects cargo-cyclonedx to be installed and it +# requires the collectlicensefiles from +# https://github.com/eclipse-kuksa/kuksa-common/tree/main/sbom-tools +# to be available +# + +# exit on error, to not waste any time +set -e + +SCRIPT_PATH=$(realpath "$0") +SCRIPT_DIR=$(dirname "$SCRIPT_PATH") + +cd ${SCRIPT_DIR}/.. + +# need a key value matching but no bash 4 an macOS +# so this nice hack works on bash 3 as well +tmprefix=$(basename -- "$0") +TARGET_MAP=$(mktemp -dt ${tmprefix}XXXXX) +echo >${TARGET_MAP}/arm64 aarch64-unknown-linux-musl +echo >${TARGET_MAP}/amd64 x86_64-unknown-linux-musl +# RISCV64 is a glibc based build, as musl is not +# yet supported +echo >${TARGET_MAP}/riscv64 riscv64gc-unknown-linux-gnu + +CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse + + +# Check if a certain feature set was requested +if [ -z "$KUKSA_DATABROKERCLI_FEATURES" ]; then + # If not set, assign a default value + KUKSA_DATABROKERCLI_FEATURES="databroker-cli/default" +fi + +SBOM=0 +# Check whether to build SBOM +if [ ! -z "$KUKSA_DATABROKERCLI_SBOM" ]; then + # If set, check whether it is "y" + if [[ $KUKSA_DATABROKERCLI_SBOM =~ ^[Yy](es)?$ || $KUKSA_DATABROKER_SBOM =~ ^[Tt](rue)?$ ]]; then + SBOM=1 + fi +fi + +if [[ $SBOM -eq 1 ]]; then + echo "Will create SBOM" +else + echo "Will not create SBOM" +fi + +echo "Building with features: $KUKSA_DATABROKERCLI_FEATURES" + + +# Builds for a given target and collects data to be distirbuted in docker. Needs +# Rust target triplett (i.e. x86_64-unknown-linux-musl) and the corresponding docker +# architecture (i.e. amd64) as input +function build_target() { + target_rust=$1 + target_docker=$2 + + # Need to set different target dir for different platforms, becasue cargo mixes things up + # when recycling the default target dir. When you do not do this, and e.g. first build amd64 + # followed by riscv64 you will get effects like + # Compiling libc v0.2.149 + #error: failed to run custom build command for `libc v0.2.149` + # + #Caused by: + # process didn't exit successfully: `/target/release/build/libc-2dd22ab6b5fb9fd2/#build-script-build` (exit status: 1) + # --- stderr + # /target/release/build/libc-2dd22ab6b5fb9fd2/build-script-build: /lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.29' not found (required by /target/release/build/libc-2dd22ab6b5fb9fd2/build-script-build) + # + # this is solved by using different target-dirs for each platform + echo "Building databroker-cli for target $target_rust" + cross build --target $target_rust --target-dir ./target-$target_docker --features $KUKSA_DATABROKERCLI_FEATURES --bin databroker-cli --release + + echo "Prepare $target_docker dist folder" + rm -rf ./dist/$target_docker || true + mkdir ./dist/$target_docker + cp ./target-$target_docker/$target_rust/release/databroker-cli ./dist/$target_docker + + if [[ $SBOM -eq 1 ]]; then + echo "Create $target_rust SBOM" + cargo cyclonedx -v -f json --describe binaries --spec-version 1.4 --target $target_rust --manifest-path ./Cargo.toml + cp ./databroker-cli/databroker-cli_bin.cdx.json ./dist/$target_docker/sbom.json + rm -rf ./dist/$target_docker/thirdparty-licenses || true + collectlicensefiles ./databroker-cli/databroker-cli_bin.cdx.json ./dist/$target_docker/thirdparty-licenses --curation ./scripts/licensecuration.yaml + fi +} + + +# Check valid platforms +for platform in "$@" +do + if [ ! -f ${TARGET_MAP}/$platform ]; then + echo "Invalid platform \"$platform\"" + echo "Supported platforms:" + echo "$(ls ${TARGET_MAP})" + rm -rf ${TARGET_MAP} + exit 1 + fi +done + + +mkdir -p ./dist + +for platform in "$@" +do + target=$(cat ${TARGET_MAP}/$platform) + build_target $target $platform +done + +rm -rf ${TARGET_MAP} +echo "All done." diff --git a/scripts/build-databroker.sh b/scripts/build-databroker.sh new file mode 100755 index 00000000..7567d913 --- /dev/null +++ b/scripts/build-databroker.sh @@ -0,0 +1,155 @@ +#!/bin/bash +# +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# Building all currently supported targets. +# Uses cross for cross-compiling. Needs to be executed +# before docker build, as docker collects the artifacts +# created by this script +# this needs the have cross, cargo-license and the kuksa-sbom helper +# installed +# +# SPDX-License-Identifier: Apache-2.0 + + +# This script will build databroker for different architectures uing cross +# (https://github.com/cross-rs/cross) +# Artifacts will be out in the directory ./dist in the form required for the +# Dockerfile +# +# You run it like +# +# ./build-databroker.sh plattforms +# +# where platform can be one or more of +# +# arm64, amd64, riscv64, i.e. the following are valid commandlines +# +# ./build-databroker.sh amd64 +# ./build-databroker.sh amd64 arm64 riscv64 +# +# you can enable features that will be passed to cargo +# by setting the environment variable KUKSA_DATABROKER_FEATURES, i.e. +# +# KUKSA_DATABROKER_FEATURES=databroker/viss,databroker/tls +# +# If you want generate an SBOM and assemble a list of licenses set +# KUKSA_DATABROKER_SBOM to "y(es)" or "true", +# +# KUKSA_DATABROKER_SBOM=y +# +# This will generate a Cyclone DX SBOM and collect license. For +# this to work it expects cargo-cyclonedx to be installed and it +# requires the collectlicensefiles from +# https://github.com/eclipse-kuksa/kuksa-common/tree/main/sbom-tools +# to be available +# + + +# exit on error, to not waste any time +set -e + +SCRIPT_PATH=$(realpath "$0") +SCRIPT_DIR=$(dirname "$SCRIPT_PATH") + +cd ${SCRIPT_DIR}/.. + +# need a key value matching but no bash 4 an macOS +# so this nice hack works on bash 3 as well +tmprefix=$(basename -- "$0") +TARGET_MAP=$(mktemp -dt ${tmprefix}XXXXX) +echo >${TARGET_MAP}/arm64 aarch64-unknown-linux-musl +echo >${TARGET_MAP}/amd64 x86_64-unknown-linux-musl +# RISCV64 is a glibc based build, as musl is not +# yet supported +echo >${TARGET_MAP}/riscv64 riscv64gc-unknown-linux-gnu + +CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse + +# Check if a certain feature set was requested +if [ -z "$KUKSA_DATABROKER_FEATURES" ]; then + # If not set, assign a default value + KUKSA_DATABROKER_FEATURES="databroker/default" +fi + +SBOM=0 +# Check whether to build SBOM +if [ ! -z "$KUKSA_DATABROKER_SBOM" ]; then + # If set, check whether it is "y" + if [[ $KUKSA_DATABROKER_SBOM =~ ^[Yy](es)?$ || $KUKSA_DATABROKER_SBOM =~ ^[Tt](rue)?$ ]]; then + SBOM=1 + fi +fi + +if [[ $SBOM -eq 1 ]]; then + echo "Will create SBOM" +else + echo "Will not create SBOM" +fi + +echo "Building with features: $KUKSA_DATABROKER_FEATURES" + + + + +# Builds for a given target and collects data to be distirbuted in docker. Needs +# Rust target triplett (i.e. x86_64-unknown-linux-musl) and the corresponding docker +# architecture (i.e. amd64) as input +function build_target() { + target_rust=$1 + target_docker=$2 + + # Need to set different target dir for different platforms, becasue cargo mixes things up + # when recycling the default target dir. When you do not do this, and e.g. first build amd64 + # followed by riscv64 you will get effects like + # Compiling libc v0.2.149 + #error: failed to run custom build command for `libc v0.2.149` + # + #Caused by: + # process didn't exit successfully: `/target/release/build/libc-2dd22ab6b5fb9fd2/#build-script-build` (exit status: 1) + # --- stderr + # /target/release/build/libc-2dd22ab6b5fb9fd2/build-script-build: /lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.29' not found (required by /target/release/build/libc-2dd22ab6b5fb9fd2/build-script-build) + # + # this is solved by using different target-dirs for each platform + echo "Building databroker for target $target_rust" + cross build --target $target_rust --target-dir ./target-$target_docker --features $KUKSA_DATABROKER_FEATURES --bin databroker --release + + echo "Prepare $target_docker dist folder" + rm -rf ./dist/$target_docker || true + mkdir ./dist/$target_docker + cp ./target-$target_docker/$target_rust/release/databroker ./dist/$target_docker + + if [[ $SBOM -eq 1 ]]; then + echo "Create $target_rust SBOM" + cargo cyclonedx -v -f json --describe binaries --spec-version 1.4 --target $target_rust --manifest-path ./Cargo.toml + cp ./databroker/databroker_bin.cdx.json ./dist/$target_docker/sbom.json + rm -rf ./dist/$target_docker/thirdparty-licenses || true + collectlicensefiles ./databroker/databroker_bin.cdx.json ./dist/$target_docker/thirdparty-licenses --curation ./scripts/licensecuration.yaml + fi +} + + +# Check valid platforms +for platform in "$@" +do + if [ ! -f ${TARGET_MAP}/$platform ]; then + echo "Invalid platform \"$platform\"" + echo "Supported platforms:" + echo "$(ls ${TARGET_MAP})" + rm -rf ${TARGET_MAP} + exit 1 + fi +done + + + +mkdir -p ./dist + +for platform in "$@" +do + target=$(cat ${TARGET_MAP}/$platform) + build_target $target $platform +done + +rm -rf ${TARGET_MAP} +echo "All done." diff --git a/scripts/licensecuration.yaml b/scripts/licensecuration.yaml new file mode 100644 index 00000000..3a5d87cc --- /dev/null +++ b/scripts/licensecuration.yaml @@ -0,0 +1,22 @@ + +# Curation file for thirdparty licenses +# Will be used by collectlicensefromcyclonedx tool +# to inform it about choosing the correct license +# artifacts to be stored in deliverables + +components: + # ring is based on the standard OpenSSL and BoringSSL + # projects, thus licensing of the codebase is complex depending + # on parts. This makes sure that except the explanatory text + # in ring (and thus in the generated CycloneDX file) are copied + # as well as the vanilla text of the ISC and OPENSSL license + # OpenSSL seems to be in the process to move to Apache-2.0, but this + # may take some time + ring: + expression: "ring AND ISC AND OPENSSL" + + +expressions: + # LLVM exception not relevant for the project, thus in situation + # like this we choose to use vanilla Apache-2.0 terms + "Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT": "Apache-2.0" diff --git a/prepare_release.sh b/scripts/prepare_release.sh similarity index 93% rename from prepare_release.sh rename to scripts/prepare_release.sh index d7b422a7..a41b81fe 100755 --- a/prepare_release.sh +++ b/scripts/prepare_release.sh @@ -26,8 +26,10 @@ if [ "$(echo "$1" | sed -E "s/$VERSION_REGEX//")" ]; then exit 1 fi -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" -DATA_BROKER_ROOT="$SCRIPT_DIR" +SCRIPT_PATH=$(realpath "$0") +SCRIPT_DIR=$(dirname "$SCRIPT_PATH") + +DATA_BROKER_ROOT=$SCRIPT_DIR/.. # Update Cargo.toml versions. sed -i -E "s/^version = \"${VERSION_REGEX}\"$/version = \"${VERSION}\"/" \