Do not ignore node_modules

Signed-off-by: prabhu <[email protected]> nydus test Signed-off-by: Prabhu Subramanian <[email protected]> Added docker example for speed comparison Signed-off-by: Prabhu Subramanian <[email protected]>
CycloneDX · Nov 21, 2023 · 78d1e98 · 78d1e98
1 parent fa40e3c
commit 78d1e98
Show file tree

Hide file tree

Showing 6 changed files with 130 additions and 3 deletions.
diff --git a/.github/workflows/java-reachables-test.yml b/.github/workflows/java-reachables-test.yml
@@ -35,7 +35,7 @@ jobs:
       - name: compile
         run: |
           cd repotests/dependency-track
-          mvn clean compile -DskipTests -Dmaven.test.skip=true
+          mvn -q clean compile -DskipTests -Dmaven.test.skip=true
       - name: repotests
         run: |         
           bin/cdxgen.js -p -t java --profile research -o repotests/dependency-track/bom.json repotests/dependency-track

diff --git a/.github/workflows/nydus-demo.yml b/.github/workflows/nydus-demo.yml
@@ -0,0 +1,73 @@
+name: nydus tests
+
+on:
+  workflow_dispatch:
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        node-version: ['21.x']
+        os: ['ubuntu-latest']
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up JDK
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'temurin'
+          java-version: '19'
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node-version }}
+      - uses: actions/checkout@v4
+        with:
+          repository: 'DependencyTrack/dependency-track'
+          path: 'repotests/dependency-track'
+      - name: setup nydus
+        run: |
+          curl -LO https://github.com/dragonflyoss/nydus/releases/download/v2.2.4/nydus-static-v2.2.4-linux-amd64.tgz
+          tar -xvf nydus-static-v2.2.4-linux-amd64.tgz
+          chmod +x nydus-static/*
+          mv nydus-static/* /usr/local/bin/
+          rm -rf nydus-static-v2.2.4-linux-amd64.tgz nydus-static
+          curl -LO https://github.com/containerd/nerdctl/releases/download/v1.7.0/nerdctl-full-1.7.0-linux-amd64.tar.gz
+          sudo tar Cxzvf /usr/local nerdctl-full-1.7.0-linux-amd64.tar.gz
+          sudo mkdir -p /etc/nydus/
+          sudo rm -rf /var/lib/containerd-nydus
+          curl -LO https://github.com/containerd/nydus-snapshotter/releases/download/v0.13.3/nydus-snapshotter-v0.13.3-x86_64.tgz
+          sudo tar Cxzvf /usr/local/bin nydus-snapshotter-v0.13.3-x86_64.tgz
+          sudo mv /usr/local/bin/nydus-snapshotter/* /usr/local/bin/
+          sudo cp ./ci/nydusd-config.fusedev.json /etc/nydus/nydusd-config.fusedev.json
+          sudo mkdir -p /etc/containerd
+          sudo cp ./ci/containerd-config.toml /etc/containerd/config.toml
+          sudo systemctl enable --now containerd
+          sudo systemctl restart containerd
+      - name: compile
+        run: |
+          cd repotests/dependency-track
+          mvn -q clean compile -DskipTests -Dmaven.test.skip=true
+      - name: nydustests
+        run: |
+          sudo /usr/local/bin/containerd-nydus-grpc --nydusd-config /etc/nydus/nydusd-config.fusedev.json &
+          sleep 15
+          cd repotests/dependency-track
+          sudo nerdctl --snapshotter nydus run --rm -e JAVA_TOOL_OPTIONS -v $HOME/.m2:$HOME/.m2 -v $(pwd):/app ghcr.io/cyclonedx/cdxgen:master-nydus --deep -t java /app -o /app/bom.json
+          ls -ltr
+        env:
+          JAVA_TOOL_OPTIONS: "-Dfile.encoding=UTF-8"
+      - name: cleanup
+        run: |
+          cd repotests/dependency-track
+          rm bom.json
+          sudo rm -rf /var/lib/containerd-nydus
+        continue-on-error: true
+      - name: dockertests
+        run: |
+          sleep 15
+          cd repotests/dependency-track
+          sudo nerdctl run --rm -e JAVA_TOOL_OPTIONS -v $HOME/.m2:$HOME/.m2 -v $(pwd):/app ghcr.io/cyclonedx/cdxgen:master --deep -t java /app -o /app/bom.json
+          ls -ltr
+        env:
+          JAVA_TOOL_OPTIONS: "-Dfile.encoding=UTF-8"
diff --git a/ci/Dockerfile b/ci/Dockerfile
@@ -37,11 +37,15 @@ ENV GOPATH=/opt/app-root/go \
     PYTHONIOENCODING="utf-8" \
     COMPOSER_ALLOW_SUPERUSER=1 \
     ANDROID_HOME=/opt/android-sdk-linux \
+    JAVA_TOOL_OPTIONS="-Dfile.encoding=UTF-8"
     SWIFT_SIGNING_KEY=$SWIFT_SIGNING_KEY \
     SWIFT_PLATFORM=$SWIFT_PLATFORM \
     SWIFT_BRANCH=$SWIFT_BRANCH \
     SWIFT_VERSION=$SWIFT_VERSION \
-    SWIFT_WEBROOT=$SWIFT_WEBROOT
+    SWIFT_WEBROOT=$SWIFT_WEBROOT \
+    LC_ALL=en_US.UTF-8 \
+    LANG=en_US.UTF-8 \
+    LANGUAGE=en_US.UTF-8
 ENV PATH=${PATH}:${JAVA_HOME}/bin:${MAVEN_HOME}/bin:${GRADLE_HOME}/bin:${SBT_HOME}/bin:${GOPATH}/bin:/usr/local/go/bin:/usr/local/bin/:/root/.local/bin:${ANDROID_HOME}/cmdline-tools/latest/bin:${ANDROID_HOME}/tools:${ANDROID_HOME}/tools/bin:${ANDROID_HOME}/platform-tools:
 
 RUN set -e; \
@@ -61,7 +65,7 @@ RUN set -e; \
     echo -e "[nodejs]\nname=nodejs\nstream=21\nprofiles=\nstate=enabled\n" > /etc/dnf/modules.d/nodejs.module \
     && microdnf module enable php ruby -y \
     && microdnf install -y php php-curl php-zip php-bcmath php-json php-pear php-mbstring php-devel make gcc git-core \
-        python3.11 python3.11-devel python3.11-pip ruby ruby-devel \
+        python3.11 python3.11-devel python3.11-pip ruby ruby-devel glibc-common glibc-all-langpacks \
         pcre2 which tar gzip zip unzip sudo nodejs ncurses sqlite-devel \
     && alternatives --install /usr/bin/python3 python /usr/bin/python3.11 1 \
     && python3 --version \

diff --git a/ci/containerd-config.toml b/ci/containerd-config.toml
@@ -0,0 +1,4 @@
+[proxy_plugins]
+  [proxy_plugins.nydus]
+    type = "snapshot"
+    address = "/run/containerd-nydus/containerd-nydus-grpc.sock"
diff --git a/ci/nydusd-config.fusedev.json b/ci/nydusd-config.fusedev.json
@@ -0,0 +1,30 @@
+{
+    "device": {
+      "backend": {
+        "type": "registry",
+        "config": {
+          "scheme": "",
+          "skip_verify": true,
+          "timeout": 5,
+          "connect_timeout": 5,
+          "retry_limit": 4,
+          "auth": ""
+        }
+      },
+      "cache": {
+        "type": "blobcache",
+        "config": {
+          "work_dir": "cache"
+        }
+      }
+    },
+    "mode": "direct",
+    "digest_validate": false,
+    "iostats_files": false,
+    "enable_xattr": true,
+    "fs_prefetch": {
+      "enable": true,
+      "threads_count": 4
+    }
+  }
+
diff --git a/docs/ADVANCED.md b/docs/ADVANCED.md
@@ -331,3 +331,19 @@ With profiles, cdxgen can generate a BOM that is optimized for a specific use ca
 | research           | BOM for security research                                                 | Enables deep and evidence mode. Disable ignore directory for JavaScript/TypeScript |
 | operational        | Generate OBOM                                                             | projectType set to os                                                              |
 | license-compliance | Fetch license data                                                        | Set FETCH_LICENSE environment variable                                             |
+
+## Nydus - next-generation container image
+
+[Nydus](https://github.com/dragonflyoss/nydus) enhances the current OCI image specification by improving container launch speed, image space and network bandwidth efficiency, and data integrity. cdxgen container images are available in nydus format with the `-nydus` suffix.
+
+```
+ghcr.io/cyclonedx/cdxgen:master-nydus
+```
+
+### Example invocation using nerdctl
+
+Refer to the nydus-demo.yml workflow for an example github action that demonstrates the use of nydus snapshotter to improve the performance of cdxgen.
+
+```shell
+sudo nerdctl --snapshotter nydus run --rm -v $HOME/.m2:/root/.m2 -v $(pwd):/app ghcr.io/cyclonedx/cdxgen:master-nydus -p -t java /app
+```