diff --git a/.github/cleanup-rootfs.sh b/.github/cleanup-rootfs.sh new file mode 100755 index 00000000000..7d1b255c41c --- /dev/null +++ b/.github/cleanup-rootfs.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +# There is no need to wait for this to finish, at least the rm part can be done +# while the process moves forward (for apt we'll need to wait for it to finish +# before we install dependencies later on, but it'll only give us a 1-3GBs so +# we can skip it. +WAIT=0 +RMONLY=1 + +PACKAGES=( + "firefox" + "google-chrome-stable" + "microsoft-edge-stable" + "php-pear" + "ruby-full" + "^aspnetcore-.*" + "^dotnet-.*" + "powershell*" +) + +PATHS=( + "/opt/hostedtoolcache" + "/usr/local/.ghcup/" + "/usr/share/swift" + "/usr/local/lib/android" + "/usr/local/share/edge_driver" + "/usr/local/share/gecko_driver" + "/usr/local/share/chromedriver-linux64" + "/usr/local/share/chromium" + "/home/linuxbrew" + "/usr/local/share/vcpkg" + "/usr/share/kotlinc" + "/usr/local/bin/minikube" +) + + +function cleanup_packages() +{ + if [[ ${RMONLY} == 0 ]]; then + apt-get purge -y "${PACKAGES[@]}" + apt-get autoremove --purge -y + apt-get clean + fi +} + +function cleanup_paths() +{ + for i in "${PATHS[@]}"; do + rm -rf "${i}" & + done + if [[ ${WAIT} == 1 ]]; then + wait + fi +} + +if [[ ${WAIT} == 1 ]]; then + echo "---=== Before ===---" + df -hT + cleanup_packages + cleanup_paths + echo "---=== After ===---" + df -hT +else + cleanup_packages + cleanup_paths +fi diff --git a/.github/dedup-dir.sh b/.github/dedup-dir.sh new file mode 100755 index 00000000000..079fdbf7234 --- /dev/null +++ b/.github/dedup-dir.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +function deduplicate_files() { + local DUPLICATES=() + local DIR=${1} + local IFS=$'\n' + local LINK_CHECK="" + + readarray -t DUPLICATES < <(for i in `find ${DIR} -type f ! -empty`; do sha1sum ${i}; done | sort | uniq -w 40 --all-repeated=separate) + + for ((i=1; i < ${#DUPLICATES[@]}; i++ )); do + if [[ ${DUPLICATES[$i]} == "" ]]; then + continue + elif [[ ${DUPLICATES[$i-1]} = "" ]]; then + continue + else + LINK_CHECK=$(ls -li "${DUPLICATES[$i]:42}" "${DUPLICATES[$i-1]:42}" |awk '{print $1}' | uniq | wc -l) + if [[ ${LINK_CHECK} != "1" ]]; then + ln -f "${DUPLICATES[$i-1]:42}" "${DUPLICATES[$i]:42}" + fi + fi + done + + return 0 +} + +deduplicate_files ${1} diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 532332b2990..7c06fb7b23e 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -1,6 +1,7 @@ name: Build on: + workflow_dispatch: push: branches: - master @@ -8,9 +9,59 @@ on: branches: - master +env: + submodule_paths: | + binutils + dejagnu + gcc + gdb + glibc + llvm + musl + newlib + pk + qemu + spike + uclibc-ng + .git/modules + jobs: + submodule_cache: + name: Initialize submodule cache + runs-on: ubuntu-latest + outputs: + key: ${{ steps.keygen.outputs.smcache_key }} + steps: + - uses: actions/checkout@v4 + + - name: Remove unneeded frameworks to recover disk space + run: sudo ./.github/cleanup-rootfs.sh + + - name: Generate submodule cache key + id: keygen + run: echo "smcache_key=smcache-$(printf $(git submodule | sha1sum))" >> $GITHUB_OUTPUT + + - name: Setup submodule cache + id: smcache + uses: actions/cache@v4 + with: + path: ${{ env.submodule_paths }} + key: ${{ steps.keygen.outputs.smcache_key }} + + - name: Checkout required submodules + if: steps.smcache.outputs.cache-hit != 'true' + run: git submodule update --init -j $(nproc) --depth 1 $(echo ${submodule_paths} | sed '$d' | tr '\n' ' ') + + - name: Storage size optimization + if: steps.smcache.outputs.cache-hit != 'true' + run: | + git submodule foreach 'git maintenance run' + build: runs-on: ${{ matrix.os }} + needs: [submodule_cache] + env: + smcache_key: ${{ needs.submodule_cache.outputs.key }} strategy: matrix: os: [ubuntu-22.04, ubuntu-24.04] @@ -23,30 +74,38 @@ jobs: - mode: uclibc compiler: llvm steps: - - name: Remove unneeded frameworks to recover disk space - run: | - echo "-- Before --" - df -h - sudo rm -rf /usr/share/dotnet - sudo rm -rf /usr/local/lib/android - echo "-- After --" - df -h - - uses: actions/checkout@v4 + - name: Remove unneeded frameworks to recover disk space + run: sudo ./.github/cleanup-rootfs.sh + - name: install dependencies run: sudo ./.github/setup-apt.sh + - name: Load submodule cache + uses: actions/cache/restore@v4 + with: + path: ${{ env.submodule_paths }} + key: ${{ env.smcache_key }} + - name: build toolchain run: | TARGET_TUPLE=($(echo ${{ matrix.target }} | tr "-" "\n")) - BUILD_TOOLCHAIN="./configure --prefix=/opt/riscv --with-arch=${TARGET_TUPLE[0]} --with-abi=${TARGET_TUPLE[1]}" + BUILD_TOOLCHAIN="./configure --prefix=/mnt/riscv --with-arch=${TARGET_TUPLE[0]} --with-abi=${TARGET_TUPLE[1]}" if [ "${{ matrix.compiler }}" == "llvm" ]; then # build toolchain with llvm $BUILD_TOOLCHAIN --enable-llvm else $BUILD_TOOLCHAIN fi - sudo make -j $(nproc) ${{ matrix.mode }} + sudo mkdir /mnt/riscv + sudo chown runner:runner /mnt/riscv + make -j $(nproc) ${{ matrix.mode }} + + - name: tarball build + run: | + du -s -h /mnt/riscv + ./.github/dedup-dir.sh /mnt/riscv/ + XZ_OPT="-e -T0" tar cJvf riscv.tar.xz -C /mnt/ riscv/ - name: make report if: | @@ -54,16 +113,7 @@ jobs: && (matrix.mode == 'linux' || matrix.mode == 'newlib') && matrix.compiler == 'gcc' run: | - sudo make report-${{ matrix.mode }} -j $(nproc) - - - name: recover space - run: | - sudo du -hs / 2> /dev/null || true - sudo rm -rf binutils dejagnu gcc gdb glibc llvm musl newlib pk qemu spike uclibc-ng || true - sudo du -hs / 2> /dev/null || true - - - name: tarball build - run: tar czvf riscv.tar.gz -C /opt/ riscv/ + make report-${{ matrix.mode }} -j $(nproc) - name: generate prebuilt toolchain name id: toolchain-name-generator @@ -84,10 +134,13 @@ jobs: - uses: actions/upload-artifact@v4 with: name: ${{ steps.toolchain-name-generator.outputs.TOOLCHAIN_NAME }} - path: riscv.tar.gz + path: riscv.tar.xz test-sim: runs-on: ${{ matrix.os }} + needs: [submodule_cache] + env: + smcache_key: ${{ needs.submodule_cache.outputs.key }} strategy: matrix: os: [ubuntu-24.04] @@ -95,24 +148,26 @@ jobs: target: [rv64gc-lp64d] sim: [spike] steps: - - name: Remove unneeded frameworks to recover disk space - run: | - echo "-- Before --" - df -h - sudo rm -rf /usr/share/dotnet - sudo rm -rf /usr/local/lib/android - echo "-- After --" - df -h - - uses: actions/checkout@v4 + - name: Remove unneeded frameworks to recover disk space + run: sudo ./.github/cleanup-rootfs.sh + - name: install dependencies run: sudo ./.github/setup-apt.sh + - name: Load submodule cache + uses: actions/cache/restore@v4 + with: + path: ${{ env.submodule_paths }} + key: ${{ env.smcache_key }} + - name: build toolchain run: | TARGET_TUPLE=($(echo ${{ matrix.target }} | tr "-" "\n")) - ./configure --prefix=/opt/riscv --with-arch=${TARGET_TUPLE[0]} --with-abi=${TARGET_TUPLE[1]} --with-sim=${{ matrix.sim }} + ./configure --prefix=/mnt/riscv --with-arch=${TARGET_TUPLE[0]} --with-abi=${TARGET_TUPLE[1]} --with-sim=${{ matrix.sim }} + sudo mkdir /mnt/riscv + sudo chown runner:runner /mnt/riscv make -j $(nproc) ${{ matrix.mode }} - name: make report @@ -121,38 +176,46 @@ jobs: build-multilib: if: ${{ false }} # Disable until multilib errors are triaged runs-on: ${{ matrix.os }} + needs: [submodule_cache] + env: + smcache_key: ${{ needs.submodule_cache.outputs.key }} strategy: matrix: os: [ubuntu-24.04] mode: [newlib, linux] target: [rv64gc-lp64d] steps: - - name: Remove unneeded frameworks to recover disk space - run: | - echo "-- Before --" - df -h - sudo rm -rf /usr/share/dotnet - sudo rm -rf /usr/local/lib/android - echo "-- After --" - df -h - - uses: actions/checkout@v4 + - name: Remove unneeded frameworks to recover disk space + run: sudo ./.github/cleanup-rootfs.sh + - name: install dependencies run: sudo ./.github/setup-apt.sh + - name: Load submodule cache + uses: actions/cache/restore@v4 + with: + path: ${{ env.submodule_paths }} + key: ${{ env.smcache_key }} + - name: build toolchain run: | TARGET_TUPLE=($(echo ${{ matrix.target }} | tr "-" "\n")) - ./configure --prefix=/opt/riscv --with-arch=${TARGET_TUPLE[0]} --with-abi=${TARGET_TUPLE[1]} --enable-multilib - sudo make -j $(nproc) ${{ matrix.mode }} + ./configure --prefix=/mnt/riscv --with-arch=${TARGET_TUPLE[0]} --with-abi=${TARGET_TUPLE[1]} --enable-multilib + sudo mkdir /mnt/riscv + sudo chown runner:runner /mnt/riscv + make -j $(nproc) ${{ matrix.mode }} - - name: make report + - name: tarball build run: | - sudo make report-${{ matrix.mode }} -j $(nproc) + du -s -h /mnt/riscv + ./.github/dedup-dir.sh /mnt/riscv/ + XZ_OPT="-e -T0" tar cJvf riscv.tar.xz -C /mnt/ riscv/ - - name: tarball build - run: tar czvf riscv.tar.gz -C /opt/ riscv/ + - name: make report + run: | + make report-${{ matrix.mode }} -j $(nproc) - name: generate prebuilt toolchain name id: toolchain-name-generator @@ -173,4 +236,4 @@ jobs: - uses: actions/upload-artifact@v4 with: name: ${{ steps.toolchain-name-generator.outputs.TOOLCHAIN_NAME }} - path: riscv.tar.gz + path: riscv.tar.xz diff --git a/.github/workflows/nightly-release.yaml b/.github/workflows/nightly-release.yaml index 5f97d9b2860..a374d8b38f6 100644 --- a/.github/workflows/nightly-release.yaml +++ b/.github/workflows/nightly-release.yaml @@ -59,44 +59,32 @@ jobs: - mode: uclibc compiler: llvm steps: - - name: Remove unneeded frameworks to recover disk space - run: | - echo "-- Before --" - df -h - sudo rm -rf /usr/share/dotnet - sudo rm -rf /usr/local/lib/android - echo "-- After --" - df -h - - uses: actions/checkout@v4 + - name: Remove unneeded frameworks to recover disk space + run: sudo ./.github/cleanup-rootfs.sh + - name: install apt dependencies run: sudo ./.github/setup-apt.sh - name: build toolchain run: | TARGET_TUPLE=($(echo ${{ matrix.target }} | tr "-" "\n")) - BUILD_TOOLCHAIN="./configure --prefix=/opt/riscv --with-arch=${TARGET_TUPLE[0]} --with-abi=${TARGET_TUPLE[1]}" + BUILD_TOOLCHAIN="./configure --prefix=/mnt/riscv --with-arch=${TARGET_TUPLE[0]} --with-abi=${TARGET_TUPLE[1]}" if [ "${{ matrix.compiler }}" == "llvm" ]; then # build toolchain with llvm $BUILD_TOOLCHAIN --enable-llvm else $BUILD_TOOLCHAIN fi - sudo make -j $(nproc) ${{ matrix.mode }} - - - name: build qemu - if: ${{ matrix.mode }} == 'linux' - run: | - sudo make -j$(nproc) build-sim SIM=qemu - - - name: recover space - run: | - sudo du -hs / 2> /dev/null || true - sudo rm -rf binutils dejagnu gcc gdb glibc llvm musl newlib pk qemu spike uclibc-ng || true - sudo du -hs / 2> /dev/null || true + sudo mkdir /mnt/riscv + sudo chown runner:runner /mnt/riscv + make -j $(nproc) ${{ matrix.mode }} - name: tarball build - run: tar czvf riscv.tar.gz -C /opt/ riscv/ + run: | + du -s -h /mnt/riscv + ./.github/dedup-dir.sh /mnt/riscv/ + XZ_OPT="-e -T0" tar cJvf riscv.tar.xz -C /mnt/ riscv/ - name: generate prebuilt toolchain name id: toolchain-name-generator @@ -117,7 +105,7 @@ jobs: - uses: actions/upload-artifact@v4 with: name: ${{ steps.toolchain-name-generator.outputs.TOOLCHAIN_NAME }} - path: riscv.tar.gz + path: riscv.tar.xz create-release: @@ -142,8 +130,9 @@ jobs: echo "Version: ${DATESTAMP}-nightly" # Setup Artifacts Directory - ARTIFACTS_DIR="/opt/artifacts/" - mkdir -p $ARTIFACTS_DIR + ARTIFACTS_DIR="/mnt/artifacts/" + sudo mkdir -p $ARTIFACTS_DIR + sudo chown runner:runner $ARTIFACTS_DIR # Setup environment variables echo "DATESTAMP=${DATESTAMP}" >> $GITHUB_ENV @@ -227,4 +216,4 @@ jobs: upload_url: ${{ needs.create-release.outputs.upload_url }} asset_path: ${{ matrix.file }} asset_name: ${{ matrix.artifact }}-${{ needs.create-release.outputs.datestamp }}-nightly${{ matrix.extension }} - asset_content_type: application/gzip + asset_content_type: application/x-xz diff --git a/.gitmodules b/.gitmodules index 09f37f36668..9dc3a60554d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,44 +2,56 @@ path = binutils url = https://sourceware.org/git/binutils-gdb.git branch = binutils-2_43-branch + shallow = true [submodule "gcc"] path = gcc url = https://gcc.gnu.org/git/gcc.git branch = releases/gcc-14 + shallow = true [submodule "glibc"] path = glibc url = https://sourceware.org/git/glibc.git + shallow = true [submodule "dejagnu"] path = dejagnu url = https://git.savannah.gnu.org/git/dejagnu.git branch = master + shallow = true [submodule "newlib"] path = newlib url = https://sourceware.org/git/newlib-cygwin.git branch = master + shallow = true [submodule "gdb"] path = gdb url = https://sourceware.org/git/binutils-gdb.git branch = gdb-15-branch + shallow = true [submodule "qemu"] path = qemu url = https://gitlab.com/qemu-project/qemu.git + shallow = true [submodule "musl"] path = musl url = https://git.musl-libc.org/git/musl branch = master + shallow = true [submodule "spike"] path = spike url = https://github.com/riscv-software-src/riscv-isa-sim.git branch = master + shallow = true [submodule "pk"] path = pk url = https://github.com/riscv-software-src/riscv-pk.git branch = master + shallow = true [submodule "llvm"] path = llvm url = https://github.com/llvm/llvm-project.git branch = release/18.x + shallow = true [submodule "uclibc-ng"] path = uclibc-ng - url = https://git.uclibc-ng.org/git/uclibc-ng.git + url = https://github.com/wbx-github/uclibc-ng.git + shallow = true diff --git a/Makefile.in b/Makefile.in index 4449aa00d58..b597a4057c2 100644 --- a/Makefile.in +++ b/Makefile.in @@ -347,7 +347,7 @@ endif $(srcdir)/%/.git: cd $(srcdir) && \ flock `git rev-parse --git-dir`/config git submodule init $(dir $@) && \ - flock `git rev-parse --git-dir`/config git submodule update --progress $(dir $@) + flock `git rev-parse --git-dir`/config git submodule update --progress --depth 1 $(dir $@) stamps/install-host-gcc: $(GCC_SRCDIR) $(GCC_SRC_GIT) if test -f $