build_aarch64 #40
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # | |
| # Copyright (c) 2014-present, The osquery authors | |
| # | |
| # This source code is licensed as defined by the LICENSE file found in the | |
| # root directory of this source tree. | |
| # | |
| # SPDX-License-Identifier: (Apache-2.0 OR GPL-2.0-only) | |
| # | |
| # Due to a limitation in how GitHub Actions works, we can't reference | |
| # jobs in another file inside the `needs` statement. | |
| # | |
| # This configuration file takes care of the AArch64 Linux build | |
| name: build_aarch64 | |
| on: | |
| # Run this workflow once every day against the master branch | |
| schedule: | |
| - cron: "0 7 * * *" | |
| # Do not run this on pull requests | |
| push: | |
| branches: | |
| - '**' | |
| tags: | |
| - '**' | |
| # Please remember to update values for both x86 and aarch64 workflows. | |
| env: | |
| PACKAGING_REPO: https://github.com/osquery/osquery-packaging | |
| PACKAGING_COMMIT: c089fb2d3d796d976e3b2fbea7ee69a1616b9576 | |
| SUBMODULE_CACHE_VERSION: 3 | |
| # If the initial code sanity checks are passing, then one job | |
| # per [`platform` * `build_type`] will start, building osquery | |
| # and generating packages that are later attached to the commit | |
| # (or PR) as build artifacts. | |
| jobs: | |
| # This job performs basic source code check, looking for formatting | |
| # issues and missing copyright headers | |
| check_code_style: | |
| runs-on: ubuntu-20.04 | |
| container: | |
| image: osquery/builder20.04:7e9ee0339 | |
| options: --user 1001 | |
| steps: | |
| - name: Setup the build paths | |
| shell: bash | |
| id: build_paths | |
| run: | | |
| rel_build_path="workspace/build" | |
| rel_source_path="workspace/src" | |
| mkdir -p "${rel_build_path}" | |
| mkdir -p "${rel_source_path}" | |
| echo "SOURCE=$(realpath ${rel_source_path})" >> $GITHUB_OUTPUT | |
| echo "BINARY=$(realpath ${rel_build_path})" >> $GITHUB_OUTPUT | |
| - name: Clone the osquery repository | |
| uses: actions/checkout@v4 | |
| with: | |
| path: ${{ steps.build_paths.outputs.SOURCE }} | |
| fetch-depth: 0 | |
| # This script makes sure that the copyright headers have been correctly | |
| # placed on all the source code files | |
| - name: Check the copyright headers | |
| working-directory: ${{ steps.build_paths.outputs.SOURCE }} | |
| run: | | |
| ./tools/ci/scripts/check_copyright_headers.py | |
| - name: Configure the project | |
| working-directory: ${{ steps.build_paths.outputs.BINARY }} | |
| run: | | |
| cmake -G "Unix Makefiles" \ | |
| -DOSQUERY_TOOLCHAIN_SYSROOT:PATH="/usr/local/osquery-toolchain" \ | |
| -DOSQUERY_ENABLE_FORMAT_ONLY=ON \ | |
| "${{ steps.build_paths.outputs.SOURCE }}" | |
| # Formatting is tested against the clang-format binary we ship | |
| # with the osquery-toolchain, so this job is only performed once on | |
| # a Linux machine. | |
| - name: Check code formatting | |
| working-directory: ${{ steps.build_paths.outputs.BINARY }} | |
| run: | |
| cmake --build . --target format_check | |
| # NOTE: Here we manually create two different steps and outputs because there's no easy way | |
| # to dynamically access variables per matrix job. Having outputs with different names | |
| # lets us easily select the correct one later. | |
| # Starts self-hosted runner. | |
| start_ec2_runners: | |
| needs: check_code_style | |
| runs-on: ubuntu-latest | |
| outputs: | |
| label-Release: ${{ steps.start_ec2_runner-Release.outputs.label }} | |
| ec2-instance-id-Release: ${{ steps.start_ec2_runner-Release.outputs.ec2-instance-id }} | |
| label-RelWithDebInfo: ${{ steps.start_ec2_runner-RelWithDebInfo.outputs.label }} | |
| ec2-instance-id-RelWithDebInfo: ${{ steps.start_ec2_runner-RelWithDebInfo.outputs.ec2-instance-id }} | |
| steps: | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-access-key-id: ${{ secrets.EC2_GITHUB_RUNNER_AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.EC2_GITHUB_RUNNER_AWS_SECRET_ACCESS_KEY }} | |
| aws-region: ${{ secrets.EC2_GITHUB_RUNNER_AWS_REGION }} | |
| - name: Start aarch64 EC2 runner for a Release build | |
| id: start_ec2_runner-Release | |
| uses: osquery/[email protected] | |
| with: | |
| mode: start | |
| github-token: ${{ secrets.EC2_GITHUB_RUNNER_GH_PERSONAL_ACCESS_TOKEN }} | |
| ec2-image-id: ami-063d99d9ce1bebbe8 | |
| ec2-instance-type: c6g.4xlarge | |
| subnet-id: subnet-06390365e72579736 | |
| security-group-id: sg-0757a3162c999331b | |
| - name: Start aarch64 EC2 runner for a RelWithDebInfo build | |
| id: start_ec2_runner-RelWithDebInfo | |
| uses: osquery/[email protected] | |
| with: | |
| mode: start | |
| github-token: ${{ secrets.EC2_GITHUB_RUNNER_GH_PERSONAL_ACCESS_TOKEN }} | |
| ec2-image-id: ami-063d99d9ce1bebbe8 | |
| ec2-instance-type: c6g.4xlarge | |
| subnet-id: subnet-06390365e72579736 | |
| security-group-id: sg-0757a3162c999331b | |
| # Stops self-hosted runners. | |
| stop_ec2_runners: | |
| needs: | |
| - start_ec2_runners | |
| - build_linux | |
| if: always() | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-access-key-id: ${{ secrets.EC2_GITHUB_RUNNER_AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.EC2_GITHUB_RUNNER_AWS_SECRET_ACCESS_KEY }} | |
| aws-region: ${{ secrets.EC2_GITHUB_RUNNER_AWS_REGION }} | |
| - name: Stop aarch64 EC2 runner for Release mode build | |
| uses: osquery/[email protected] | |
| with: | |
| mode: stop | |
| github-token: ${{ secrets.EC2_GITHUB_RUNNER_GH_PERSONAL_ACCESS_TOKEN }} | |
| label: ${{ needs.start_ec2_runners.outputs.label-Release }} | |
| ec2-instance-id: ${{ needs.start_ec2_runners.outputs.ec2-instance-id-Release }} | |
| - name: Stop aarch64 EC2 runner for RelWithDebInfo mode build | |
| uses: osquery/[email protected] | |
| with: | |
| mode: stop | |
| github-token: ${{ secrets.EC2_GITHUB_RUNNER_GH_PERSONAL_ACCESS_TOKEN }} | |
| label: ${{ needs.start_ec2_runners.outputs.label-RelWithDebInfo }} | |
| ec2-instance-id: ${{ needs.start_ec2_runners.outputs.ec2-instance-id-RelWithDebInfo }} | |
| # The Linux build will only start once we know that the code | |
| # has been properly formatted | |
| build_linux: | |
| needs: start_ec2_runners | |
| runs-on: ${{ matrix.os }} | |
| container: | |
| image: osquery/builder20.04:7e9ee0339 | |
| options: --privileged --init -v /var/run/docker.sock:/var/run/docker.sock --pid=host --user root | |
| strategy: | |
| matrix: | |
| include: | |
| - os: ${{ needs.start_ec2_runners.outputs.label-Release }} | |
| build_type: Release | |
| cache_key: ubuntu-20.04_aarch64 | |
| - os: ${{ needs.start_ec2_runners.outputs.label-RelWithDebInfo }} | |
| build_type: RelWithDebInfo | |
| cache_key: ubuntu-20.04_aarch64 | |
| steps: | |
| # Due to how the RPM packaging tools work, we have to adhere to some | |
| # character count requirements in the build path vs source path. | |
| # | |
| # Failing to do so, will break the debuginfo RPM package. | |
| - name: Setup the build paths | |
| id: build_paths | |
| run: | | |
| # The group for the docker socket doesn't have gid docker | |
| sudo chown root:docker /var/run/docker.sock | |
| rel_build_path="workspace/usr/src/debug/osquery/build" | |
| rel_src_path="workspace/padding-required-by-rpm-packages/src" | |
| rel_ccache_path="workspace/ccache" | |
| rel_package_data_path="workspace/package_data" | |
| rel_packaging_path="workspace/osquery-packaging" | |
| rel_package_build_path="workspace/package-build" | |
| mkdir -p ${rel_build_path} \ | |
| ${rel_src_path} \ | |
| ${rel_ccache_path} \ | |
| ${rel_src_path} \ | |
| ${rel_package_data_path} \ | |
| ${rel_package_build_path} | |
| echo "SOURCE=$(realpath ${rel_src_path})" >> $GITHUB_OUTPUT | |
| echo "BINARY=$(realpath ${rel_build_path})" >> $GITHUB_OUTPUT | |
| echo "CCACHE=$(realpath ${rel_ccache_path})" >> $GITHUB_OUTPUT | |
| echo "PACKAGING=$(realpath ${rel_packaging_path})" >> $GITHUB_OUTPUT | |
| echo "PACKAGE_DATA=$(realpath ${rel_package_data_path})" >> $GITHUB_OUTPUT | |
| echo "REL_PACKAGE_BUILD=${rel_package_build_path}" >> $GITHUB_OUTPUT | |
| echo "PACKAGE_BUILD=$(realpath ${rel_package_build_path})" >> $GITHUB_OUTPUT | |
| - name: Clone the osquery repository | |
| uses: actions/checkout@v4 | |
| with: | |
| path: ${{ steps.build_paths.outputs.SOURCE }} | |
| fetch-depth: 0 | |
| - name: Select the build job count | |
| shell: bash | |
| id: build_job_count | |
| run: | | |
| echo "VALUE=$(($(nproc) + 1))" >> $GITHUB_OUTPUT | |
| - name: Select the build options for the tests | |
| shell: bash | |
| id: tests_build_settings | |
| run: | | |
| if [[ "${{ matrix.build_type }}" == "RelWithDebInfo" ]] ; then | |
| echo "VALUE=OFF" >> $GITHUB_OUTPUT | |
| else | |
| echo "VALUE=ON" >> $GITHUB_OUTPUT | |
| fi | |
| # try to gather some info about build space | |
| - name: Disk Space Information | |
| shell: bash | |
| id: disk_space_info | |
| run: | | |
| df -h | |
| # We don't have enough space on the worker to actually generate all | |
| # the debug symbols (osquery + dependencies), so we have a flag to | |
| # disable them when running a Debug build | |
| - name: Select the debug symbols options | |
| shell: bash | |
| id: debug_symbols_settings | |
| run: | | |
| if [[ "${{ matrix.build_type }}" == "Debug" ]] ; then | |
| echo "VALUE=ON" >> $GITHUB_OUTPUT | |
| else | |
| echo "VALUE=OFF" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Clone the osquery-packaging repository | |
| run: | | |
| git clone ${{ env.PACKAGING_REPO }} \ | |
| ${{ steps.build_paths.outputs.PACKAGING }} | |
| cd ${{ steps.build_paths.outputs.PACKAGING }} | |
| git checkout ${{ env.PACKAGING_COMMIT }} | |
| - name: Update the cache (ccache) | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ steps.build_paths.outputs.CCACHE }} | |
| key: | | |
| ccache_${{ matrix.cache_key }}_${{ matrix.build_type }}_${{ github.sha }} | |
| restore-keys: | | |
| ccache_${{ matrix.cache_key }}_${{ matrix.build_type }} | |
| - name: Update the cache (git submodules) | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ steps.build_paths.outputs.SOURCE }}/.git/modules | |
| key: | | |
| gitmodules_${{ matrix.cache_key }}_${{env.SUBMODULE_CACHE_VERSION}}_${{ github.sha }} | |
| restore-keys: | | |
| gitmodules_${{ matrix.cache_key }}_${{env.SUBMODULE_CACHE_VERSION}} | |
| - name: Update the git submodules | |
| working-directory: ${{ steps.build_paths.outputs.SOURCE }} | |
| run: | | |
| git submodule sync --recursive | |
| - name: Configure the project | |
| working-directory: ${{ steps.build_paths.outputs.BINARY }} | |
| env: | |
| CCACHE_DIR: ${{ steps.build_paths.outputs.CCACHE }} | |
| run: | | |
| cmake -G "Unix Makefiles" \ | |
| -DOSQUERY_NO_DEBUG_SYMBOLS=${{ steps.debug_symbols_settings.outputs.VALUE }} \ | |
| -DOSQUERY_TOOLCHAIN_SYSROOT:PATH="/usr/local/osquery-toolchain" \ | |
| -DCMAKE_BUILD_TYPE:STRING="${{ matrix.build_type }}" \ | |
| -DOSQUERY_BUILD_TESTS=${{ steps.tests_build_settings.outputs.VALUE }} \ | |
| -DOSQUERY_BUILD_ROOT_TESTS=${{ steps.tests_build_settings.outputs.VALUE }} \ | |
| "${{ steps.build_paths.outputs.SOURCE }}" | |
| - name: Build the project | |
| working-directory: ${{ steps.build_paths.outputs.BINARY }} | |
| env: | |
| CCACHE_DIR: ${{ steps.build_paths.outputs.CCACHE }} | |
| run: | | |
| cmake --build . -j ${{ steps.build_job_count.outputs.VALUE }} | |
| - name: Disk Space Information | |
| shell: bash | |
| id: disk_space_info_post_build | |
| run: | | |
| df -h | |
| du -sh ${{ steps.build_paths.outputs.BINARY }} | |
| - name: Run the tests as normal user | |
| working-directory: ${{ steps.build_paths.outputs.BINARY }} | |
| run: | | |
| sudo chown -R runner:runner . | |
| sudo -u runner ctest --build-nocmake -LE "root-required" -V | |
| - name: Run the tests as root user | |
| working-directory: ${{ steps.build_paths.outputs.BINARY }} | |
| run: | | |
| sudo -u root ctest --build-nocmake -L "root-required" -V | |
| - name: Run the install target | |
| if: matrix.build_type == 'RelWithDebInfo' | |
| working-directory: ${{ steps.build_paths.outputs.BINARY }} | |
| env: | |
| CCACHE_DIR: ${{ steps.build_paths.outputs.CCACHE }} | |
| DESTDIR: ${{ steps.build_paths.outputs.PACKAGE_DATA }} | |
| run: | | |
| cmake \ | |
| --build . \ | |
| --target install \ | |
| -j ${{ steps.build_job_count.outputs.VALUE }} | |
| # Do some targeted cleanups to get additional free space before installing dependencies, | |
| # we are unfortunately out of space otherwise. | |
| find . -name "*.a" -exec rm {} \; | |
| find . -name "*.o" -exec rm {} \; | |
| - name: Create the packages | |
| if: matrix.build_type == 'RelWithDebInfo' | |
| working-directory: ${{ steps.build_paths.outputs.PACKAGE_BUILD }} | |
| shell: bash | |
| run: | | |
| osquery_version=$(cd ${{ steps.build_paths.outputs.SOURCE }} && git describe --tags --always ) | |
| tar pcvzf package_data.tar.gz \ | |
| ${{ steps.build_paths.outputs.PACKAGE_DATA }} | |
| package_format_list=( "DEB" "RPM" "TGZ" ) | |
| for package_format in "${package_format_list[@]}" ; do | |
| cmake -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \ | |
| -DCPACK_GENERATOR=${package_format} \ | |
| -DOSQUERY_PACKAGE_VERSION=${osquery_version} \ | |
| -DOSQUERY_DATA_PATH=${{ steps.build_paths.outputs.PACKAGE_DATA }} \ | |
| -DOSQUERY_SOURCE_DIRECTORY_LIST="${{ steps.build_paths.outputs.SOURCE }};${{ steps.build_paths.outputs.BINARY }}" \ | |
| ${{ steps.build_paths.outputs.PACKAGING }} | |
| cmake --build . \ | |
| --target package | |
| done | |
| - name: Locate the packages | |
| if: matrix.build_type == 'RelWithDebInfo' | |
| id: packages | |
| shell: bash | |
| run: | | |
| echo "REL_UNSIGNED_RELEASE_PACKAGE_DATA_PATH=${{ steps.build_paths.outputs.REL_PACKAGE_BUILD }}/package_data.tar.gz" >> $GITHUB_OUTPUT | |
| echo "REL_UNSIGNED_RELEASE_DEB_PATH=$(ls ${{ steps.build_paths.outputs.REL_PACKAGE_BUILD }}/*.deb)" >> $GITHUB_OUTPUT | |
| echo "REL_UNSIGNED_DEBUG_DEB_PATH=$(ls ${{ steps.build_paths.outputs.REL_PACKAGE_BUILD }}/*.ddeb)" >> $GITHUB_OUTPUT | |
| echo "REL_UNSIGNED_RELEASE_RPM_PATH=$(ls ${{ steps.build_paths.outputs.REL_PACKAGE_BUILD }}/osquery-?.*.rpm)" >> $GITHUB_OUTPUT | |
| echo "REL_UNSIGNED_DEBUG_RPM_PATH=$(ls ${{ steps.build_paths.outputs.REL_PACKAGE_BUILD }}/osquery-debuginfo-*.rpm)" >> $GITHUB_OUTPUT | |
| echo "REL_UNSIGNED_RELEASE_TGZ_PATH=$(ls ${{ steps.build_paths.outputs.REL_PACKAGE_BUILD }}/*linux_aarch64.tar.gz)" >> $GITHUB_OUTPUT | |
| - name: Store the unsigned release package data artifact | |
| if: matrix.build_type == 'RelWithDebInfo' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: linux_unsigned_release_package_data_aarch64 | |
| path: ${{ steps.packages.outputs.REL_UNSIGNED_RELEASE_PACKAGE_DATA_PATH }} | |
| - name: Store the unsigned release DEB artifact | |
| if: matrix.build_type == 'RelWithDebInfo' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: linux_unsigned_release_deb_aarch64 | |
| path: ${{ steps.packages.outputs.REL_UNSIGNED_RELEASE_DEB_PATH }} | |
| - name: Store the unsigned debug DEB artifact | |
| if: matrix.build_type == 'RelWithDebInfo' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: linux_unsigned_debug_deb_aarch64 | |
| path: ${{ steps.packages.outputs.REL_UNSIGNED_DEBUG_DEB_PATH }} | |
| - name: Store the unsigned release RPM artifact | |
| if: matrix.build_type == 'RelWithDebInfo' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: linux_unsigned_release_rpm_aarch64 | |
| path: ${{ steps.packages.outputs.REL_UNSIGNED_RELEASE_RPM_PATH }} | |
| - name: Store the unsigned debug RPM artifact | |
| if: matrix.build_type == 'RelWithDebInfo' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: linux_unsigned_debug_rpm_aarch64 | |
| path: ${{ steps.packages.outputs.REL_UNSIGNED_DEBUG_RPM_PATH }} | |
| - name: Store the unsigned release TGZ artifact | |
| if: matrix.build_type == 'RelWithDebInfo' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: linux_unsigned_release_tgz_aarch64 | |
| path: ${{ steps.packages.outputs.REL_UNSIGNED_RELEASE_TGZ_PATH }} | |
| # Before we terminate this job, delete the build folder. The cache | |
| # actions will require the disk space to create the archives. | |
| - name: Reclaim disk space | |
| run: | | |
| rm -rf ${{ steps.build_paths.outputs.BINARY }} |