From fbf11389777e518aab4de7e71822f38c05b2aeb7 Mon Sep 17 00:00:00 2001 From: ClawSeven Date: Thu, 27 Jun 2024 17:38:02 +0800 Subject: [PATCH] [ci] Integrate io_uring feature CI --- .../workflows/composite_action/hw/action.yml | 10 +- .github/workflows/demo_test.yml | 116 +++++++++++++++++- .github/workflows/hw_mode_test.yml | 47 ++++--- 3 files changed, 150 insertions(+), 23 deletions(-) diff --git a/.github/workflows/composite_action/hw/action.yml b/.github/workflows/composite_action/hw/action.yml index c5059649f2..70fca98a64 100644 --- a/.github/workflows/composite_action/hw/action.yml +++ b/.github/workflows/composite_action/hw/action.yml @@ -87,6 +87,12 @@ runs: - name: Configure Occlum features run: | if [[ "${{ matrix.self_runner[2] }}" == "EDMM" ]]; then - docker exec ${{ env.CONTAINER_NAME }} bash -c "jq '.feature.enable_posix_shm = true | .feature.enable_edmm = true' /opt/occlum/etc/template/Occlum.json > /tmp.json && mv /tmp.json /opt/occlum/etc/template/Occlum.json" + if [[ "${{ matrix.self_runner[3] }}" == "IO_Uring" ]]; then + docker exec ${{ env.CONTAINER_NAME }} bash -c "jq '.feature.enable_posix_shm = true | .feature.enable_edmm = true | .feature.io_uring = 1' /opt/occlum/etc/template/Occlum.json > /tmp.json && mv /tmp.json /opt/occlum/etc/template/Occlum.json"; + else + docker exec ${{ env.CONTAINER_NAME }} bash -c "jq '.feature.enable_posix_shm = true | .feature.enable_edmm = true' /opt/occlum/etc/template/Occlum.json > /tmp.json && mv /tmp.json /opt/occlum/etc/template/Occlum.json"; + fi; + elif [[ "${{ matrix.features }}" == "IO_Uring" ]]; then + docker exec ${{ env.CONTAINER_NAME }} bash -c "jq '.feature.io_uring = 1' /opt/occlum/etc/template/Occlum.json > /tmp.json && mv /tmp.json /opt/occlum/etc/template/Occlum.json"; fi; - shell: bash + shell: bash \ No newline at end of file diff --git a/.github/workflows/demo_test.yml b/.github/workflows/demo_test.yml index e89229a48a..722bd2cce1 100644 --- a/.github/workflows/demo_test.yml +++ b/.github/workflows/demo_test.yml @@ -14,6 +14,9 @@ concurrency: jobs: C_cpp_rust_golang_embedded_mode_support_test: runs-on: ubuntu-20.04 + strategy: + matrix: + features: [Legacy, IO_Uring] steps: - name: Remove unnecessary files run: | @@ -112,6 +115,9 @@ jobs: Java_support_test: runs-on: ubuntu-20.04 + strategy: + matrix: + features: [Legacy, IO_Uring] steps: - name: Remove unnecessary files run: | @@ -146,6 +152,9 @@ jobs: Fish_test: runs-on: ubuntu-20.04 + strategy: + matrix: + features: [Legacy, IO_Uring] steps: - name: Remove unnecessary files run: | @@ -177,6 +186,9 @@ jobs: Bazel_test: runs-on: ubuntu-20.04 + strategy: + matrix: + features: [Legacy, IO_Uring] steps: - name: Remove unnecessary files run: | @@ -213,6 +225,9 @@ jobs: Https_server_test: runs-on: ubuntu-20.04 + strategy: + matrix: + features: [Legacy, IO_Uring] steps: - name: Remove unnecessary files run: | @@ -246,6 +261,9 @@ jobs: Local_attestation_test: runs-on: ubuntu-20.04 + strategy: + matrix: + features: [Legacy, IO_Uring] steps: - name: Remove unnecessary files run: | @@ -275,6 +293,9 @@ jobs: Sqlite_test: runs-on: ubuntu-20.04 + strategy: + matrix: + features: [Legacy, IO_Uring] steps: - name: Remove unnecessary files run: | @@ -303,6 +324,9 @@ jobs: Xgboost_test: runs-on: ubuntu-20.04 + strategy: + matrix: + features: [Legacy, IO_Uring] steps: - name: Remove unnecessary files run: | @@ -334,6 +358,9 @@ jobs: Tensorflow_lite_test: runs-on: ubuntu-20.04 + strategy: + matrix: + features: [Legacy, IO_Uring] steps: - name: Remove unnecessary files run: | @@ -365,6 +392,9 @@ jobs: Pytorch_test: runs-on: ubuntu-20.04 + strategy: + matrix: + features: [Legacy, IO_Uring] steps: - name: Remove unnecessary files run: | @@ -393,6 +423,9 @@ jobs: Distributed_Pytorch_test: runs-on: ubuntu-20.04 + strategy: + matrix: + features: [Legacy, IO_Uring] steps: - name: Remove unnecessary files run: | @@ -431,6 +464,9 @@ jobs: Tensorflow_test: runs-on: ubuntu-20.04 + strategy: + matrix: + features: [Legacy, IO_Uring] steps: - name: Remove unnecessary files run: | @@ -460,6 +496,9 @@ jobs: # Below tests needs test image to run faster Grpc_musl_test: runs-on: ubuntu-20.04 + strategy: + matrix: + features: [Legacy, IO_Uring] steps: - name: Remove unnecessary files run: | @@ -504,6 +543,9 @@ jobs: Grpc_glibc_test: runs-on: ubuntu-20.04 + strategy: + matrix: + features: [Legacy, IO_Uring] steps: - name: Remove unnecessary files run: | @@ -552,6 +594,9 @@ jobs: Grpc_tls_test: runs-on: ubuntu-20.04 + strategy: + matrix: + features: [Legacy, IO_Uring] steps: - name: Remove unnecessary files run: | @@ -593,6 +638,9 @@ jobs: Openvino_test: runs-on: ubuntu-20.04 + strategy: + matrix: + features: [Legacy, IO_Uring] steps: - name: Remove unnecessary files run: | @@ -630,6 +678,9 @@ jobs: # Python test also needs its own image because in Alpine environment, modules are built locally and consumes a lot of time. Python_musl_support_test: runs-on: ubuntu-20.04 + strategy: + matrix: + features: [Legacy, IO_Uring] steps: - name: Remove unnecessary files run: | @@ -670,6 +721,9 @@ jobs: # Python glibc support test Python_glibc_support_test: runs-on: ubuntu-20.04 + strategy: + matrix: + features: [Legacy, IO_Uring] steps: - name: Remove unnecessary files run: | @@ -706,6 +760,9 @@ jobs: # Redis test Redis_support_test: runs-on: ubuntu-20.04 + strategy: + matrix: + features: [Legacy, IO_Uring] steps: - name: Remove unnecessary files run: | @@ -745,6 +802,9 @@ jobs: Flink_test: runs-on: ubuntu-20.04 + strategy: + matrix: + features: [Legacy, IO_Uring] steps: - name: Remove unnecessary files run: | @@ -797,6 +857,9 @@ jobs: Cluster_serving_test: runs-on: ubuntu-20.04 + strategy: + matrix: + features: [Legacy, IO_Uring] steps: - name: Remove unnecessary files run: | @@ -825,6 +888,9 @@ jobs: Enclave_RA_TLS_test: runs-on: ubuntu-20.04 + strategy: + matrix: + features: [Legacy, IO_Uring] steps: - name: Remove unnecessary files run: | @@ -859,6 +925,9 @@ jobs: Vault_test: runs-on: ubuntu-20.04 + strategy: + matrix: + features: [Legacy, IO_Uring] steps: - name: Remove unnecessary files run: | @@ -896,6 +965,9 @@ jobs: Sofaboot_test: runs-on: ubuntu-20.04 + strategy: + matrix: + features: [Legacy, IO_Uring] steps: - name: Remove unnecessary files run: | @@ -936,6 +1008,9 @@ jobs: Netty_UT_test: runs-on: ubuntu-20.04 + strategy: + matrix: + features: [Legacy, IO_Uring] steps: - name: Remove unnecessary files run: | @@ -967,6 +1042,9 @@ jobs: Bash_test: runs-on: ubuntu-20.04 + strategy: + matrix: + features: [Legacy, IO_Uring] steps: - name: Remove unnecessary files run: | @@ -995,6 +1073,9 @@ jobs: Sysbench_test: runs-on: ubuntu-20.04 + strategy: + matrix: + features: [Legacy, IO_Uring] steps: - name: Remove unnecessary files run: | @@ -1027,6 +1108,9 @@ jobs: Gvisor_syscalls_test: runs-on: ubuntu-20.04 + strategy: + matrix: + features: [Legacy, IO_Uring] steps: - name: Remove unnecessary files run: | @@ -1062,10 +1146,19 @@ jobs: run: docker exec $gvisor_test bash -c "git clone https://github.com/occlum/gvisor.git" - name: Run gvisor syscall test - run: docker exec $gvisor_test bash -c "cd /root/gvisor/occlum && SGX_MODE=SIM ./run_occlum_passed_tests.sh" + run: | + if [[ "${{ matrix.features }}" == "IO_Uring" ]]; then + docker exec $gvisor_test bash -c "cd /root/gvisor/occlum && SGX_MODE=SIM ./run_occlum_passed_tests.sh uring"; + else + docker exec $gvisor_test bash -c "cd /root/gvisor/occlum && SGX_MODE=SIM ./run_occlum_passed_tests.sh"; + fi; + shell: bash Flask_tls_test: runs-on: ubuntu-20.04 + strategy: + matrix: + features: [Legacy, IO_Uring] steps: - name: Remove unnecessary files run: | @@ -1108,6 +1201,9 @@ jobs: Iperf2_test: runs-on: ubuntu-20.04 + strategy: + matrix: + features: [Legacy, IO_Uring] steps: - name: Remove unnecessary files run: | @@ -1143,6 +1239,9 @@ jobs: Linux_LTP_test: runs-on: ubuntu-20.04 + strategy: + matrix: + features: [Legacy, IO_Uring] steps: - name: Remove unnecessary files run: | @@ -1175,6 +1274,9 @@ jobs: FIO_test: runs-on: ubuntu-20.04 + strategy: + matrix: + features: [Legacy, IO_Uring] steps: - name: Remove unnecessary files run: | @@ -1203,6 +1305,9 @@ jobs: PaddlePaddle_test: runs-on: ubuntu-20.04 + strategy: + matrix: + features: [Legacy, IO_Uring] steps: - name: Remove unnecessary files run: | @@ -1231,6 +1336,9 @@ jobs: RuntimeBoot_test: runs-on: ubuntu-20.04 + strategy: + matrix: + features: [Legacy, IO_Uring] steps: - name: Remove unnecessary files run: | @@ -1259,6 +1367,9 @@ jobs: Swtpm_test: runs-on: ubuntu-20.04 + strategy: + matrix: + features: [Legacy, IO_Uring] steps: - name: Remove unnecessary files run: | @@ -1293,6 +1404,9 @@ jobs: Filebench_test: runs-on: ubuntu-20.04 + strategy: + matrix: + features: [Legacy, IO_Uring] steps: - uses: actions/checkout@v1 with: diff --git a/.github/workflows/hw_mode_test.yml b/.github/workflows/hw_mode_test.yml index 8d17942310..8fd3170d2a 100644 --- a/.github/workflows/hw_mode_test.yml +++ b/.github/workflows/hw_mode_test.yml @@ -25,7 +25,7 @@ jobs: runs-on: ${{ matrix.self_runner }} strategy: matrix: - self_runner: [[self-hosted, SGX2-HW], [self-hosted, SGX2-HW, EDMM]] + self_runner: [[self-hosted, SGX2-HW], [self-hosted, SGX2-HW, EDMM, IO_Uring]] steps: - name: Clean before running @@ -80,7 +80,7 @@ jobs: runs-on: ${{ matrix.self_runner }} strategy: matrix: - self_runner: [[self-hosted, SGX2-HW], [self-hosted, SGX2-HW, EDMM]] + self_runner: [[self-hosted, SGX2-HW], [self-hosted, SGX2-HW, EDMM, IO_Uring]] steps: - name: Clean before running @@ -179,7 +179,7 @@ jobs: runs-on: ${{ matrix.self_runner }} strategy: matrix: - self_runner: [[self-hosted, SGX2-HW], [self-hosted, SGX2-HW, EDMM, PKU]] + self_runner: [[self-hosted, SGX2-HW], [self-hosted, SGX2-HW, EDMM, IO_Uring, PKU]] steps: - name: Clean before running @@ -220,7 +220,7 @@ jobs: - name: Run hello PKU run: | - if [[ "${{ matrix.self_runner[3] }}" == "PKU" ]]; then + if [[ "${{ matrix.self_runner[4] }}" == "PKU" ]]; then docker exec ${{ env.CONTAINER_NAME }} bash -c "cd /root/occlum/demos/java && ./run_java_on_occlum.sh hello_pku" else echo "Skip PKU test" @@ -237,7 +237,7 @@ jobs: runs-on: ${{ matrix.self_runner }} strategy: matrix: - self_runner: [[self-hosted, SGX2-HW], [self-hosted, SGX2-HW, EDMM]] + self_runner: [[self-hosted, SGX2-HW], [self-hosted, SGX2-HW, EDMM, IO_Uring]] steps: - name: Clean before running @@ -289,7 +289,7 @@ jobs: runs-on: ${{ matrix.self_runner }} strategy: matrix: - self_runner: [[self-hosted, SGX2-HW], [self-hosted, SGX2-HW, EDMM]] + self_runner: [[self-hosted, SGX2-HW], [self-hosted, SGX2-HW, EDMM, IO_Uring]] steps: - name: Clean before running @@ -336,7 +336,7 @@ jobs: runs-on: ${{ matrix.self_runner }} strategy: matrix: - self_runner: [[self-hosted, SGX2-HW], [self-hosted, SGX2-HW, EDMM]] + self_runner: [[self-hosted, SGX2-HW], [self-hosted, SGX2-HW, EDMM, IO_Uring]] steps: - name: Clean before running @@ -383,7 +383,7 @@ jobs: runs-on: ${{ matrix.self_runner }} strategy: matrix: - self_runner: [[self-hosted, SGX2-HW], [self-hosted, SGX2-HW, EDMM]] + self_runner: [[self-hosted, SGX2-HW], [self-hosted, SGX2-HW, EDMM, IO_Uring]] steps: - name: Clean before running @@ -427,7 +427,7 @@ jobs: runs-on: ${{ matrix.self_runner }} strategy: matrix: - self_runner: [[self-hosted, SGX2-HW], [self-hosted, SGX2-HW, EDMM]] + self_runner: [[self-hosted, SGX2-HW], [self-hosted, SGX2-HW, EDMM, IO_Uring]] steps: - name: Clean before running @@ -505,7 +505,7 @@ jobs: runs-on: ${{ matrix.self_runner }} strategy: matrix: - self_runner: [[self-hosted, SGX2-HW], [self-hosted, SGX2-HW, EDMM]] + self_runner: [[self-hosted, SGX2-HW], [self-hosted, SGX2-HW, EDMM, IO_Uring]] steps: - name: Clean before running @@ -580,7 +580,7 @@ jobs: runs-on: ${{ matrix.self_runner }} strategy: matrix: - self_runner: [[self-hosted, SGX2-HW], [self-hosted, SGX2-HW, EDMM]] + self_runner: [[self-hosted, SGX2-HW], [self-hosted, SGX2-HW, EDMM, IO_Uring]] steps: - name: Clean before running @@ -677,7 +677,7 @@ jobs: runs-on: ${{ matrix.self_runner }} strategy: matrix: - self_runner: [[self-hosted, SGX2-HW], [self-hosted, SGX2-HW, EDMM]] + self_runner: [[self-hosted, SGX2-HW], [self-hosted, SGX2-HW, EDMM, IO_Uring]] steps: - name: Clean before running @@ -741,8 +741,15 @@ jobs: - name: Clone gvisor code run: docker exec $gvisor_test bash -c "git clone https://github.com/occlum/gvisor.git" + - name: Run gvisor syscall test - run: docker exec $gvisor_test bash -c "cd /root/gvisor/occlum; ./run_occlum_passed_tests.sh" + run: | + if [[ "${{ matrix.self_runner[3] }}" == "IO_Uring" ]]; then + docker exec $gvisor_test bash -c "cd /root/gvisor/occlum; ./run_occlum_passed_tests.sh uring"; + else + docker exec $gvisor_test bash -c "cd /root/gvisor/occlum; ./run_occlum_passed_tests.sh"; + fi; + shell: bash - name: Clean the environment if: ${{ always() }} @@ -920,7 +927,7 @@ jobs: runs-on: ${{ matrix.self_runner }} strategy: matrix: - self_runner: [[self-hosted, SGX2-HW], [self-hosted, SGX2-HW, EDMM]] + self_runner: [[self-hosted, SGX2-HW], [self-hosted, SGX2-HW, EDMM, IO_Uring]] steps: - name: Clean before running @@ -960,7 +967,7 @@ jobs: runs-on: ${{ matrix.self_runner }} strategy: matrix: - self_runner: [[self-hosted, SGX2-HW], [self-hosted, SGX2-HW, EDMM]] + self_runner: [[self-hosted, SGX2-HW], [self-hosted, SGX2-HW, EDMM, IO_Uring]] steps: - name: Clean before running @@ -1014,7 +1021,7 @@ jobs: runs-on: ${{ matrix.self_runner }} strategy: matrix: - self_runner: [[self-hosted, SGX2-HW], [self-hosted, SGX2-HW, EDMM]] + self_runner: [[self-hosted, SGX2-HW], [self-hosted, SGX2-HW, EDMM, IO_Uring]] steps: - name: Clean before running @@ -1069,7 +1076,7 @@ jobs: runs-on: ${{ matrix.self_runner }} strategy: matrix: - self_runner: [[self-hosted, SGX2-HW], [self-hosted, SGX2-HW, EDMM]] + self_runner: [[self-hosted, SGX2-HW], [self-hosted, SGX2-HW, EDMM, IO_Uring]] steps: - name: Clean before running @@ -1117,7 +1124,7 @@ jobs: runs-on: ${{ matrix.self_runner }} strategy: matrix: - self_runner: [[self-hosted, SGX2-HW], [self-hosted, SGX2-HW, EDMM]] + self_runner: [[self-hosted, SGX2-HW], [self-hosted, SGX2-HW, EDMM, IO_Uring]] steps: - name: Clean before running @@ -1163,7 +1170,7 @@ jobs: runs-on: ${{ matrix.self_runner }} strategy: matrix: - self_runner: [[self-hosted, SGX2-HW], [self-hosted, SGX2-HW, EDMM]] + self_runner: [[self-hosted, SGX2-HW], [self-hosted, SGX2-HW, EDMM, IO_Uring]] steps: - name: Clean before running @@ -1198,7 +1205,7 @@ jobs: runs-on: ${{ matrix.self_runner }} strategy: matrix: - self_runner: [[self-hosted, SGX2-HW], [self-hosted, SGX2-HW, EDMM]] + self_runner: [[self-hosted, SGX2-HW], [self-hosted, SGX2-HW, EDMM, IO_Uring]] steps: - name: Clean before running