Circi dynamic config (#2763)

Circleci
* build cmake armgcc and arm clang on circleci
* use docker medium+

.circleci/config.yml

@@ -1,92 +1,39 @@
 version: 2.1
 
-commands:
-  setup-toolchain:
-    parameters:
-        toolchain:
-            type: string
-        toolchain_url:
-            type: string
-    steps:
-#      - run:
-#          name: Make toolchain cache key
-#          command: echo "<< parameters.toolchain >>-<< parameters.toolchain_url>>" > toolchain_key
-#      - restore_cache:
-#          name: Restore Toolchain Cache
-#          key: deps-{{ checksum "toolchain_key" }}
-#          paths:
-#            - ~/cache/<< parameters.toolchain >>
-      - run:
-          name: Install Toolchain
-          command: |
-            # Only download if folder does not exist (not cached)
-            if [ ! -d ~/cache/<< parameters.toolchain >> ]; then
-              mkdir -p ~/cache/<< parameters.toolchain >>
-              wget << parameters.toolchain_url>> -O toolchain.tar.gz
-              tar -C ~/cache/<< parameters.toolchain >> -xaf toolchain.tar.gz
-            fi
-#      - save_cache:
-#          name: Save Toolchain Cache
-#          key: deps-{{ checksum "toolchain_key" }}
-#          paths:
-#            - ~/cache/<< parameters.toolchain >>
-      - run:
-          name: Setup build environment
-          command: |
-            echo "export PATH=$PATH:`echo ~/cache/<< parameters.toolchain >>/*/bin`" >> $BASH_ENV
-            # Install Ninja
-            NINJA_URL=https://github.com/ninja-build/ninja/releases/download/v1.12.1/ninja-linux.zip
-            wget $NINJA_URL -O ninja-linux.zip
-            unzip ninja-linux.zip -d ~/bin
-
-  get-deps:
-    parameters:
-      family:
-        type: string
-    steps:
-      - run:
-          name: Get Dependencies
-          command: |
-            python tools/get_deps.py << parameters.family >>
+setup: true
+orbs:
+  continuation: circleci/continuation@1
 
 jobs:
-  arm-clang:
-    parameters:
-      family:
-        type: string
-      build-system:
-        type: string
-
+  set-matrix:
+    executor: continuation/default
     docker:
       - image: cimg/base:current
-    resource_class: medium
-    environment:
-      TOOLCHAIN_URL: https://github.com/ARM-software/LLVM-embedded-toolchain-for-Arm/releases/download/release-17.0.1/LLVMEmbeddedToolchainForArm-17.0.1-Linux-x86_64.tar.xz
+    resource_class: small
     steps:
       - checkout
-      - setup-toolchain:
-          toolchain: clang
-          toolchain_url: $TOOLCHAIN_URL
-      - get-deps:
-          family: << parameters.family >>
       - run:
-          name: Build
+          name: Set matrix
           command: |
-            # Only build one board per family for non PRs i.e commit to master
-            ONE_PER_FAMILY=""
-            if [ -z "$CIRCLE_PULL_REQUEST" ]; then
-              ONE_PER_FAMILY="--one-per-family"
-            fi
-            python tools/build.py $ONE_PER_FAMILY -s << parameters.build-system >> --toolchain clang << parameters.family >>
+            MATRIX_JSON=$(python .github/workflows/ci_set_matrix.py)
+            echo "MATRIX_JSON=$MATRIX_JSON"
+
+            TOOLCHAIN_LIST=("arm-clang" "arm-gcc")
+            for toolchain in "${TOOLCHAIN_LIST[@]}"; do
+              FAMILY=$(echo $MATRIX_JSON | jq -r ".\"$toolchain\".family")
+              echo "${toolchain}_FAMILY=$FAMILY"
+              echo "      - build:" >> .circleci/config2.yml
+              echo "          matrix:" >> .circleci/config2.yml
+              echo "            parameters:" >> .circleci/config2.yml
+              echo "              toolchain: ['$toolchain']" >> .circleci/config2.yml
+              echo "              build-system: ['cmake']" >> .circleci/config2.yml
+              echo "              family: $FAMILY" >> .circleci/config2.yml
+            done
+
+      - continuation/continue:
+          configuration_path: .circleci/config2.yml
 
 workflows:
-  build:
+  set-matrix:
     jobs:
-      - arm-clang:
-          matrix:
-            parameters:
-              build-system:
-                - cmake
-              #family: ['stm32f1']
-              #family: ['stm32f1', 'stm32f2']
-              family: ['imxrt', 'kinetis_k kinetis_kl kinetis_k32l2', 'lpc11 lpc13 lpc15', 'lpc17 lpc18 lpc40 lpc43', 'lpc51 lpc54 lpc55', 'nrf', 'samd11 samd21 saml2x', 'samd5x_e5x samg', 'stm32f0 stm32f1 stm32f2 stm32f3', 'stm32f4', 'stm32f7', 'stm32g0 stm32g4 stm32h5', 'stm32h7', 'stm32l4 stm32u5 stm32wb']
+      - set-matrix

.circleci/config2.yml

@@ -0,0 +1,98 @@
+version: 2.1
+
+commands:
+  setup-toolchain:
+    parameters:
+        toolchain:
+            type: string
+    steps:
+      - run:
+          name: Install Toolchain
+          command: |
+            TOOLCHAIN_JSON='{
+              "arm-clang": "https://github.com/ARM-software/LLVM-embedded-toolchain-for-Arm/releases/download/release-17.0.1/LLVMEmbeddedToolchainForArm-17.0.1-Linux-x86_64.tar.xz",
+              "arm-gcc": "https://github.com/xpack-dev-tools/arm-none-eabi-gcc-xpack/releases/download/v12.3.1-1.1/xpack-arm-none-eabi-gcc-12.3.1-1.1-linux-x64.tar.gz"
+            }'
+            toolchain_url=$(echo $TOOLCHAIN_JSON | jq -r '.["<< parameters.toolchain >>"]')
+            echo "toolchain_url=$toolchain_url"
+
+            # download and extract toolchain
+            mkdir -p ~/cache/<< parameters.toolchain >>
+            wget $toolchain_url -O toolchain.tar.gz
+            tar -C ~/cache/<< parameters.toolchain >> -xaf toolchain.tar.gz
+
+            # Add toolchain to PATH
+            echo "export PATH=$PATH:`echo ~/cache/<< parameters.toolchain >>/*/bin`" >> $BASH_ENV
+
+  get-deps:
+    parameters:
+      family:
+        type: string
+    steps:
+      - run:
+          name: Get Dependencies
+          command: |
+            python tools/get_deps.py << parameters.family >>
+
+            # Install Pico SDK
+            if [ << parameters.family >> == "rp2040" ]; then
+              git clone --depth 1 https://github.com/raspberrypi/pico-sdk.git ~/pico-sdk
+              echo "export PICO_SDK_PATH=~/pico-sdk" >> $BASH_ENV
+            fi
+
+jobs:
+  build:
+    parameters:
+      build-system:
+        type: string
+      toolchain:
+        type: string
+      family:
+        type: string
+
+    docker:
+      - image: cimg/base:current
+    resource_class: medium+
+    steps:
+      - checkout
+      - when:
+          condition: << parameters.build-system >> == 'cmake'
+          steps:
+            - run:
+                name: Install Ninja
+                command: |
+                  # Install Ninja
+                  NINJA_URL=https://github.com/ninja-build/ninja/releases/download/v1.12.1/ninja-linux.zip
+                  wget $NINJA_URL -O ninja-linux.zip
+                  unzip ninja-linux.zip -d ~/bin
+      - setup-toolchain:
+          toolchain: << parameters.toolchain >>
+      - get-deps:
+          family: << parameters.family >>
+      - run:
+          name: Build
+          command: |
+            # Only build one board per family for non PRs i.e commit to master
+            ONE_PER_FAMILY=""
+            if [ -z "$CIRCLE_PULL_REQUEST" ]; then
+              ONE_PER_FAMILY="--one-per-family"
+            fi
+
+            # Toolchain option default is gcc
+            if [ "<< parameters.toolchain >>" == "arm-clang" ]; then
+              TOOLCHAIN_OPTION="--toolchain clang"
+            elif [ "<< parameters.toolchain >>" == "arm-gcc" ]; then
+              TOOLCHAIN_OPTION="--toolchain gcc"
+            fi
+
+            python tools/build.py $ONE_PER_FAMILY -s << parameters.build-system >> $TOOLCHAIN_OPTION << parameters.family >>
+
+workflows:
+  build:
+    jobs:
+#      - build:
+#          matrix:
+#            parameters:
+#              toolchain: ['arm-clang']
+#              build-system: ['cmake']
+#              family: ['imxrt']

.github/workflows/build.yml

@@ -58,9 +58,8 @@ jobs:
       fail-fast: false
       matrix:
         toolchain:
-          # - 'arm-clang' is built by circle-ci
+          # - 'arm-clang' is built by circle-ci in PR
           - 'aarch64-gcc'
-          - 'arm-gcc'
           - 'msp430-gcc'
           - 'riscv-gcc'
     with:
@@ -69,6 +68,20 @@ jobs:
       build-args: ${{ toJSON(fromJSON(needs.set-matrix.outputs.json)[matrix.toolchain].family) }}
       one-per-family: ${{ github.event_name != 'pull_request' }}
 
+  # ---------------------------------------
+  # Build CMake arm-gcc
+  # only build with push, for PR: all board is built by circle-ci
+  # ---------------------------------------
+  cmake-arm-gcc:
+    if: github.event_name == 'push'
+    needs: set-matrix
+    uses: ./.github/workflows/build_util.yml
+    with:
+      build-system: 'cmake'
+      toolchain: 'arm-gcc'
+      build-args: ${{ toJSON(fromJSON(needs.set-matrix.outputs.json)['arm-gcc'].family) }}
+      one-per-family: true
+
   # ---------------------------------------
   # Build Make
   # ---------------------------------------
@@ -80,7 +93,7 @@ jobs:
       fail-fast: false
       matrix:
         toolchain:
-          # 'arm-clang' is built by circle-ci
+          # 'arm-clang' would be built by circle-ci
           - 'aarch64-gcc'
           - 'arm-gcc'
           - 'msp430-gcc'

tools/build_utils.py

@@ -81,6 +81,19 @@ def skip_example(example, board):
     return False
 
 
+def build_size(make_cmd):
+    size_output = subprocess.run(make_cmd + ' size', shell=True, stdout=subprocess.PIPE).stdout.decode("utf-8").splitlines()
+    for i, l in enumerate(size_output):
+        text_title = 'text	   data	    bss	    dec'
+        if text_title in l:
+            size_list = size_output[i+1].split('\t')
+            flash_size = int(size_list[0])
+            sram_size = int(size_list[1]) + int(size_list[2])
+            return (flash_size, sram_size)
+
+    return (0, 0)
+
+
 def build_example(example, board, make_option):
     start_time = time.monotonic()
     flash_size = "-"
@@ -89,22 +102,20 @@ def build_example(example, board, make_option):
     # succeeded, failed, skipped
     ret = [0, 0, 0]
 
-    make_cmd = "make -j -C examples/{} BOARD={} {}".format(example, board, make_option)
+    make_cmd = f"make -j -C examples/{example} BOARD={board} {make_option}"
 
     # Check if board is skipped
     if skip_example(example, board):
         status = SKIPPED
         ret[2] = 1
         print(build_format.format(example, board, status, '-', flash_size, sram_size))
     else:
-        #subprocess.run(make_cmd + " clean", shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-        build_result = subprocess.run(make_cmd + " all", shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+        build_result = subprocess.run(f"{make_cmd} all", shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
 
         if build_result.returncode == 0:
             status = SUCCEEDED
             ret[0] = 1
             (flash_size, sram_size) = build_size(make_cmd)
-            #subprocess.run(make_cmd + " copy-artifact", shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
         else:
             status = FAILED
             ret[1] = 1
@@ -116,16 +127,3 @@ def build_example(example, board, make_option):
             print(build_result.stdout.decode("utf-8"))
 
     return ret
-
-
-def build_size(make_cmd):
-    size_output = subprocess.run(make_cmd + ' size', shell=True, stdout=subprocess.PIPE).stdout.decode("utf-8").splitlines()
-    for i, l in enumerate(size_output):
-        text_title = 'text	   data	    bss	    dec'
-        if text_title in l:
-            size_list = size_output[i+1].split('\t')
-            flash_size = int(size_list[0])
-            sram_size = int(size_list[1]) + int(size_list[2])
-            return (flash_size, sram_size)
-
-    return (0, 0)