Extend the wwan connection tests to support multiple cycle connection tests (New)

.github/actions/checkbox_source_deb/action.yaml

@@ -0,0 +1,83 @@
+name: Submit a Checkbox Test plan (or subset of it) to the lab
+inputs:
+  data_source:
+    description: "Target image and provisioning data (ex. `url:` or `distro:`)"
+    required: false
+    default: null
+  queue:
+    description: "Queue that will run the testing (ex. 202012-28526)"
+    required: true
+  test_plan:
+    description: "Test plan to run (ex. com.canonical.certification::sru)"
+    required: true
+  match:
+    description: "Subset of jobs to run (ex. .*wireless.*)"
+    required: false
+    default: ".*"
+  launcher_override:
+    description: "Launcher with additional values that will take priority over the defaults"
+    default: ""
+    required: false
+  checkbox_revision:
+    description: "Revision of checkbox that has to be provisioned (ex. commit_hash, branch name, can be `beta`)"
+    required: true
+  zapper_channel:
+    description: "Zapper channel to be used, will be ignored if no Zapper (ex. edge, beta, stable)"
+    required: false
+    default: "beta"
+runs:
+  using: composite
+  steps:
+    - uses: actions/checkout@v4
+    - name: Install dependencies
+      shell: bash
+      run: |
+        sudo apt install -y -qq gettext
+    - name: Build test resource
+      shell: bash
+      env:
+        INPUT_DATA_SOURCE: ${{ inputs.data_source }}
+        INPUT_QUEUE: ${{ inputs.queue }}
+        INPUT_MATCH: ${{ inputs.match || '.*' }}
+        INPUT_TEST_PLAN: ${{ inputs.test_plan }}
+        INPUT_LAUNCHER_OVERRIDE: ${{ inputs.launcher_override }}
+        INPUT_CHECKBOX_REVISION: ${{ inputs.checkbox_revision }}
+        INPUT_ZAPPER_CHANNEL: ${{ inputs.zapper_channel || 'beta' }}
+      working-directory: ${{ github.action_path }}/../../../tools/lab_dispatch
+      run: |
+        echo "::group::Building the testflinger job"
+        if [ -n "$INPUT_DATA_SOURCE" ]; then
+          INPUT_DATA_SOURCE="provision_data: $INPUT_DATA_SOURCE"
+        fi
+        envsubst '$INPUT_CHECKBOX_REVISION $INPUT_DATA_SOURCE $INPUT_QUEUE $INPUT_ZAPPER_CHANNEL' < generic_source.yaml | tee job.yaml
+        echo "::endgroup::"
+
+        echo "::group::Building the Checkbox launcher"
+        # this goes from .template. (missing secret, testplan, match etc. to .partial.)
+        # this is partial as some values are filled in on the agent (like wireless access points names)
+        envsubst '$INPUT_TEST_PLAN $INPUT_MATCH' < resources/checkbox.no-manifest.template.conf | tee resources/checkbox.no-manifest.partial.conf
+        echo "::endgroup::"
+
+        echo "::group::Dumping launcher overrides"
+        echo "$INPUT_LAUNCHER_OVERRIDE" | tee launcher_override.conf
+        echo "::endgroup::"
+    - name: Workaroud cwd
+      shell: bash
+      run: |
+        # this allows us to dispatch the action and the attachments with relative
+        # paths even when called form outside the Checkbox repo
+        action_path=$(realpath ${{ github.action_path }}/../../../tools/)
+        workdir_path=$(realpath tools/)
+        if [ ! -e "$workdir_path" ]; then
+          cp -rT "$action_path" "$workdir_path"
+        fi
+        if [ "$action_path" = "$workdir_path" ]; then
+          echo "Skipping copy as the action is already running in workdir"
+        else
+          cp -rT "$action_path" "$workdir_path"
+        fi
+    - name: Submit and monitor job
+      uses: canonical/testflinger/.github/actions/submit@main
+      with:
+        poll: true
+        job-path: tools/lab_dispatch/job.yaml

.github/workflows/dispatch_lab_job.yaml

@@ -9,6 +9,7 @@ on:
       # - queue: machine that will run the job (ex. 202012-28526)
       # - test_plan: Checkbox test plan to run (ex. com.canonical.certification::sru)
       # - match: subset of jobs to run (ex. .*wireless.*)
+      # - zapper_channel: refreshes the zapper snap to the channel if provided, default is beta (ex. "beta")
       #
       # One possible matrix_to_create would therefore look like this:
       # matrix_to_create=[{ data_source: "distro: desktop-22-04-2-uefi", queue: "202012-28526", match: ".*wireless.*", test_plan: "com.canonical.certification::sru" }]'
@@ -24,39 +25,34 @@ jobs:
   run-matrix:
     runs-on: [self-hosted, testflinger]
     strategy:
+      fail-fast: false
       matrix:
         spec: ${{ fromJson(inputs.matrix_to_create) }}
     defaults:
       run:
         working-directory: tools/lab_dispatch
     steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-      - name: Install dependencies
-        run: |
-          sudo apt install gettext
-      - name: Build test resource
-        env:
-          INPUT_DATA_SOURCE: ${{ matrix.spec.data_source }}
-          INPUT_QUEUE: ${{ matrix.spec.queue }}
-          INPUT_MATCH: ${{ matrix.spec.match }}
-          INPUT_TEST_PLAN: ${{ matrix.spec.test_plan }}
-          INPUT_PASSWORD_SECRET: ${{ secrets.INPUT_PASSWORD_SECRET }}
-        run: |
-          echo "::group::Building the testflinger job"
-          export INPUT_CHECKBOX_REVISION="$(git rev-parse HEAD)"
-          envsubst '$INPUT_CHECKBOX_REVISION $INPUT_DATA_SOURCE $INPUT_QUEUE' < generic_source.yaml | tee job.yaml
-          echo "::endgroup::"
 
-          echo "::group::Building the Checkbox launcher"
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Get current commit SHA
+        id: get_sha
+        run: echo "sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
 
-          # this goes from .template. (missing secret, testplan, match etc. to .partial.)
-          # this is partial as some values are filled in on the agent (like wireless access points names)
-          envsubst '$INPUT_TEST_PLAN $INPUT_MATCH $INPUT_PASSWORD_SECRET' < resources/checkbox.no-manifest.template.conf | tee resources/checkbox.no-manifest.partial.conf
-          echo "::endgroup::"
-      - name: Submit and monitor job
-        uses: canonical/testflinger/.github/actions/submit@main
+      - name: Run the spec
+        uses: canonical/checkbox/.github/actions/checkbox_source_deb@main
         with:
-          poll: true
-          job-path: tools/lab_dispatch/job.yaml
+          data_source: ${{ matrix.spec.data_source }}
+          queue: ${{ matrix.spec.queue }}
+          test_plan: ${{ matrix.spec.test_plan }}
+          match: ${{ matrix.spec.match }}
+          zapper_channel: ${{ matrix.spec.zapper_channel }}
+          launcher_override: |
+            [environment]
+            WPA_BG_PSK = ${{ secrets.INPUT_PASSWORD_SECRET }}
+            WPA_N_PSK = ${{ secrets.INPUT_PASSWORD_SECRET }}
+            WPA_AC_PSK = ${{ secrets.INPUT_PASSWORD_SECRET }}
+            WPA_AX_PSK = ${{ secrets.INPUT_PASSWORD_SECRET }}
+            WPA3_AX_PSK = ${{ secrets.INPUT_PASSWORD_SECRET }}
+          checkbox_revision: ${{ steps.get_sha.outputs.sha }}

.github/workflows/validate_workflows.yaml

@@ -16,7 +16,7 @@ jobs:
         uses: asdf-vm/actions/install@v3
         with:
           tool_versions: |
-            action-validator 0.5.1
+            action-validator 0.6.0
       - name: Lint Actions
         run: |
           find .github/workflows -type f \( -iname \*.yaml -o -iname \*.yml \) \

CONTRIBUTING.md

@@ -27,7 +27,7 @@ install everything you need in a Python virtual environment.
 
 Install the required tools:
 
-    $ sudo apt install git python3-virtualenv
+    $ sudo apt install git python3-virtualenv libasound2-dev
 
 Prepare the development environment. If you are an external contributor and
 plan on submitting some changes, you will have to [fork the Checkbox repository

checkbox-ng/checkbox_ng/launcher/subcommands.py

@@ -24,8 +24,8 @@
 from collections import defaultdict
 from string import Formatter
 from tempfile import TemporaryDirectory
-import textwrap
 import fnmatch
+import itertools
 import contextlib
 import gettext
 import json
@@ -1324,6 +1324,37 @@
             help=_("output format: 'text' or 'json' (default: %(default)s)"),
         )
 
+    def _get_relevant_manifest_units(self, jobs_and_templates_list):
+        """
+        Get all manifest units that are cited in the jobs_and_templates_list
+        resource expressions
+        """
+        # get all manifest units
+        manifest_units = filter(
+            lambda unit: unit.unit == "manifest entry",
+            self.sa._context.unit_list,
+        )
+        # get all jobs/templates that have a requires and do require a manifest
+        # entry
+        job_requires = [
+            requires
+            for requires in map(
+                lambda x: x.get_record_value("requires"),
+                jobs_and_templates_list,
+            )
+            if requires and "manifest" in requires
+        ]
+
+        # only return manifest entries that are actually required by any job in
+        # the list
+        return filter(
+            lambda manifest_unit: any(
+                "manifest.{}".format(manifest_unit.partial_id) in require
+                for require in job_requires
+            ),
+            manifest_units,
+        )
+
     def invoked(self, ctx):
         self.ctx = ctx
         session_title = "checkbox-expand-{}".format(ctx.args.TEST_PLAN)
@@ -1340,31 +1371,48 @@
         tp = self.sa._context._test_plan_list[0]
         tp_us = TestPlanUnitSupport(tp)
         self.override_list = tp_us.override_list
+
         jobs_and_templates_list = select_units(
             all_jobs_and_templates,
             [tp.get_mandatory_qualifier()] + [tp.get_qualifier()],
         )
+        relevant_manifest_units = self._get_relevant_manifest_units(
+            jobs_and_templates_list
+        )
 
+        units_to_print = itertools.chain(
+            relevant_manifest_units, iter(jobs_and_templates_list)
+        )
         obj_list = []
-        for unit in jobs_and_templates_list:
+        for unit in units_to_print:
             obj = unit._raw_data.copy()
             obj["unit"] = unit.unit
             obj["id"] = unit.id  # To get the fully qualified id
-            obj["certification-status"] = (
-                self.get_effective_certification_status(unit)
-            )
-            if unit.template_id:
-                obj["template-id"] = unit.template_id
+            # these two don't make sense for manifest units
+            if unit.unit != "manifest entry":
+                obj["certification-status"] = (
+                    self.get_effective_certification_status(unit)
+                )
+                if unit.template_id:
+                    obj["template-id"] = unit.template_id
             obj_list.append(obj)
-        obj_list.sort(key=lambda x: x.get("template-id", x["id"]))
+
+        obj_list.sort(key=lambda x: x.get("template-id", x["id"]) or x["id"])
+
         if ctx.args.format == "json":
-            print(json.dumps(obj_list, sort_keys=True))
+            json.dump(obj_list, sys.stdout, sort_keys=True)
         else:
             for obj in obj_list:
                 if obj["unit"] == "template":
                     print("Template '{}'".format(obj["template-id"]))
-                else:
+                elif obj["unit"] == "manifest entry":
+                    print("Manifest '{}'".format(obj["id"]))
+                elif obj["unit"] == "job":
                     print("Job '{}'".format(obj["id"]))
+                else:
+                    raise AssertionError(
+                        "Unknown unit type {}".format(obj["unit"])
+                    )
 
     def get_effective_certification_status(self, unit):
         if unit.unit == "template":

checkbox-ng/checkbox_ng/launcher/test_subcommands.py

@@ -813,6 +813,16 @@ def setUp(self):
         self.launcher = Expand()
         self.ctx = Mock()
         self.ctx.args = Mock(TEST_PLAN="", format="")
+
+        selected_1 = Mock(unit="manifest entry", id="some", partial_id="some")
+        selected_1._raw_data.copy.return_value = {}
+        selected_2 = Mock(
+            unit="manifest entry", id="other", partial_id="other"
+        )
+        selected_2._raw_data.copy.return_value = {}
+        not_selected = Mock(unit="manifest entry", partial_id="not_selected")
+        not_selected._raw_data.copy.return_value = {}
+
         self.ctx.sa = Mock(
             start_new_session=Mock(),
             get_test_plans=Mock(return_value=["test-plan1", "test-plan2"]),
@@ -821,6 +831,7 @@ def setUp(self):
             _context=Mock(
                 state=Mock(unit_list=[]),
                 _test_plan_list=[Mock()],
+                unit_list=[selected_1, selected_2, not_selected],
             ),
         )
 
@@ -844,17 +855,22 @@ def test_invoke__text(self, mock_select_units, mock_tpus, stdout):
                 "template-id": "test-template",
                 "id": "test-{res}",
                 "template-summary": "Test Template Summary",
+                "requires": "manifest.some == 'True'",
             }
         )
         job1 = JobDefinition(
             {
                 "id": "job1",
+                "requires": "manifest.other == 'Other'",
             }
         )
         mock_select_units.return_value = [job1, template1]
         self.ctx.args.TEST_PLAN = "test-plan1"
         self.launcher.invoked(self.ctx)
         self.assertIn("Template 'test-template'", stdout.getvalue())
+        self.assertIn("Manifest 'some'", stdout.getvalue())
+        self.assertIn("Manifest 'other'", stdout.getvalue())
+        self.assertNotIn("Manifest 'not_selected'", stdout.getvalue())
 
     @patch("sys.stdout", new_callable=StringIO)
     @patch("checkbox_ng.launcher.subcommands.TestPlanUnitSupport")
@@ -865,18 +881,24 @@ def test_invoke__json(self, mock_select_units, mock_tpus, stdout):
                 "template-id": "test-template",
                 "id": "test-{res}",
                 "template-summary": "Test Template Summary",
+                "requires": "manifest.some == 'True'",
             }
         )
         job1 = JobDefinition(
             {
                 "id": "job1",
+                "requires": "manifest.other == 'Other'",
             }
         )
+
         mock_select_units.return_value = [job1, template1]
         self.ctx.args.TEST_PLAN = "test-plan1"
         self.ctx.args.format = "json"
         self.launcher.invoked(self.ctx)
         self.assertIn('"template-id": "test-template"', stdout.getvalue())
+        self.assertIn('"id": "some"', stdout.getvalue())
+        self.assertIn('"id": "other"', stdout.getvalue())
+        self.assertNotIn('"id": "not_selected"', stdout.getvalue())
 
     def test_get_effective_certificate_status(self):
         job1 = JobDefinition(

docs/how-to/index.rst

@@ -13,4 +13,5 @@ These how-to guides cover key operations and processes in Checkbox.
    side-loading
    agent-service
    freeze-checkbox-version
-   launcher/*
+   launcher/*
+   using-match