From 8ed34981ea4c99e587b1c0612c3151cf6d6d9a3b Mon Sep 17 00:00:00 2001 From: Hook25 Date: Fri, 6 Sep 2024 16:34:57 +0200 Subject: [PATCH 1/5] Also print manifest to expand --- .../checkbox_ng/launcher/subcommands.py | 33 +++++++++++++++++-- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/checkbox-ng/checkbox_ng/launcher/subcommands.py b/checkbox-ng/checkbox_ng/launcher/subcommands.py index 55d4666f80..d50472621e 100644 --- a/checkbox-ng/checkbox_ng/launcher/subcommands.py +++ b/checkbox-ng/checkbox_ng/launcher/subcommands.py @@ -1344,7 +1344,28 @@ def invoked(self, ctx): all_jobs_and_templates, [tp.get_mandatory_qualifier()] + [tp.get_qualifier()], ) - + manifest_units = filter( + lambda unit: unit.unit == "manifest entry", + self.sa._context.unit_list, + ) + requires = list( + filter( + lambda x: x and "manifest" in x, + map( + lambda x: x.get_record_value("requires"), + jobs_and_templates_list, + ), + ) + ) + manifest_units = list( + filter( + lambda x: any( + x.id.rsplit(":", 1)[-1] in require for require in requires + ), + manifest_units, + ) + ) + jobs_and_templates_list = manifest_units + jobs_and_templates_list obj_list = [] for unit in jobs_and_templates_list: obj = unit._raw_data.copy() @@ -1353,7 +1374,7 @@ def invoked(self, ctx): obj["certification-status"] = ( self.get_effective_certification_status(unit) ) - if unit.template_id: + if hasattr(unit, "template_id") and unit.template_id: obj["template-id"] = unit.template_id obj_list.append(obj) obj_list.sort(key=lambda x: x.get("template-id", x["id"])) @@ -1363,8 +1384,14 @@ def invoked(self, ctx): for obj in obj_list: if obj["unit"] == "template": print("Template '{}'".format(obj["template-id"])) - else: + elif obj["unit"] == "manifest entry": + print("Manifest '{}'".format(obj["id"])) + elif obj["unit"] == "job": print("Job '{}'".format(obj["id"])) + else: + raise AssertionError( + "Unknown unit type {}".format(obj["unit"]) + ) def get_effective_certification_status(self, unit): if unit.unit == "template": From fe1eb99b51785a408917f17658fefb8610970f1d Mon Sep 17 00:00:00 2001 From: Hook25 Date: Thu, 19 Sep 2024 17:06:50 +0200 Subject: [PATCH 2/5] Document and partially de-functionalize code --- .../checkbox_ng/launcher/subcommands.py | 76 ++++++++++++------- 1 file changed, 48 insertions(+), 28 deletions(-) diff --git a/checkbox-ng/checkbox_ng/launcher/subcommands.py b/checkbox-ng/checkbox_ng/launcher/subcommands.py index d50472621e..20c1d0374a 100644 --- a/checkbox-ng/checkbox_ng/launcher/subcommands.py +++ b/checkbox-ng/checkbox_ng/launcher/subcommands.py @@ -24,8 +24,8 @@ from collections import defaultdict from string import Formatter from tempfile import TemporaryDirectory -import textwrap import fnmatch +import itertools import contextlib import gettext import json @@ -1324,6 +1324,37 @@ def register_arguments(self, parser): help=_("output format: 'text' or 'json' (default: %(default)s)"), ) + def _get_relevant_manifest_units(self, jobs_and_templates_list): + """ + Get all manifest units that are cited in the jobs_and_templates_list + resource expressions + """ + # get all manifest units + manifest_units = filter( + lambda unit: unit.unit == "manifest entry", + self.sa._context.unit_list, + ) + # get all jobs/templates that have a requires and do require a manifest + # entry + job_requires = [ + requires + for requires in map( + lambda x: x.get_record_value("requires"), + jobs_and_templates_list, + ) + if requires and "manifest" in requires + ] + + # only return manifest entries that are actually required by any job in + # the list + return filter( + lambda manifest_unit: any( + "manifest.{}".format(manifest_unit.partial_id) in require + for require in job_requires + ), + manifest_units, + ) + def invoked(self, ctx): self.ctx = ctx session_title = "checkbox-expand-{}".format(ctx.args.TEST_PLAN) @@ -1340,46 +1371,35 @@ def invoked(self, ctx): tp = self.sa._context._test_plan_list[0] tp_us = TestPlanUnitSupport(tp) self.override_list = tp_us.override_list + jobs_and_templates_list = select_units( all_jobs_and_templates, [tp.get_mandatory_qualifier()] + [tp.get_qualifier()], ) - manifest_units = filter( - lambda unit: unit.unit == "manifest entry", - self.sa._context.unit_list, - ) - requires = list( - filter( - lambda x: x and "manifest" in x, - map( - lambda x: x.get_record_value("requires"), - jobs_and_templates_list, - ), - ) + relevant_manifest_units = self._get_relevant_manifest_units( + jobs_and_templates_list ) - manifest_units = list( - filter( - lambda x: any( - x.id.rsplit(":", 1)[-1] in require for require in requires - ), - manifest_units, - ) + + units_to_print = itertools.chain( + relevant_manifest_units, iter(jobs_and_templates_list) ) - jobs_and_templates_list = manifest_units + jobs_and_templates_list obj_list = [] - for unit in jobs_and_templates_list: + for unit in units_to_print: obj = unit._raw_data.copy() obj["unit"] = unit.unit obj["id"] = unit.id # To get the fully qualified id - obj["certification-status"] = ( - self.get_effective_certification_status(unit) - ) - if hasattr(unit, "template_id") and unit.template_id: + # these two don't make sense for manifest units + if unit.unit != "manifest entry": + obj["certification-status"] = ( + self.get_effective_certification_status(unit) + ) obj["template-id"] = unit.template_id obj_list.append(obj) - obj_list.sort(key=lambda x: x.get("template-id", x["id"])) + + obj_list.sort(key=lambda x: x.get("template-id", x["id"]) or x["id"]) + if ctx.args.format == "json": - print(json.dumps(obj_list, sort_keys=True)) + json.dump(obj_list, sys.stdout, sort_keys=True) else: for obj in obj_list: if obj["unit"] == "template": From d9627b169b0333d4d40ff9de2ead8e7362662513 Mon Sep 17 00:00:00 2001 From: Hook25 Date: Mon, 23 Sep 2024 18:21:02 +0200 Subject: [PATCH 3/5] Add tests for the new export units --- .../checkbox_ng/launcher/test_subcommands.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/checkbox-ng/checkbox_ng/launcher/test_subcommands.py b/checkbox-ng/checkbox_ng/launcher/test_subcommands.py index 949a5b9004..6bce13ecc7 100644 --- a/checkbox-ng/checkbox_ng/launcher/test_subcommands.py +++ b/checkbox-ng/checkbox_ng/launcher/test_subcommands.py @@ -813,6 +813,16 @@ def setUp(self): self.launcher = Expand() self.ctx = Mock() self.ctx.args = Mock(TEST_PLAN="", format="") + + selected_1 = Mock(unit="manifest entry", id="some", partial_id="some") + selected_1._raw_data.copy.return_value = {} + selected_2 = Mock( + unit="manifest entry", id="other", partial_id="other" + ) + selected_2._raw_data.copy.return_value = {} + not_selected = Mock(unit="manifest entry", partial_id="not_selected") + not_selected._raw_data.copy.return_value = {} + self.ctx.sa = Mock( start_new_session=Mock(), get_test_plans=Mock(return_value=["test-plan1", "test-plan2"]), @@ -821,6 +831,7 @@ def setUp(self): _context=Mock( state=Mock(unit_list=[]), _test_plan_list=[Mock()], + unit_list=[selected_1, selected_2, not_selected], ), ) @@ -865,18 +876,24 @@ def test_invoke__json(self, mock_select_units, mock_tpus, stdout): "template-id": "test-template", "id": "test-{res}", "template-summary": "Test Template Summary", + "requires": "manifest.some == 'True'", } ) job1 = JobDefinition( { "id": "job1", + "requires": "manifest.other == 'Other'", } ) + mock_select_units.return_value = [job1, template1] self.ctx.args.TEST_PLAN = "test-plan1" self.ctx.args.format = "json" self.launcher.invoked(self.ctx) self.assertIn('"template-id": "test-template"', stdout.getvalue()) + self.assertIn('"id": "some"', stdout.getvalue()) + self.assertIn('"id": "other"', stdout.getvalue()) + self.assertNotIn('"id": "not_selected"', stdout.getvalue()) def test_get_effective_certificate_status(self): job1 = JobDefinition( From add8c6e2b1e1c99814bf5be076338a1d8ad9b229 Mon Sep 17 00:00:00 2001 From: Hook25 Date: Mon, 23 Sep 2024 18:30:40 +0200 Subject: [PATCH 4/5] Also test text exporter for manifest exporting --- checkbox-ng/checkbox_ng/launcher/test_subcommands.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/checkbox-ng/checkbox_ng/launcher/test_subcommands.py b/checkbox-ng/checkbox_ng/launcher/test_subcommands.py index 6bce13ecc7..466411a766 100644 --- a/checkbox-ng/checkbox_ng/launcher/test_subcommands.py +++ b/checkbox-ng/checkbox_ng/launcher/test_subcommands.py @@ -855,17 +855,22 @@ def test_invoke__text(self, mock_select_units, mock_tpus, stdout): "template-id": "test-template", "id": "test-{res}", "template-summary": "Test Template Summary", + "requires": "manifest.some == 'True'", } ) job1 = JobDefinition( { "id": "job1", + "requires": "manifest.other == 'Other'", } ) mock_select_units.return_value = [job1, template1] self.ctx.args.TEST_PLAN = "test-plan1" self.launcher.invoked(self.ctx) self.assertIn("Template 'test-template'", stdout.getvalue()) + self.assertIn("Manifest 'some'", stdout.getvalue()) + self.assertIn("Manifest 'other'", stdout.getvalue()) + self.assertNotIn("Manifest 'not_selected'", stdout.getvalue()) @patch("sys.stdout", new_callable=StringIO) @patch("checkbox_ng.launcher.subcommands.TestPlanUnitSupport") From 570c8c60d8e6b91c423df74dffcfbffb9bcfb8dc Mon Sep 17 00:00:00 2001 From: Hook25 Date: Tue, 24 Sep 2024 12:49:45 +0200 Subject: [PATCH 5/5] Check null tempalte-id --- checkbox-ng/checkbox_ng/launcher/subcommands.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/checkbox-ng/checkbox_ng/launcher/subcommands.py b/checkbox-ng/checkbox_ng/launcher/subcommands.py index 20c1d0374a..a5d3ef559b 100644 --- a/checkbox-ng/checkbox_ng/launcher/subcommands.py +++ b/checkbox-ng/checkbox_ng/launcher/subcommands.py @@ -1393,7 +1393,8 @@ def invoked(self, ctx): obj["certification-status"] = ( self.get_effective_certification_status(unit) ) - obj["template-id"] = unit.template_id + if unit.template_id: + obj["template-id"] = unit.template_id obj_list.append(obj) obj_list.sort(key=lambda x: x.get("template-id", x["id"]) or x["id"])