Skip to content

Commit

Permalink
Add origin information to Checkbox JSON submission (new) (#1644)
Browse files Browse the repository at this point in the history
* Create a function to retrieve version and packaging information

* Use get_origin() to store origin information in submission JSON

* Switch version 2020-12 of schema (from draft-06)

This version is more up-to-date and allows to use conditions (if..then)

* Remove client_version and checkbox_version and add origin into the JSON schema

* Refactor Jinja2SessionStateExporter to set origin at init time

By setting the origin at init, it is more readable and easier to write
unit tests.

* Update unit tests to use the new origin information

* fix unit test + black formatting issues

* Add extra required fields for the packaging section of origin field

* Add "unknown" packaging type if none of the package check worked

* Handle CalledProcessError as well

FileNotFoundError is raised when the dpkg command is not available, but
otherwise subprocess.check_output will raise CalledProcessError, which
is handled in this commit.
  • Loading branch information
pieqq authored Dec 11, 2024
1 parent 027b98a commit 2504b78
Show file tree
Hide file tree
Showing 7 changed files with 209 additions and 17 deletions.
59 changes: 59 additions & 0 deletions checkbox-ng/plainbox/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,3 +54,62 @@ def get_version_string():
else:
version_string = "{} {}".format("Checkbox", __version__)
return version_string


def get_origin():
"""
Return a dictionary containing information such as the version and what
packaging method is being used (Python virtual environment, Snap or
Debian).
"""
import os
import subprocess

if os.getenv("SNAP_NAME"):
origin = {
"name": "Checkbox",
"version": __version__,
"packaging": {
"type": "snap",
"name": os.getenv("SNAP_NAME"),
"version": os.getenv("SNAP_VERSION"),
"revision": os.getenv("SNAP_REVISION"),
},
}
elif os.getenv("VIRTUAL_ENV"):
origin = {
"name": "Checkbox",
"version": __version__,
"packaging": {
"type": "source",
"version": __version__,
},
}
else:
try:
dpkg_info = subprocess.check_output(
["dpkg", "-S", __path__[0]], universal_newlines=True
)
# 'python3-checkbox-ng: /usr/lib/python3/dist-packages/plainbox\n'
package_name = dpkg_info.split(":")[0]
origin = {
"name": "Checkbox",
"version": __version__,
"packaging": {
"type": "debian",
"name": package_name,
"version": __version__,
},
}
# if all of the above failed and dpkg is not available on the system...
except (FileNotFoundError, subprocess.CalledProcessError):
origin = {
"name": "Checkbox",
"version": __version__,
"packaging": {
"type": "unknown",
"name": "unknown",
"version": "unknown",
},
}
return origin
9 changes: 5 additions & 4 deletions checkbox-ng/plainbox/impl/exporter/jinja2.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@


from plainbox import get_version_string
from plainbox import __version__ as checkbox_version
from plainbox import get_origin
from plainbox.abc import ISessionStateExporter
from plainbox.impl.exporter import SessionStateExporterBase
from plainbox.impl.result import OUTCOME_METADATA_MAP
Expand Down Expand Up @@ -94,6 +94,7 @@ def __init__(
timestamp=None,
client_version=None,
client_name="plainbox",
origin=None,
exporter_unit=None,
):
"""
Expand All @@ -110,7 +111,7 @@ def __init__(
self._client_version = client_version or get_version_string()
# Remember client name
self._client_name = client_name
self._checkbox_version = checkbox_version
self._origin = origin or get_origin()

self.option_list = None
self.template = None
Expand Down Expand Up @@ -197,7 +198,7 @@ def dump_from_session_manager(self, session_manager, stream):
"OUTCOME_METADATA_MAP": OUTCOME_METADATA_MAP,
"client_name": self._client_name,
"client_version": self._client_version,
"checkbox_version": self._checkbox_version,
"origin": self._origin,
"manager": session_manager,
"app_blob": app_blob_data,
"options": self.option_list,
Expand All @@ -223,7 +224,7 @@ def dump_from_session_manager_list(self, session_manager_list, stream):
"OUTCOME_METADATA_MAP": OUTCOME_METADATA_MAP,
"client_name": self._client_name,
"client_version": self._client_version,
"checkbox_version": self._checkbox_version,
"origin": self._origin,
"manager_list": session_manager_list,
"app_blob": {},
"options": self.option_list,
Expand Down
10 changes: 10 additions & 0 deletions checkbox-ng/plainbox/impl/exporter/test_html.py
Original file line number Diff line number Diff line change
Expand Up @@ -179,6 +179,11 @@ def test_perfect_match_without_certification_status(self):
system_id="",
timestamp="2012-12-21T12:00:00",
client_version="Checkbox 1.0",
origin={
"name": "Checkbox",
"version": "1.0",
"packaging": {"type": "source"},
},
exporter_unit=self.exporter_unit,
)
stream = io.BytesIO()
Expand All @@ -202,6 +207,11 @@ def test_perfect_match_with_certification_blocker(self):
system_id="",
timestamp="2012-12-21T12:00:00",
client_version="Checkbox 1.0",
origin={
"name": "Checkbox",
"version": "1.0",
"packaging": {"type": "source"},
},
exporter_unit=self.exporter_unit,
)
stream = io.BytesIO()
Expand Down
36 changes: 32 additions & 4 deletions checkbox-ng/plainbox/impl/exporter/test_jinja2.py
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,14 @@ def test_template(self):
exporter_unit.option_list = ()
with open(pathname, "w") as f:
f.write(tmpl)
exporter = Jinja2SessionStateExporter(exporter_unit=exporter_unit)
exporter = Jinja2SessionStateExporter(
origin={
"name": "Checkbox",
"version": "1.0",
"packaging": {"type": "source"},
},
exporter_unit=exporter_unit,
)
stream = BytesIO()
exporter.dump_from_session_manager(self.manager_single_job, stream)
expected_bytes = " fail : job name\n".encode("UTF-8")
Expand All @@ -95,7 +102,14 @@ def test_validation_chooses_json(self):
exporter_unit.data_dir = tmp
exporter_unit.template = template_filename
exporter_unit.option_list = ()
exporter = Jinja2SessionStateExporter(exporter_unit=exporter_unit)
exporter = Jinja2SessionStateExporter(
origin={
"name": "Checkbox",
"version": "1.0",
"packaging": {"type": "source"},
},
exporter_unit=exporter_unit,
)
exporter.validate_json = mock.Mock(return_value=[])
stream = BytesIO()
exporter.validate(stream)
Expand All @@ -114,7 +128,14 @@ def test_validation_json(self):
exporter_unit.data_dir = tmp
exporter_unit.template = template_filename
exporter_unit.option_list = ()
exporter = Jinja2SessionStateExporter(exporter_unit=exporter_unit)
exporter = Jinja2SessionStateExporter(
origin={
"name": "Checkbox",
"version": "1.0",
"packaging": {"type": "source"},
},
exporter_unit=exporter_unit,
)
stream = BytesIO()
exporter.dump_from_session_manager(self.manager_single_job, stream)

Expand All @@ -131,7 +152,14 @@ def test_validation_json_throws(self):
exporter_unit.data_dir = tmp
exporter_unit.template = template_filename
exporter_unit.option_list = ()
exporter = Jinja2SessionStateExporter(exporter_unit=exporter_unit)
exporter = Jinja2SessionStateExporter(
origin={
"name": "Checkbox",
"version": "1.0",
"packaging": {"type": "source"},
},
exporter_unit=exporter_unit,
)
stream = BytesIO()
with self.assertRaises(ExporterError):
exporter.dump_from_session_manager(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,7 @@
{%- set system_information = state.system_information -%}
{
"title": {{ state.metadata.title | jsonify | safe }},
"client_version": {{ client_version | jsonify | safe }},
"checkbox_version": {{ checkbox_version | jsonify | safe }},
"origin": {{ origin | jsonify | safe }},
{%- if "testplan_id" in app_blob %}
{%- if app_blob['testplan_id'] %}
"testplan_id": {{ app_blob['testplan_id'] | jsonify | safe }},
Expand Down
56 changes: 56 additions & 0 deletions checkbox-ng/plainbox/test_init.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
# This file is part of Checkbox.
#
# Copyright 2024 Canonical Ltd.
# Written by:
# Pierre Equoy <[email protected]>
#
# Checkbox is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 3,
# as published by the Free Software Foundation.
#
# Checkbox is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Checkbox. If not, see <http://www.gnu.org/licenses/>.

from unittest import TestCase, mock

import os
from subprocess import CalledProcessError

import plainbox


class PlainboxInitTests(TestCase):
@mock.patch.dict(os.environ, {"VIRTUAL_ENV": "test"})
def test_get_origin_venv(self):
origin = plainbox.get_origin()
self.assertEqual(origin["packaging"]["type"], "source")

@mock.patch.dict(os.environ, {"SNAP_NAME": "test"})
def test_get_origin_snap(self):
origin = plainbox.get_origin()
self.assertEqual(origin["packaging"]["type"], "snap")

@mock.patch.dict(os.environ, {}, clear=True)
@mock.patch("subprocess.check_output")
def test_get_origin_debian(self, mock_sp_check_output):
mock_sp_check_output.return_value = (
"python3-checkbox-ng: /usr/lib/python3/dist-packages/plainbox\n"
)
origin = plainbox.get_origin()
self.assertEqual(origin["packaging"]["type"], "debian")
self.assertEqual(origin["packaging"]["name"], "python3-checkbox-ng")

@mock.patch.dict(os.environ, {}, clear=True)
@mock.patch("subprocess.check_output")
def test_get_origin_exception(self, mock_sp_check_output):
mock_sp_check_output.side_effect = FileNotFoundError
origin = plainbox.get_origin()
self.assertEqual(origin["packaging"]["type"], "unknown")
mock_sp_check_output.side_effect = CalledProcessError(1, "error")
origin = plainbox.get_origin()
self.assertEqual(origin["packaging"]["type"], "unknown")
53 changes: 46 additions & 7 deletions submission-schema/schema.json
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
{
"$schema": "http://json-schema.org/draft-06/schema#",
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://certification.canonical.com/checkbox-submission.json",
"description": "Output format for the Checkbox test framework (see more info at https://checkbox.readthedocs.io)",
"$ref": "#/definitions/Submission",
Expand All @@ -11,12 +11,51 @@
"title": {
"type": "string"
},
"client_version": {
"type": "string"
},
"checkbox_version": {
"type": "string"
},
"origin": {
"type": "object",
"required": [
"name",
"version",
"packaging"
],
"properties": {
"name": {
"type": "string",
"description": "Name of the application used to run the test plan ('Checkbox')"
},
"version": {
"type": "string",
"description": "Version of Checkbox used to run the test plan."
},
"packaging": {
"type": "object",
"required": [
"type",
"version"
],
"properties": {
"type": {
"type": "string",
"description": "Packaging type ('debian', 'snap', or 'source' if running in a Python virtual env"
},
"name": {
"type": "string",
"description": "Name of the package used, if any"
},
"version": {
"type": "string",
"description": "Version installed"
},
"revision": {
"type": "string",
"description": "Revision installed (only for Snaps)"
}
},
"if": {"properties": {"type": {"const": "snap"}}},
"then": {"required": ["revision"]}
}
}
},
"testplan_id": {
"type": "string"
},
Expand Down

0 comments on commit 2504b78

Please sign in to comment.