diff --git a/docs/run_test/runbook.rst b/docs/run_test/runbook.rst index db33647a60..d4dbb3158b 100644 --- a/docs/run_test/runbook.rst +++ b/docs/run_test/runbook.rst @@ -50,6 +50,7 @@ Runbook Reference - `batch combinator <#batch-combinator>`__ - `items <#items-1>`__ + - `bisect combinator <#bisect-combinator>`__ - `notifier <#notifier>`__ @@ -571,6 +572,30 @@ For example, - image: CentOS vm_size: Standard_DS3_v2 + +bisect combinator +^^^^^^^^^^^^^^^^^ + +Specify a git repo url, the good commit and bad commit. The combinator +performs bisect operations on VM specified under 'connection'. + +The runbook will be iterated until the bisect operations completes. + +For example, + +.. code:: yaml + + combinator: + type: git_bisect + repo: $(repo_url) + bad_commit: $(bad_commit) + good_commit: $(good_commit) + connection: + address: $(bisect_vm_address) + private_key_file: $(admin_private_key_file) + +Refer `Sample runbook `__ + notifier ~~~~~~~~ diff --git a/lisa/combinators/git_bisect_combinator.py b/lisa/combinators/git_bisect_combinator.py new file mode 100644 index 0000000000..2df2abce5e --- /dev/null +++ b/lisa/combinators/git_bisect_combinator.py @@ -0,0 +1,191 @@ +import pathlib +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional, Type + +from dataclasses_json import dataclass_json + +from lisa import messages, notifier, schema +from lisa.combinator import Combinator +from lisa.messages import KernelBuildMessage, TestResultMessage, TestStatus +from lisa.node import Node, quick_connect +from lisa.tools.git import Git, GitBisect +from lisa.util import LisaException, constants, field_metadata + +STOP_PATTERNS = ["first bad commit", "This means the bug has been fixed between"] + + +# Combinator requires a node to clone the source code. +@dataclass_json() +@dataclass +class GitBisectCombinatorSchema(schema.Combinator): + connection: Optional[schema.RemoteNode] = field( + default=None, metadata=field_metadata(required=True) + ) + repo: str = field( + default="", + metadata=field_metadata( + required=True, + ), + ) + good_commit: str = field( + default="", + metadata=field_metadata( + required=True, + ), + ) + bad_commit: str = field( + default="", + metadata=field_metadata( + required=True, + ), + ) + + +# GitBisect Combinator is a loop that runs "expanded" phase +# of runbook until the bisect is complete. +# There can be any number of expanded phases, but the +# GitBisectTestResult notifier should have on boolean/None output per +# phase. + + +class GitBisectCombinator(Combinator): + def __init__( + self, + runbook: GitBisectCombinatorSchema, + **kwargs: Any, + ) -> None: + super().__init__(runbook) + self._iteration = 0 + self._result_notifier = GitBisectResult(schema.Notifier()) + notifier.register_notifier(self._result_notifier) + self._source_path: pathlib.PurePath + self._node: Optional[Node] = None + + def _initialize(self, *args: Any, **kwargs: Any) -> None: + self._clone_source() + if self._source_path: + self._start_bisect() + else: + raise LisaException( + "Source path is not set. Please check the source clone." + ) + + @classmethod + def type_name(cls) -> str: + return constants.COMBINATOR_GITBISECT + + @classmethod + def type_schema(cls) -> Type[schema.TypedSchema]: + return GitBisectCombinatorSchema + + def _next(self) -> Optional[Dict[str, Any]]: + _next: Optional[Dict[str, Any]] = None + self._process_result() + if not self._check_bisect_complete(): + _next = {} + _next["ref"] = self._get_current_commit_hash() + else: + self._log.info("Bisect Complete") + self._result_notifier.result = None + self._iteration += 1 + return _next + + def _process_result(self) -> None: + if self._iteration == 0: + return + if self._result_notifier.result is not None: + results = self._result_notifier.result + if results: + self._bisect_good() + else: + self._bisect_bad() + else: + raise LisaException( + "Bisect combinator does not get result for next iteration. Please check" + " GitBisectResult notifier." + ) + + def _get_remote_node(self) -> Node: + if not self._node or not self._node.is_connected: + self._node = quick_connect(self.runbook.connection, "source_node") + return self._node + + def _clone_source(self) -> None: + node = self._get_remote_node() + git = node.tools[Git] + self._source_path = git.clone( + url=self.runbook.repo, cwd=node.working_path, timeout=1200 + ) + node.close() + + def _start_bisect(self) -> None: + node = self._get_remote_node() + git_bisect = node.tools[GitBisect] + git_bisect.start(cwd=self._source_path) + git_bisect.good(cwd=self._source_path, ref=self.runbook.good_commit) + git_bisect.bad(cwd=self._source_path, ref=self.runbook.bad_commit) + node.close() + + def _bisect_bad(self) -> None: + node = self._get_remote_node() + git_bisect = node.tools[GitBisect] + git_bisect.bad(cwd=self._source_path) + node.close() + + def _bisect_good(self) -> None: + node = self._get_remote_node() + git_bisect = node.tools[GitBisect] + git_bisect.good(cwd=self._source_path) + node.close() + + def _check_bisect_complete(self) -> bool: + node = self._get_remote_node() + git_bisect = node.tools[GitBisect] + result = git_bisect.check_bisect_complete(cwd=self._source_path) + node.close() + return result + + def _get_current_commit_hash(self) -> str: + node = self._get_remote_node() + git = node.tools[Git] + result = git.get_current_commit_hash(cwd=self._source_path) + node.close() + return result + + +class GitBisectResult(notifier.Notifier): + @classmethod + def type_name(cls) -> str: + return "git_bisect_result" + + @classmethod + def type_schema(cls) -> Type[schema.TypedSchema]: + return schema.Notifier + + def _initialize(self, *args: Any, **kwargs: Any) -> None: + self.result: Optional[bool] = None + + def _received_message(self, message: messages.MessageBase) -> None: + if isinstance(message, messages.TestResultMessage): + self._update_test_result(message) + elif isinstance(message, messages.KernelBuildMessage): + self._update_result(message.is_success) + else: + raise LisaException(f"Received unsubscribed message type: {type(message)}") + + def _update_test_result(self, message: messages.TestResultMessage) -> None: + if message.is_completed: + if message.status == TestStatus.FAILED: + self._update_result(False) + elif message.status == TestStatus.PASSED: + self._update_result(True) + + def _update_result(self, result: bool) -> None: + current_result = self.result + if current_result is not None: + self.result = current_result and result + else: + self.result = result + + def _subscribed_message_type(self) -> List[Type[messages.MessageBase]]: + return [TestResultMessage, KernelBuildMessage] diff --git a/lisa/messages.py b/lisa/messages.py index c4d03cffa9..7caba6e874 100644 --- a/lisa/messages.py +++ b/lisa/messages.py @@ -263,6 +263,15 @@ class ProvisionBootTimeMessage(MessageBase): information: Dict[str, str] = field(default_factory=dict) +@dataclass +class KernelBuildMessage(MessageBase): + type: str = "KernelBuild" + old_kernel_version: str = "" + new_kernel_version: str = "" + is_success: bool = False + error_message: str = "" + + def _is_completed_status(status: TestStatus) -> bool: return status in [ TestStatus.FAILED, diff --git a/lisa/mixin_modules.py b/lisa/mixin_modules.py index 446d00bf7b..d624d7298e 100644 --- a/lisa/mixin_modules.py +++ b/lisa/mixin_modules.py @@ -8,6 +8,7 @@ import lisa.combinators.batch_combinator # noqa: F401 import lisa.combinators.csv_combinator # noqa: F401 +import lisa.combinators.git_bisect_combinator # noqa: F401 import lisa.combinators.grid_combinator # noqa: F401 import lisa.notifiers.console # noqa: F401 import lisa.notifiers.env_stats # noqa: F401 diff --git a/lisa/transformers/kernel_installer.py b/lisa/transformers/kernel_installer.py index c6cba144b5..5524773045 100644 --- a/lisa/transformers/kernel_installer.py +++ b/lisa/transformers/kernel_installer.py @@ -6,10 +6,11 @@ from dataclasses import dataclass, field from typing import Any, Dict, List, Optional, Type, cast -from assertpy import assert_that +from assertpy.assertpy import assert_that from dataclasses_json import dataclass_json -from lisa import schema +from lisa import notifier, schema +from lisa.messages import KernelBuildMessage from lisa.node import Node, quick_connect from lisa.operating_system import Posix, Ubuntu from lisa.secret import PATTERN_HEADTAIL, add_secret @@ -73,6 +74,7 @@ class KernelInstallerTransformerSchema(schema.Transformer): installer: Optional[BaseInstallerSchema] = field( default=None, metadata=field_metadata(required=True) ) + raise_exception: Optional[bool] = True class BaseInstaller(subclasses.BaseClassWithRunbookMixin): @@ -101,6 +103,8 @@ def install(self) -> str: class KernelInstallerTransformer(Transformer): _information_output_name = "information" + _is_success_output_name = "is_success" + _information: Dict[str, Any] = dict() @classmethod @@ -120,6 +124,10 @@ def _internal_run(self) -> Dict[str, Any]: assert runbook.connection, "connection must be defined." assert runbook.installer, "installer must be defined." + message = KernelBuildMessage() + build_sucess: bool = False + boot_success: bool = False + node = quick_connect(runbook.connection, "installer_node") uname = node.tools[Uname] @@ -133,24 +141,68 @@ def _internal_run(self) -> Dict[str, Any]: ) installer.validate() - installed_kernel_version = installer.install() - self._information = installer.information - self._log.info(f"installed kernel version: {installed_kernel_version}") - - # for ubuntu cvm kernel, there is no menuentry added into grub file - if hasattr(installer.runbook, "source"): - if installer.runbook.source != "linux-image-azure-fde": - posix = cast(Posix, node.os) - posix.replace_boot_kernel(installed_kernel_version) - - self._log.info("rebooting") - node.reboot() - kernel_version_after_install = uname.get_linux_information(force_run=True) - self._log.info(f"kernel version after install: {kernel_version_after_install}") - assert_that( - kernel_version_after_install, "Kernel installation Failed" - ).is_not_equal_to(kernel_version_before_install) - return {self._information_output_name: self._information} + + try: + message.old_kernel_version = uname.get_linux_information( + force_run=True + ).kernel_version_raw + + installed_kernel_version = installer.install() + build_sucess = True + self._information = installer.information + self._log.info(f"installed kernel version: {installed_kernel_version}") + + # for ubuntu cvm kernel, there is no menuentry added into grub file + if hasattr(installer.runbook, "source"): + if installer.runbook.source != "linux-image-azure-fde": + posix = cast(Posix, node.os) + posix.replace_boot_kernel(installed_kernel_version) + else: + efi_files = node.execute( + "ls -t /usr/lib/linux/efi/kernel.efi-*-azure-cvm", + sudo=True, + shell=True, + expected_exit_code=0, + expected_exit_code_failure_message=( + "fail to find kernel.efi file for kernel type " + " linux-image-azure-fde" + ), + ) + efi_file = efi_files.stdout.splitlines()[0] + node.execute( + ( + "cp /boot/efi/EFI/ubuntu/grubx64.efi " + "/boot/efi/EFI/ubuntu/grubx64.efi.bak" + ), + sudo=True, + ) + node.execute( + f"cp {efi_file} /boot/efi/EFI/ubuntu/grubx64.efi", + sudo=True, + shell=True, + ) + + self._log.info("rebooting") + node.reboot() + boot_success = True + new_kernel_version = uname.get_linux_information(force_run=True) + message.new_kernel_version = new_kernel_version.kernel_version_raw + self._log.info(f"kernel version after install: " f"{new_kernel_version}") + assert_that( + new_kernel_version.kernel_version_raw, "Kernel installation Failed" + ).is_not_equal_to(kernel_version_before_install.kernel_version_raw) + except Exception as e: + message.error_message = str(e) + if runbook.raise_exception: + raise e + self._log.info(f"Kernel build failed: {e}") + finally: + message.is_success = build_sucess and boot_success + notifier.notify(message) + return { + self._information_output_name: self._information, + self._is_success_output_name: build_sucess and boot_success, + } class RepoInstaller(BaseInstaller): diff --git a/lisa/util/constants.py b/lisa/util/constants.py index c5f58ad036..f9b87c0820 100644 --- a/lisa/util/constants.py +++ b/lisa/util/constants.py @@ -84,6 +84,7 @@ COMBINATOR = "combinator" COMBINATOR_GRID = "grid" COMBINATOR_BATCH = "batch" +COMBINATOR_GITBISECT = "git_bisect" ENVIRONMENT = "environment" ENVIRONMENTS = "environments" diff --git a/microsoft/runbook/examples/git_bisect.yml b/microsoft/runbook/examples/git_bisect.yml new file mode 100644 index 0000000000..d0d51dfb64 --- /dev/null +++ b/microsoft/runbook/examples/git_bisect.yml @@ -0,0 +1,95 @@ +name: git_bisect +extension: + - "../../testsuites" + +include: + - path: ../tiers/tier.yml + - path: ../azure.yml + +variable: + - name: subscription_id + value: "" + - name: tier + value: 0 + - name: test_case_name + value: "smoke_test" + - name: marketplace_image + value: "canonical ubuntuserver 18.04-lts latest" + - name: repo_url + value: "git://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable.git" + - name: good_commit + value: "" + - name: bad_commit + value: "" + + # Do not assign values to below variables. These are used by the combinator. + - name: build_vm_address + value: "" + - name: bisect_vm_address + value: "" + - name: build_vm_resource_group_name + value: "" + - name: bisect_vm_resource_group_name + value: "" + - name: vhd + value: "" + - name: kernel_installer_is_success + value: False + - name: ref + value: "" + +transformer: + - type: azure_deploy + name: bisect_vm + requirement: + azure: + marketplace: $(marketplace_image) + location: $(location) + core_count: 2 + enabled: true + - type: azure_deploy + phase: expanded + name: build_vm + requirement: + azure: + marketplace: $(marketplace_image) + location: $(location) + core_count: 16 + enabled: true + - type: kernel_installer + phase: expanded + connection: + address: $(build_vm_address) + private_key_file: $(admin_private_key_file) + installer: + type: source + location: + type: repo + path: /mnt/code + ref: $(ref) + repo: $(repo_url) + raise_exception: False + rename: + kernel_installer_is_success: enable_tests + # Do not create vhd when build fails + - type: azure_vhd + enabled: $(enable_tests) + phase: expanded + resource_group_name: $(build_vm_resource_group_name) + rename: + azure_vhd_url: vhd + - type: azure_delete + resource_group_name: $(build_vm_resource_group_name) + phase: expanded_cleanup + - type: azure_delete + resource_group_name: $(bisect_vm_resource_group_name) + phase: cleanup + +combinator: + type: git_bisect + repo: $(repo_url) + bad_commit: $(bad_commit) + good_commit: $(good_commit) + connection: + address: $(bisect_vm_address) + private_key_file: $(admin_private_key_file) diff --git a/microsoft/runbook/tiers/tdebug.yml b/microsoft/runbook/tiers/tdebug.yml index 0cda0878cb..e0e0d4e268 100644 --- a/microsoft/runbook/tiers/tdebug.yml +++ b/microsoft/runbook/tiers/tdebug.yml @@ -15,3 +15,4 @@ testcase: retry: $(retry) use_new_environment: $(use_new_environment) ignore_failure: $(ignore_failure) + enabled: $(enable_tests)