diff --git a/backend/kernelCI_app/helpers/hardwareDetails.py b/backend/kernelCI_app/helpers/hardwareDetails.py index 1c887968..53ec313a 100644 --- a/backend/kernelCI_app/helpers/hardwareDetails.py +++ b/backend/kernelCI_app/helpers/hardwareDetails.py @@ -20,9 +20,29 @@ from kernelCI_app.helpers.misc import env_misc_value_or_default, handle_environment_misc from kernelCI_app.helpers.trees import get_tree_heads from kernelCI_app.models import Tests -from kernelCI_app.typeModels.databases import FAIL_STATUS -from kernelCI_app.typeModels.hardwareDetails import DefaultRecordValues, PostBody, Tree -from kernelCI_app.utils import create_issue, extract_error_message, is_boot +from kernelCI_app.typeModels.databases import FAIL_STATUS, NULL_STATUS +from kernelCI_app.typeModels.commonDetails import ( + BuildArchitectures, + BuildStatusCount, + BuildSummary, + Misc, + TestArchSummaryItem, + TestHistoryItem, + TestStatusCount, + TestSummary, +) +from kernelCI_app.typeModels.hardwareDetails import ( + DefaultRecordValues, + HardwareBuildHistoryItem, + PostBody, + Tree, +) +from kernelCI_app.utils import ( + convert_issues_dict_to_list_typed, + create_issue, + extract_error_message, + is_boot, +) from pydantic import ValidationError from django.db.models import Subquery from kernelCI_app.typeModels.hardwareDetails import ( @@ -209,7 +229,7 @@ def get_hardware_details_data( def query_records( - *, hardware_id, origin, trees: List[Tree], start_date=int, end_date=int + *, hardware_id: str, origin: str, trees: List[Tree], start_date=int, end_date=int ): commit_hashes = [tree.head_git_commit_hash for tree in trees] @@ -257,6 +277,7 @@ def query_records( ) +# deprecated, use get_arch_summary_typed def get_arch_summary(record: Dict) -> Dict: return { "arch": record["build__architecture"], @@ -265,6 +286,15 @@ def get_arch_summary(record: Dict) -> Dict: } +def get_arch_summary_typed(record: Dict) -> TestArchSummaryItem: + return TestArchSummaryItem( + arch=record["build__architecture"], + compiler=record["build__compiler"], + status=TestStatusCount(), + ) + + +# deprecated, use get_build_typed def get_build(record: Dict, tree_idx: int) -> Dict: return { "id": record["build_id"], @@ -286,6 +316,27 @@ def get_build(record: Dict, tree_idx: int) -> Dict: } +def get_build_typed(record: Dict, tree_idx: int) -> HardwareBuildHistoryItem: + return HardwareBuildHistoryItem( + id=record["build_id"], + architecture=record["build__architecture"], + config_name=record["build__config_name"], + misc=record["build__misc"], + config_url=record["build__config_url"], + compiler=record["build__compiler"], + valid=record["build__valid"], + duration=record["build__duration"], + log_url=record["build__log_url"], + start_time=record["build__start_time"], + git_repository_url=record["build__checkout__git_repository_url"], + git_repository_branch=record["build__checkout__git_repository_branch"], + tree_index=tree_idx, + tree_name=record["build__checkout__tree_name"], + issue_id=record["build__incidents__issue__id"], + issue_version=record["build__incidents__issue__version"], + ) + + def get_tree_key(record: Dict) -> str: return ( record["build__checkout__tree_name"] @@ -294,16 +345,6 @@ def get_tree_key(record: Dict) -> str: ) -def get_tree(record: Dict) -> Dict[str, str]: - return { - "tree_name": record["build__checkout__tree_name"], - "git_repository_branch": record["build__checkout__git_repository_branch"], - "git_repository_url": record["build__checkout__git_repository_url"], - "git_commit_name": record["build__checkout__git_commit_name"], - "git_commit_hash": record["build__checkout__git_commit_hash"], - } - - def get_history(record: Dict): return { "id": record["id"], @@ -374,7 +415,29 @@ def generate_test_dict() -> Dict[str, any]: } -def generate_tree_status_summary_dict() -> Dict[str]: +def generate_build_summary_typed() -> BuildSummary: + return BuildSummary( + status=BuildStatusCount(), + architectures={}, + configs={}, + issues=[], + unknown_issues=0, + ) + + +def generate_test_summary_typed() -> TestSummary: + return TestSummary( + status=TestStatusCount(), + architectures=[], + configs={}, + issues=[], + unknown_issues=0, + fail_reasons={}, + failed_platforms=[], + ) + + +def generate_tree_status_summary_dict() -> Dict[str, defaultdict[int]]: return { "builds": defaultdict(int), "boots": defaultdict(int), @@ -399,14 +462,20 @@ def handle_tree_status_summary( tree_status_summary[tree_index]["builds"][build_status] += 1 -def handle_test_or_boot(record: Dict, task: Dict) -> None: - status = record["status"] - +def create_record_test_platform(*, record: Dict) -> str: environment_misc = handle_environment_misc(record["environment_misc"]) test_platform = env_misc_value_or_default(environment_misc).get("platform") - record["test_platform"] = test_platform + return test_platform + + +# deprecated, use handle_test_history and handle_test_summary separately instead, with typing +def handle_test_or_boot(record: Dict, task: Dict) -> None: + status = record["status"] + + test_platform = create_record_test_platform(record=record) + task["history"].append(get_history(record)) task["statusSummary"][status] += 1 task["configs"][record["build__config_name"]][status] += 1 @@ -437,6 +506,125 @@ def handle_test_or_boot(record: Dict, task: Dict) -> None: ) +def handle_test_history( + *, + record: Dict, + task: List[TestHistoryItem], +) -> None: + create_record_test_platform(record=record) + + test_history_item = TestHistoryItem( + id=record["id"], + status=record["status"], + duration=record["duration"], + path=record["path"], + start_time=record["start_time"], + environment_compatible=record["environment_compatible"], + config=record["build__config_name"], + log_url=record["log_url"], + architecture=record["build__architecture"], + compiler=record["build__compiler"], + misc=Misc(platform=record["test_platform"]), + ) + + task.append(test_history_item) + + +def handle_test_summary( + *, + record: Dict, + task: TestSummary, + processed_issues: Dict, + processed_archs: Dict[str, TestArchSummaryItem], +) -> None: + status = record["status"] + + if status is None: + status = NULL_STATUS + + setattr(task.status, status, getattr(task.status, status) + 1) + + config_name = record["build__config_name"] + if task.configs.get(config_name) is None: + task.configs[config_name] = TestStatusCount() + setattr( + task.configs[config_name], + status, + getattr(task.configs[config_name], status) + 1, + ) + + environment_misc = handle_environment_misc(record["environment_misc"]) + test_platform = env_misc_value_or_default(environment_misc).get("platform") + if task.platforms is None: + task.platforms = {} + if task.platforms.get(test_platform) is None: + task.platforms[test_platform] = TestStatusCount() + setattr( + task.platforms[test_platform], + status, + getattr(task.platforms[test_platform], status) + 1, + ) + + arch_key = f'{record["build__architecture"]}{record["build__compiler"]}' + arch_summary = processed_archs.get(arch_key) + if not arch_summary: + arch_summary = get_arch_summary_typed(record) + processed_archs[arch_key] = arch_summary + setattr(arch_summary.status, status, getattr(arch_summary.status, status) + 1) + + process_issue(record=record, task_issues_dict=processed_issues, issue_from="test") + + +def handle_build_summary( + *, + record: Dict, + builds_summary: BuildSummary, + processed_issues: Dict, + tree_index: int, +) -> None: + build: HardwareBuildHistoryItem = get_build_typed(record, tree_idx=tree_index) + + # TODO: use build_status_map values or BuildStatusCount keys + status_key: Literal["valid", "invalid", "null"] = build_status_map.get(build.valid) + setattr( + builds_summary.status, + status_key, + getattr(builds_summary.status, status_key) + 1, + ) + + if config := build.config_name: + build_config_summary = builds_summary.configs.get(config) + if not build_config_summary: + build_config_summary = BuildStatusCount() + builds_summary.configs[config] = build_config_summary + setattr( + builds_summary.configs[config], + status_key, + getattr(builds_summary.configs[config], status_key) + 1, + ) + + if arch := build.architecture: + build_arch_summary = builds_summary.architectures.get(arch) + if not build_arch_summary: + build_arch_summary = BuildArchitectures() + builds_summary.architectures[arch] = build_arch_summary + setattr( + builds_summary.architectures[arch], + status_key, + getattr(builds_summary.architectures[arch], status_key) + 1, + ) + + compiler = build.compiler + if ( + compiler is not None + and compiler not in builds_summary.architectures.get(arch).compilers + ): + builds_summary.architectures[arch].compilers.append(compiler) + + process_issue(record=record, task_issues_dict=processed_issues, issue_from="build") + + +# deprecated, use handle_build_history and handle_build_summary separately instead, with typing def handle_build(*, instance, record: Dict, build: Dict) -> None: instance.builds["items"].append(build) update_issues( @@ -452,6 +640,27 @@ def handle_build(*, instance, record: Dict, build: Dict) -> None: ) +def process_issue( + *, record, task_issues_dict: Dict, issue_from: Literal["build", "test"] +) -> None: + if issue_from == "build": + is_failed_task = record["build__valid"] is not True + else: + is_failed_task = record["status"] == FAIL_STATUS + + update_issues( + issue_id=record["incidents__issue__id"], + issue_version=record["incidents__issue__version"], + incident_test_id=record["incidents__test_id"], + build_valid=record["build__valid"], + issue_comment=record["incidents__issue__comment"], + issue_report_url=record["incidents__issue__report_url"], + is_failed_task=is_failed_task, + issue_from=issue_from, + task=task_issues_dict, + ) + + # TODO unify with treeDetails def update_issues( *, @@ -533,8 +742,12 @@ def decide_if_is_build_in_filter( def decide_if_is_test_in_filter( - *, instance, test_type: PossibleTestType, record: Dict + *, instance, test_type: PossibleTestType, record: Dict, processed_tests: Set[str] ) -> bool: + is_test_processed = record["id"] in processed_tests + if is_test_processed: + return False + test_filter_pass = True status = record["status"] @@ -624,3 +837,24 @@ def assign_default_record_values(record: Dict) -> None: record["build__incidents__issue__id"] = UNKNOWN_STRING if record["incidents__issue__id"] is None and record["status"] == FAIL_STATUS: record["incidents__issue__id"] = UNKNOWN_STRING + + +def format_issue_summary_for_response( + *, + builds_summary: BuildSummary, + boots_summary: TestSummary, + tests_summary: TestSummary, + processed_issues: Dict, +) -> None: + builds_summary.issues = convert_issues_dict_to_list_typed( + issues_dict=processed_issues["build"]["issues"] + ) + boots_summary.issues = convert_issues_dict_to_list_typed( + issues_dict=processed_issues["boot"]["issues"] + ) + tests_summary.issues = convert_issues_dict_to_list_typed( + issues_dict=processed_issues["test"]["issues"] + ) + builds_summary.unknown_issues = processed_issues["build"]["failedWithUnknownIssues"] + boots_summary.unknown_issues = processed_issues["boot"]["failedWithUnknownIssues"] + tests_summary.unknown_issues = processed_issues["test"]["failedWithUnknownIssues"] diff --git a/backend/kernelCI_app/typeModels/databases.py b/backend/kernelCI_app/typeModels/databases.py index 0ef7f3f2..0ae022de 100644 --- a/backend/kernelCI_app/typeModels/databases.py +++ b/backend/kernelCI_app/typeModels/databases.py @@ -6,6 +6,7 @@ ERROR_STATUS = "ERROR" MISS_STATUS = "MISS" PASS_STATUS = "PASS" +NULL_STATUS = "NULL" failure_status_list = [ERROR_STATUS, FAIL_STATUS, MISS_STATUS] diff --git a/backend/kernelCI_app/typeModels/hardwareDetails.py b/backend/kernelCI_app/typeModels/hardwareDetails.py index 93f32d30..f515b6c8 100644 --- a/backend/kernelCI_app/typeModels/hardwareDetails.py +++ b/backend/kernelCI_app/typeModels/hardwareDetails.py @@ -79,6 +79,14 @@ class HardwareDetailsFullResponse(BaseModel): summary: HardwareSummary -type HardwareTreeList = List[Dict[str, str]] +class HardwareDetailsSummaryResponse(BaseModel): + summary: HardwareSummary + PossibleTestType = Literal["test", "boot"] + + +class HardwareBuildHistoryItem(BuildHistoryItem): + tree_name: Optional[str] + issue_id: Optional[str] + issue_version: Optional[str] diff --git a/backend/kernelCI_app/urls.py b/backend/kernelCI_app/urls.py index 800f0daa..1777c95b 100644 --- a/backend/kernelCI_app/urls.py +++ b/backend/kernelCI_app/urls.py @@ -81,6 +81,10 @@ def viewCache(view): viewCache(views.HardwareDetailsCommitHistoryView), name="hardwareDetailsCommitHistory" ), + path("hardware//summary", + views.HardwareDetailsSummary.as_view(), + name="hardwareDetailsSummary" + ), path("hardware/", viewCache(views.HardwareView), name="hardware"), diff --git a/backend/kernelCI_app/utils.py b/backend/kernelCI_app/utils.py index cc5c84e8..3b409039 100644 --- a/backend/kernelCI_app/utils.py +++ b/backend/kernelCI_app/utils.py @@ -4,7 +4,7 @@ from datetime import timedelta from kernelCI_app.helpers.logger import log_message -from kernelCI_app.typeModels.issues import Issue +from kernelCI_app.typeModels.issues import IncidentInfo, Issue DEFAULT_QUERY_TIME_INTERVAL = {"days": 7} @@ -25,10 +25,28 @@ def create_issue( } +# deprecated, use convert_issues_dict_to_list_typed instead and use type validation def convert_issues_dict_to_list(issues_dict: Dict[str, Issue]) -> List[Issue]: return list(issues_dict.values()) +def convert_issues_dict_to_list_typed(*, issues_dict: Dict) -> List[Issue]: + issues: List[Issue] = [] + for issue in issues_dict.values(): + issues.append( + Issue( + id=issue["id"], + version=issue["version"], + comment=issue["comment"], + report_url=issue["report_url"], + incidents_info=IncidentInfo( + incidentsCount=issue["incidents_info"]["incidentsCount"], + ) + ) + ) + return issues + + # TODO misc is not stable and should be used as a POC only def extract_error_message(misc: Union[str, dict, None]): parsedEnv = None diff --git a/backend/kernelCI_app/views/hardwareDetailsSummaryView.py b/backend/kernelCI_app/views/hardwareDetailsSummaryView.py new file mode 100644 index 00000000..749bbfff --- /dev/null +++ b/backend/kernelCI_app/views/hardwareDetailsSummaryView.py @@ -0,0 +1,270 @@ +from collections import defaultdict +from datetime import datetime +from django.utils.decorators import method_decorator +from django.views.decorators.csrf import csrf_exempt +from drf_spectacular.utils import extend_schema +from http import HTTPStatus +import json +from kernelCI_app.helpers.hardwareDetails import ( + assign_default_record_values, + decide_if_is_build_in_filter, + decide_if_is_full_record_filtered_out, + decide_if_is_test_in_filter, + generate_build_summary_typed, + generate_test_summary_typed, + generate_tree_status_summary_dict, + get_build, + get_filter_options, + get_hardware_details_data, + get_hardware_trees_data, + get_trees_with_selected_commit, + get_validated_current_tree, + handle_build_summary, + handle_test_summary, + handle_tree_status_summary, + set_trees_status_summary, + format_issue_summary_for_response, + unstable_parse_post_body, +) +from kernelCI_app.typeModels.commonDetails import ( + TestArchSummaryItem, +) +from kernelCI_app.typeModels.hardwareDetails import ( + HardwareDetailsSummaryResponse, + HardwareSummary, + PossibleTestType, + Tree, +) +from kernelCI_app.utils import is_boot +from pydantic import ValidationError +from rest_framework.response import Response +from rest_framework.views import APIView +from typing import Dict, List, Set + + +# disable django csrf protection https://docs.djangoproject.com/en/5.0/ref/csrf/ +# that protection is recommended for ‘unsafe’ methods (POST, PUT, and DELETE) +# but we are using POST here just to follow the convention to use the request body +# also the csrf protection require the usage of cookies which is not currently +# supported in this project +@method_decorator(csrf_exempt, name="dispatch") +class HardwareDetailsSummary(APIView): + required_params_get = ["origin"] + + def __init__(self): + self.origin: str = None + self.start_datetime: datetime = None + self.end_datetime: datetime = None + self.selected_commits: Dict[str, str] = None + + self.processed_builds = set() + self.processed_tests = set() + + self.processed_compatibles: Set[str] = set() + + self.processed_issues = { + "build": { + "issues": {}, + "failedWithUnknownIssues": 0, + }, + "boot": { + "issues": {}, + "failedWithUnknownIssues": 0, + }, + "test": { + "issues": {}, + "failedWithUnknownIssues": 0, + }, + } + + self.processed_architectures: Dict[str, Dict[str, TestArchSummaryItem]] = { + "build": {}, + "boot": {}, + "test": {}, + } + + self.builds_summary = generate_build_summary_typed() + self.boots_summary = generate_test_summary_typed() + self.tests_summary = generate_test_summary_typed() + + self.tree_status_summary = defaultdict(generate_tree_status_summary_dict) + self.compatibles: List[str] = [] + + def _process_test(self, record: Dict) -> None: + is_record_boot = is_boot(record["path"]) + test_type_key: PossibleTestType = "boot" if is_record_boot else "test" + task = self.boots_summary if is_record_boot else self.tests_summary + + should_process_test = decide_if_is_test_in_filter( + instance=self, + test_type=test_type_key, + record=record, + processed_tests=self.processed_tests, + ) + + self.processed_tests.add(record["id"]) + if should_process_test: + handle_test_summary( + record=record, + task=task, + processed_issues=self.processed_issues.get(test_type_key), + processed_archs=self.processed_architectures[test_type_key], + ) + + def _process_build(self, record: Dict, tree_index: int) -> None: + build = get_build(record, tree_index) + build_id = record["build_id"] + + should_process_build = decide_if_is_build_in_filter( + instance=self, + build=build, + processed_builds=self.processed_builds, + incident_test_id=record["incidents__test_id"], + ) + + self.processed_builds.add(build_id) + if should_process_build: + handle_build_summary( + record=record, + builds_summary=self.builds_summary, + processed_issues=self.processed_issues["build"], + tree_index=tree_index, + ) + + def _sanitize_records( + self, records, trees: List[Tree], is_all_selected: bool + ) -> None: + for record in records: + current_tree = get_validated_current_tree( + record=record, selected_trees=trees + ) + if current_tree is None: + continue + + assign_default_record_values(record) + + if record["environment_compatible"] is not None: + self.processed_compatibles.update(record["environment_compatible"]) + + tree_index = current_tree.index + + handle_tree_status_summary( + record=record, + tree_status_summary=self.tree_status_summary, + tree_index=tree_index, + processed_builds=self.processed_builds, + ) + + is_record_filtered_out = decide_if_is_full_record_filtered_out( + instance=self, + record=record, + current_tree=current_tree, + is_all_selected=is_all_selected, + ) + if is_record_filtered_out: + self.processed_builds.add(record["build_id"]) + continue + + self._process_test(record=record) + + self._process_build(record, tree_index) + + def _format_processing_for_response(self) -> None: + self.compatibles = list(self.processed_compatibles) + + self.boots_summary.architectures = list( + self.processed_architectures["boot"].values() + ) + self.tests_summary.architectures = list( + self.processed_architectures["test"].values() + ) + + format_issue_summary_for_response( + builds_summary=self.builds_summary, + boots_summary=self.boots_summary, + tests_summary=self.tests_summary, + processed_issues=self.processed_issues, + ) + + # Using post to receive a body request + @extend_schema( + responses=HardwareDetailsSummaryResponse, + ) + def post(self, request, hardware_id) -> Response: + try: + unstable_parse_post_body(instance=self, request=request) + except ValidationError as e: + return Response(data={"error": e.json()}, status=HTTPStatus.BAD_REQUEST) + except json.JSONDecodeError: + return Response( + data={ + "error": "Invalid body, request body must be a valid json string" + }, + status=HTTPStatus.BAD_REQUEST, + ) + except (ValueError, TypeError): + return Response( + data={ + "error": "startTimeStamp and endTimeStamp must be a Unix Timestamp" + }, + status=HTTPStatus.BAD_REQUEST, + ) + + trees = get_hardware_trees_data( + hardware_id=hardware_id, + origin=self.origin, + selected_commits=self.selected_commits, + start_datetime=self.start_datetime, + end_datetime=self.end_datetime, + ) + + trees_with_selected_commits = get_trees_with_selected_commit( + trees=trees, selected_commits=self.selected_commits + ) + + records = get_hardware_details_data( + hardware_id=hardware_id, + origin=self.origin, + trees_with_selected_commits=trees_with_selected_commits, + start_datetime=self.start_datetime, + end_datetime=self.end_datetime, + ) + + if len(records) == 0: + return Response( + data={"error": "Hardware not found"}, status=HTTPStatus.NOT_FOUND + ) + + is_all_selected = len(self.selected_commits) == 0 + + self._sanitize_records(records, trees_with_selected_commits, is_all_selected) + + self._format_processing_for_response() + + configs, archs, compilers = get_filter_options( + records=records, + selected_trees=trees_with_selected_commits, + is_all_selected=is_all_selected, + ) + + set_trees_status_summary( + trees=trees, tree_status_summary=self.tree_status_summary + ) + + try: + valid_response = HardwareDetailsSummaryResponse( + summary=HardwareSummary( + builds=self.builds_summary, + boots=self.boots_summary, + tests=self.tests_summary, + trees=trees, + configs=configs, + architectures=archs, + compilers=compilers, + compatibles=self.compatibles, + ), + ) + except ValidationError as e: + return Response(data=e.errors(), status=HTTPStatus.INTERNAL_SERVER_ERROR) + + return Response(data=valid_response.model_dump(), status=HTTPStatus.OK) diff --git a/backend/kernelCI_app/views/hardwareDetailsView.py b/backend/kernelCI_app/views/hardwareDetailsView.py index 586dc7c9..6da33891 100644 --- a/backend/kernelCI_app/views/hardwareDetailsView.py +++ b/backend/kernelCI_app/views/hardwareDetailsView.py @@ -59,6 +59,8 @@ def __init__(self): self.processed_builds = set() self.processed_tests = set() + self.processed_compatibles: Set[str] = set() + self.builds = { "items": [], "issues": {}, @@ -68,19 +70,18 @@ def __init__(self): self.tests = generate_test_dict() self.tree_status_summary = defaultdict(generate_tree_status_summary_dict) - self.compatibles: List[str] = [] def _process_test(self, record: Dict) -> None: is_record_boot = is_boot(record["path"]) test_filter_key: PossibleTestType = "boot" if is_record_boot else "test" - is_record_processed = record["id"] in self.processed_tests - is_test_in_filter = decide_if_is_test_in_filter( - instance=self, test_type=test_filter_key, record=record + should_process_test = decide_if_is_test_in_filter( + instance=self, + test_type=test_filter_key, + record=record, + processed_tests=self.processed_tests, ) - should_process_test = not is_record_processed and is_test_in_filter - if should_process_test: self.processed_tests.add(record["id"]) handle_test_or_boot(record, self.boots if is_record_boot else self.tests) @@ -103,8 +104,6 @@ def _process_build(self, record: Dict, tree_index: int) -> None: def _sanitize_records( self, records, trees: List[Tree], is_all_selected: bool ) -> None: - compatibles: Set[str] = set() - for record in records: current_tree = get_validated_current_tree( record=record, selected_trees=trees @@ -115,7 +114,7 @@ def _sanitize_records( assign_default_record_values(record) if record["environment_compatible"] is not None: - compatibles.update(record["environment_compatible"]) + self.processed_compatibles.update(record["environment_compatible"]) tree_index = current_tree.index @@ -140,8 +139,7 @@ def _sanitize_records( self._process_build(record, tree_index) - self.builds["summary"] = create_details_build_summary(self.builds["items"]) - self.compatibles = list(compatibles) + def _format_processing_for_response(self): mutate_properties_to_list(self.builds, ["issues"]) mutate_properties_to_list( self.tests, ["issues", "platformsFailing", "archSummary"] @@ -203,6 +201,9 @@ def post(self, request, hardware_id): self._sanitize_records(records, trees_with_selected_commits, is_all_selected) + self.builds["summary"] = create_details_build_summary(self.builds["items"]) + self._format_processing_for_response() + configs, archs, compilers = get_filter_options( records=records, selected_trees=trees_with_selected_commits, @@ -249,7 +250,7 @@ def post(self, request, hardware_id): "configs": configs, "architectures": archs, "compilers": compilers, - "compatibles": self.compatibles, + "compatibles": list(self.processed_compatibles), }, } try: diff --git a/backend/requests/hardware-details-summary-post.sh b/backend/requests/hardware-details-summary-post.sh new file mode 100644 index 00000000..4238135f --- /dev/null +++ b/backend/requests/hardware-details-summary-post.sh @@ -0,0 +1,574 @@ +http POST http://localhost:8000/api/hardware/google,juniper/summary \ +Content-Type:application/json \ +<<< '{ + "origin":"maestro", + "startTimestampInSeconds":1737046800, + "endTimestampInSeconds":1737478800, + "selectedCommits":{}, + "filter":{} +}' + +# HTTP/1.1 200 OK +# Allow: POST, OPTIONS +# Content-Length: 9063 +# Content-Type: application/json +# Cross-Origin-Opener-Policy: same-origin +# Date: Tue, 21 Jan 2025 16:48:47 GMT +# Referrer-Policy: same-origin +# Server: WSGIServer/0.2 CPython/3.12.7 +# Vary: Accept, Cookie, origin +# X-Content-Type-Options: nosniff +# X-Frame-Options: DENY + +# { +# "summary": { +# "architectures": [ +# "arm64" +# ], +# "boots": { +# "architectures": [ +# { +# "arch": "arm64", +# "compiler": "gcc-12", +# "status": { +# "ERROR": 0, +# "FAIL": 1, +# "MISS": 0, +# "NULL": 6, +# "PASS": 202, +# "SKIP": 0 +# } +# } +# ], +# "configs": { +# "cros://chromeos-6.6/arm64/chromiumos-mediatek.flavour.config": { +# "ERROR": 0, +# "FAIL": 0, +# "MISS": 0, +# "NULL": 0, +# "PASS": 75, +# "SKIP": 0 +# }, +# "defconfig": { +# "ERROR": 0, +# "FAIL": 1, +# "MISS": 0, +# "NULL": 6, +# "PASS": 127, +# "SKIP": 0 +# } +# }, +# "environment_compatible": null, +# "environment_misc": null, +# "fail_reasons": {}, +# "failed_platforms": [], +# "issues": [ +# { +# "comment": "[logspec:generic_linux_boot] linux.kernel.ubsan shift-out-of-bounds: shift exponent -1 is negative", +# "id": "maestro:b91eba41d9d0281c086ee574a82bdee035760751", +# "incidents_info": { +# "incidentsCount": 1 +# }, +# "report_url": null, +# "version": "0" +# } +# ], +# "platforms": { +# "mt8183-kukui-jacuzzi-juniper-sku16": { +# "ERROR": 0, +# "FAIL": 1, +# "MISS": 0, +# "NULL": 6, +# "PASS": 202, +# "SKIP": 0 +# } +# }, +# "status": { +# "ERROR": 0, +# "FAIL": 1, +# "MISS": 0, +# "NULL": 6, +# "PASS": 202, +# "SKIP": 0 +# }, +# "unknown_issues": 0 +# }, +# "builds": { +# "architectures": { +# "arm64": { +# "compilers": [ +# "gcc-12" +# ], +# "invalid": 0, +# "null": 0, +# "valid": 39 +# } +# }, +# "configs": { +# "cros://chromeos-6.6/arm64/chromiumos-mediatek.flavour.config": { +# "invalid": 0, +# "null": 0, +# "valid": 12 +# }, +# "defconfig": { +# "invalid": 0, +# "null": 0, +# "valid": 27 +# } +# }, +# "issues": [], +# "status": { +# "invalid": 0, +# "null": 0, +# "valid": 39 +# }, +# "unknown_issues": 0 +# }, +# "compatibles": [ +# "google,juniper", +# "google,juniper-sku16", +# "mediatek,mt8183" +# ], +# "compilers": [ +# "gcc-12" +# ], +# "configs": [ +# "cros://chromeos-6.6/arm64/chromiumos-mediatek.flavour.config", +# "defconfig" +# ], +# "tests": { +# "architectures": [ +# { +# "arch": "arm64", +# "compiler": "gcc-12", +# "status": { +# "ERROR": 4, +# "FAIL": 2169, +# "MISS": 9, +# "NULL": 8, +# "PASS": 2789, +# "SKIP": 352 +# } +# } +# ], +# "configs": { +# "cros://chromeos-6.6/arm64/chromiumos-mediatek.flavour.config": { +# "ERROR": 2, +# "FAIL": 188, +# "MISS": 5, +# "NULL": 2, +# "PASS": 239, +# "SKIP": 0 +# }, +# "defconfig": { +# "ERROR": 2, +# "FAIL": 1981, +# "MISS": 4, +# "NULL": 6, +# "PASS": 2550, +# "SKIP": 352 +# } +# }, +# "environment_compatible": null, +# "environment_misc": null, +# "fail_reasons": {}, +# "failed_platforms": [], +# "issues": [], +# "platforms": { +# "mt8183-kukui-jacuzzi-juniper-sku16": { +# "ERROR": 4, +# "FAIL": 2169, +# "MISS": 9, +# "NULL": 8, +# "PASS": 2789, +# "SKIP": 352 +# } +# }, +# "status": { +# "ERROR": 4, +# "FAIL": 2169, +# "MISS": 9, +# "NULL": 8, +# "PASS": 2789, +# "SKIP": 352 +# }, +# "unknown_issues": 2169 +# }, +# "trees": [ +# { +# "git_repository_branch": "for-kernelci", +# "git_repository_url": "https://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux.git", +# "head_git_commit_hash": "1950a0af2d55e0ecbcc574927bad495bfaddcec0", +# "head_git_commit_name": "v6.13-rc7-71-g1950a0af2d55e", +# "head_git_commit_tag": [], +# "index": "0", +# "is_selected": null, +# "selected_commit_status": { +# "boots": { +# "NULL": 1, +# "PASS": 19 +# }, +# "builds": { +# "valid": 3 +# }, +# "tests": { +# "FAIL": 227, +# "MISS": 2, +# "PASS": 260, +# "SKIP": 44 +# } +# }, +# "tree_name": "arm64" +# }, +# { +# "git_repository_branch": "for-next", +# "git_repository_url": "https://git.kernel.org/pub/scm/linux/kernel/git/broonie/regmap.git", +# "head_git_commit_hash": "78798d8875315a374ac7f0e076094a9198a5edda", +# "head_git_commit_name": "v6.13-rc7-15-g78798d887531", +# "head_git_commit_tag": [], +# "index": "1", +# "is_selected": null, +# "selected_commit_status": { +# "boots": { +# "PASS": 20 +# }, +# "builds": { +# "valid": 3 +# }, +# "tests": { +# "FAIL": 8, +# "MISS": 1, +# "PASS": 266, +# "SKIP": 36 +# } +# }, +# "tree_name": "broonie-regmap" +# }, +# { +# "git_repository_branch": "for-next", +# "git_repository_url": "https://git.kernel.org/pub/scm/linux/kernel/git/broonie/regulator.git", +# "head_git_commit_hash": "5754b3acbe28cd93056e000ca239123f600a3b4d", +# "head_git_commit_name": "v6.13-rc7-16-g5754b3acbe28", +# "head_git_commit_tag": [], +# "index": "2", +# "is_selected": null, +# "selected_commit_status": { +# "boots": { +# "FAIL": 2, +# "PASS": 17 +# }, +# "builds": { +# "valid": 3 +# }, +# "tests": { +# "FAIL": 227, +# "PASS": 267, +# "SKIP": 44 +# } +# }, +# "tree_name": "broonie-regulator" +# }, +# { +# "git_repository_branch": "for-next", +# "git_repository_url": "https://git.kernel.org/pub/scm/linux/kernel/git/broonie/sound.git", +# "head_git_commit_hash": "2367adc931673c5b671724ae86b6fe819aee9d8f", +# "head_git_commit_name": "asoc-v6.14-21-g2367adc93167", +# "head_git_commit_tag": [], +# "index": "3", +# "is_selected": null, +# "selected_commit_status": { +# "boots": { +# "PASS": 22 +# }, +# "builds": { +# "valid": 3 +# }, +# "tests": { +# "ERROR": 1, +# "FAIL": 220, +# "PASS": 147, +# "SKIP": 9 +# } +# }, +# "tree_name": "broonie-sound" +# }, +# { +# "git_repository_branch": "for-next", +# "git_repository_url": "https://git.kernel.org/pub/scm/linux/kernel/git/broonie/spi.git", +# "head_git_commit_hash": "ff9e24437b18fa5a543c895f5b3d5108c67278d0", +# "head_git_commit_name": "v6.13-rc7-60-gff9e24437b18f", +# "head_git_commit_tag": [], +# "index": "4", +# "is_selected": null, +# "selected_commit_status": { +# "boots": { +# "PASS": 4 +# }, +# "builds": { +# "valid": 1 +# }, +# "tests": {} +# }, +# "tree_name": "broonie-spi" +# }, +# { +# "git_repository_branch": "clk-next", +# "git_repository_url": "https://git.kernel.org/pub/scm/linux/kernel/git/clk/linux.git", +# "head_git_commit_hash": "47b32b50ee28505eebd91eff40126483a15e5f4e", +# "head_git_commit_name": "qcom-clk-for-6.14-147-g47b32b50ee28", +# "head_git_commit_tag": [], +# "index": "5", +# "is_selected": null, +# "selected_commit_status": { +# "boots": { +# "NULL": 1, +# "PASS": 20 +# }, +# "builds": { +# "valid": 3 +# }, +# "tests": { +# "FAIL": 222, +# "NULL": 2, +# "PASS": 273, +# "SKIP": 44 +# } +# }, +# "tree_name": "clk" +# }, +# { +# "git_repository_branch": "master", +# "git_repository_url": "https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git", +# "head_git_commit_hash": "95ec54a420b8f445e04a7ca0ea8deb72c51fe1d3", +# "head_git_commit_name": "v6.13-918-g95ec54a420b8", +# "head_git_commit_tag": [], +# "index": "6", +# "is_selected": null, +# "selected_commit_status": { +# "boots": { +# "PASS": 20 +# }, +# "builds": { +# "valid": 3 +# }, +# "tests": { +# "ERROR": 2, +# "FAIL": 387, +# "MISS": 3, +# "NULL": 2, +# "PASS": 518, +# "SKIP": 44 +# } +# }, +# "tree_name": "mainline" +# }, +# { +# "git_repository_branch": "main", +# "git_repository_url": "https://git.kernel.org/pub/scm/linux/kernel/git/netdev/net-next.git", +# "head_git_commit_hash": "cf33d96f50903214226b379b3f10d1f262dae018", +# "head_git_commit_name": "v6.13-rc7-1620-gcf33d96f50903", +# "head_git_commit_tag": [], +# "index": "7", +# "is_selected": null, +# "selected_commit_status": { +# "boots": { +# "NULL": 1, +# "PASS": 3 +# }, +# "builds": { +# "valid": 1 +# }, +# "tests": {} +# }, +# "tree_name": "net-next" +# }, +# { +# "git_repository_branch": "master", +# "git_repository_url": "https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git", +# "head_git_commit_hash": "f066b5a6c7a06adfb666b7652cc99b4ff264f4ed", +# "head_git_commit_name": "next-20250121", +# "head_git_commit_tag": [ +# "next-20250121" +# ], +# "index": "8", +# "is_selected": null, +# "selected_commit_status": { +# "boots": { +# "PASS": 4 +# }, +# "builds": { +# "valid": 3 +# }, +# "tests": { +# "FAIL": 4, +# "MISS": 2, +# "PASS": 1 +# } +# }, +# "tree_name": "next" +# }, +# { +# "git_repository_branch": "pending-fixes", +# "git_repository_url": "https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git", +# "head_git_commit_hash": "9e5c7d574d79f927cb3353c4437eaf532e904115", +# "head_git_commit_name": "v6.13-939-g9e5c7d574d79", +# "head_git_commit_tag": [], +# "index": "9", +# "is_selected": null, +# "selected_commit_status": { +# "boots": { +# "PASS": 4 +# }, +# "builds": { +# "valid": 3 +# }, +# "tests": { +# "FAIL": 166, +# "NULL": 1, +# "PASS": 255 +# } +# }, +# "tree_name": "next" +# }, +# { +# "git_repository_branch": "testing", +# "git_repository_url": "https://git.kernel.org/pub/scm/linux/kernel/git/rafael/linux-pm.git", +# "head_git_commit_hash": "69bbfd3a203eff79eacc8a02b3660deaa55624bc", +# "head_git_commit_name": "acpi-6.13-rc8-131-g69bbfd3a203e", +# "head_git_commit_tag": [], +# "index": "10", +# "is_selected": null, +# "selected_commit_status": { +# "boots": { +# "NULL": 1, +# "PASS": 19 +# }, +# "builds": { +# "valid": 3 +# }, +# "tests": { +# "FAIL": 227, +# "PASS": 267, +# "SKIP": 44 +# } +# }, +# "tree_name": "pm" +# }, +# { +# "git_repository_branch": "master", +# "git_repository_url": "https://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-devel.git", +# "head_git_commit_hash": "d65183bfb94f5627b12a23700e03808b46ca9981", +# "head_git_commit_name": "renesas-devel-2025-01-21-v6.13", +# "head_git_commit_tag": [ +# "renesas-devel-2025-01-21-v6.13" +# ], +# "index": "11", +# "is_selected": null, +# "selected_commit_status": { +# "boots": { +# "PASS": 4 +# }, +# "builds": { +# "valid": 1 +# }, +# "tests": {} +# }, +# "tree_name": "renesas" +# }, +# { +# "git_repository_branch": "for-next", +# "git_repository_url": "https://git.kernel.org/pub/scm/linux/kernel/git/soc/soc.git", +# "head_git_commit_hash": "fcf4173cc889f5a246d51a3c425f11d32ce615fc", +# "head_git_commit_name": "v6.13-rc7-832-gfcf4173cc889", +# "head_git_commit_tag": [], +# "index": "12", +# "is_selected": null, +# "selected_commit_status": { +# "boots": { +# "PASS": 20 +# }, +# "builds": { +# "valid": 3 +# }, +# "tests": { +# "FAIL": 227, +# "MISS": 1, +# "PASS": 265, +# "SKIP": 43 +# } +# }, +# "tree_name": "soc" +# }, +# { +# "git_repository_branch": "linux-6.6.y", +# "git_repository_url": "https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git", +# "head_git_commit_hash": "3b4299ff7a25480d96c5e9a84b879e5193447d28", +# "head_git_commit_name": "v6.6.73", +# "head_git_commit_tag": [ +# "v6.6.73" +# ], +# "index": "13", +# "is_selected": null, +# "selected_commit_status": { +# "boots": { +# "NULL": 1, +# "PASS": 3 +# }, +# "builds": { +# "valid": 2 +# }, +# "tests": { +# "ERROR": 1, +# "FAIL": 30, +# "PASS": 2 +# } +# }, +# "tree_name": "stable" +# }, +# { +# "git_repository_branch": "master", +# "git_repository_url": "https://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git", +# "head_git_commit_hash": "e6609f8bea4a5d16e6b1648d55e0c5a24cffbe96", +# "head_git_commit_name": "v6.13-1225-ge6609f8bea4a5", +# "head_git_commit_tag": [], +# "index": "14", +# "is_selected": null, +# "selected_commit_status": { +# "boots": { +# "PASS": 4 +# }, +# "builds": { +# "valid": 1 +# }, +# "tests": {} +# }, +# "tree_name": "tip" +# }, +# { +# "git_repository_branch": "next", +# "git_repository_url": "https://git.kernel.org/pub/scm/linux/kernel/git/ulfh/mmc.git", +# "head_git_commit_hash": "20a0c37e44063997391430c4ae09973e9cbc3911", +# "head_git_commit_name": "mmc-v6.13-rc2-2-27-g20a0c37e44063", +# "head_git_commit_tag": [], +# "index": "15", +# "is_selected": null, +# "selected_commit_status": { +# "boots": { +# "NULL": 1, +# "PASS": 19 +# }, +# "builds": { +# "valid": 3 +# }, +# "tests": { +# "FAIL": 224, +# "NULL": 3, +# "PASS": 268, +# "SKIP": 44 +# } +# }, +# "tree_name": "ulfh" +# } +# ] +# } +# } diff --git a/backend/schema.yml b/backend/schema.yml index cfb3ae32..bd0271cd 100644 --- a/backend/schema.yml +++ b/backend/schema.yml @@ -26,6 +26,28 @@ paths: schema: $ref: '#/components/schemas/HardwareDetailsFullResponse' description: '' + /api/hardware/{hardware_id}/summary: + post: + operationId: hardware_summary_create + parameters: + - in: path + name: hardware_id + schema: + type: string + required: true + tags: + - hardware + security: + - cookieAuth: [] + - basicAuth: [] + - {} + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/HardwareDetailsSummaryResponse' + description: '' /api/log-downloader/: get: operationId: log_downloader_retrieve @@ -378,43 +400,33 @@ components: BuildArchitectures: properties: valid: + anyOf: + - type: integer + - type: 'null' + default: 0 title: Valid - type: integer invalid: + anyOf: + - type: integer + - type: 'null' + default: 0 title: Invalid - type: integer 'null': + anyOf: + - type: integer + - type: 'null' + default: 0 title: 'Null' - type: integer compilers: - items: - type: string + anyOf: + - items: + type: string + type: array + - type: 'null' + default: [] title: Compilers - type: array - required: - - valid - - invalid - - 'null' - - compilers title: BuildArchitectures type: object - BuildConfigs: - properties: - valid: - title: Valid - type: integer - invalid: - title: Invalid - type: integer - 'null': - title: 'Null' - type: integer - required: - - valid - - invalid - - 'null' - title: BuildConfigs - type: object BuildHistoryItem: properties: id: @@ -496,18 +508,23 @@ components: BuildStatusCount: properties: valid: + anyOf: + - type: integer + - type: 'null' + default: 0 title: Valid - type: integer invalid: + anyOf: + - type: integer + - type: 'null' + default: 0 title: Invalid - type: integer 'null': + anyOf: + - type: integer + - type: 'null' + default: 0 title: 'Null' - type: integer - required: - - valid - - invalid - - 'null' title: BuildStatusCount type: object BuildSummary: @@ -521,12 +538,12 @@ components: type: object configs: additionalProperties: - $ref: '#/components/schemas/BuildConfigs' + $ref: '#/components/schemas/BuildStatusCount' title: Configs type: object issues: items: - $ref: '#/components/schemas/BuildsIssuesItem' + $ref: '#/components/schemas/Issue' title: Issues type: array unknown_issues: @@ -540,30 +557,6 @@ components: - unknown_issues title: BuildSummary type: object - BuildsIssuesItem: - properties: - id: - title: Id - type: string - comment: - anyOf: - - type: string - - type: 'null' - title: Comment - report_url: - anyOf: - - type: string - - type: 'null' - title: Report Url - incidents_info: - $ref: '#/components/schemas/IncidentsInfo' - required: - - id - - comment - - report_url - - incidents_info - title: BuildsIssuesItem - type: object BuildsResponse: properties: builds: @@ -601,6 +594,14 @@ components: - summary title: HardwareDetailsFullResponse type: object + HardwareDetailsSummaryResponse: + properties: + summary: + $ref: '#/components/schemas/HardwareSummary' + required: + - summary + title: HardwareDetailsSummaryResponse + type: object HardwareSummary: properties: builds: @@ -645,14 +646,42 @@ components: - compatibles title: HardwareSummary type: object - IncidentsInfo: + IncidentInfo: properties: incidentsCount: title: Incidentscount type: integer required: - incidentsCount - title: IncidentsInfo + title: IncidentInfo + type: object + Issue: + properties: + id: + title: Id + type: string + version: + title: Version + type: string + comment: + anyOf: + - type: string + - type: 'null' + title: Comment + report_url: + anyOf: + - type: string + - type: 'null' + title: Report Url + incidents_info: + $ref: '#/components/schemas/IncidentInfo' + required: + - id + - version + - comment + - report_url + - incidents_info + title: Issue type: object Misc: properties: @@ -781,30 +810,6 @@ components: - misc title: TestHistoryItem type: object - TestIssuesItem: - properties: - id: - title: Id - type: string - comment: - anyOf: - - type: string - - type: 'null' - title: Comment - report_url: - anyOf: - - type: string - - type: 'null' - title: Report Url - incidents_info: - $ref: '#/components/schemas/IncidentsInfo' - required: - - id - - comment - - report_url - - incidents_info - title: TestIssuesItem - type: object TestResponse: properties: testHistory: @@ -822,31 +827,37 @@ components: anyOf: - type: integer - type: 'null' - default: null + default: 0 title: Pass ERROR: anyOf: - type: integer - type: 'null' - default: null + default: 0 title: Error FAIL: anyOf: - type: integer - type: 'null' - default: null + default: 0 title: Fail SKIP: anyOf: - type: integer - type: 'null' - default: null + default: 0 title: Skip + MISS: + anyOf: + - type: integer + - type: 'null' + default: 0 + title: Miss 'NULL': anyOf: - type: integer - type: 'null' - default: null + default: 0 title: 'Null' title: TestStatusCount type: object @@ -866,7 +877,7 @@ components: type: object issues: items: - $ref: '#/components/schemas/TestIssuesItem' + $ref: '#/components/schemas/Issue' title: Issues type: array unknown_issues: @@ -954,6 +965,11 @@ components: - type: object - type: 'null' title: Selected Commit Status + is_selected: + anyOf: + - type: boolean + - type: 'null' + title: Is Selected required: - index - tree_name @@ -963,6 +979,7 @@ components: - head_git_commit_hash - head_git_commit_tag - selected_commit_status + - is_selected title: Tree type: object TreeCommon: