feat: add hardwareDetailsSummary endpoint

Adds the new endpoint to return only the summary of a hardware, without history items

Adds refactors and type validation for hardware helper functions for future reuse

Part of #751

Closes #759

backend/kernelCI_app/helpers/hardwareDetails.py

@@ -20,9 +20,29 @@
 from kernelCI_app.helpers.misc import env_misc_value_or_default, handle_environment_misc
 from kernelCI_app.helpers.trees import get_tree_heads
 from kernelCI_app.models import Tests
-from kernelCI_app.typeModels.databases import FAIL_STATUS
-from kernelCI_app.typeModels.hardwareDetails import DefaultRecordValues, PostBody, Tree
-from kernelCI_app.utils import create_issue, extract_error_message, is_boot
+from kernelCI_app.typeModels.databases import FAIL_STATUS, NULL_STATUS
+from kernelCI_app.typeModels.commonDetails import (
+    BuildArchitectures,
+    BuildStatusCount,
+    BuildSummary,
+    Misc,
+    TestArchSummaryItem,
+    TestHistoryItem,
+    TestStatusCount,
+    TestSummary,
+)
+from kernelCI_app.typeModels.hardwareDetails import (
+    DefaultRecordValues,
+    HardwareBuildHistoryItem,
+    PostBody,
+    Tree,
+)
+from kernelCI_app.utils import (
+    convert_issues_dict_to_list_typed,
+    create_issue,
+    extract_error_message,
+    is_boot,
+)
 from pydantic import ValidationError
 from django.db.models import Subquery
 from kernelCI_app.typeModels.hardwareDetails import (
@@ -209,7 +229,7 @@ def get_hardware_details_data(
 
 
 def query_records(
-    *, hardware_id, origin, trees: List[Tree], start_date=int, end_date=int
+    *, hardware_id: str, origin: str, trees: List[Tree], start_date=int, end_date=int
 ):
     commit_hashes = [tree.head_git_commit_hash for tree in trees]
 
@@ -257,6 +277,7 @@ def query_records(
     )
 
 
+# deprecated, use get_arch_summary_typed
 def get_arch_summary(record: Dict) -> Dict:
     return {
         "arch": record["build__architecture"],
@@ -265,6 +286,15 @@ def get_arch_summary(record: Dict) -> Dict:
     }
 
 
+def get_arch_summary_typed(record: Dict) -> TestArchSummaryItem:
+    return TestArchSummaryItem(
+        arch=record["build__architecture"],
+        compiler=record["build__compiler"],
+        status=TestStatusCount(),
+    )
+
+
+# deprecated, use get_build_typed
 def get_build(record: Dict, tree_idx: int) -> Dict:
     return {
         "id": record["build_id"],
@@ -286,6 +316,27 @@ def get_build(record: Dict, tree_idx: int) -> Dict:
     }
 
 
+def get_build_typed(record: Dict, tree_idx: int) -> HardwareBuildHistoryItem:
+    return HardwareBuildHistoryItem(
+        id=record["build_id"],
+        architecture=record["build__architecture"],
+        config_name=record["build__config_name"],
+        misc=record["build__misc"],
+        config_url=record["build__config_url"],
+        compiler=record["build__compiler"],
+        valid=record["build__valid"],
+        duration=record["build__duration"],
+        log_url=record["build__log_url"],
+        start_time=record["build__start_time"],
+        git_repository_url=record["build__checkout__git_repository_url"],
+        git_repository_branch=record["build__checkout__git_repository_branch"],
+        tree_index=tree_idx,
+        tree_name=record["build__checkout__tree_name"],
+        issue_id=record["build__incidents__issue__id"],
+        issue_version=record["build__incidents__issue__version"],
+    )
+
+
 def get_tree_key(record: Dict) -> str:
     return (
         record["build__checkout__tree_name"]
@@ -294,16 +345,6 @@ def get_tree_key(record: Dict) -> str:
     )
 
 
-def get_tree(record: Dict) -> Dict[str, str]:
-    return {
-        "tree_name": record["build__checkout__tree_name"],
-        "git_repository_branch": record["build__checkout__git_repository_branch"],
-        "git_repository_url": record["build__checkout__git_repository_url"],
-        "git_commit_name": record["build__checkout__git_commit_name"],
-        "git_commit_hash": record["build__checkout__git_commit_hash"],
-    }
-
-
 def get_history(record: Dict):
     return {
         "id": record["id"],
@@ -374,7 +415,29 @@ def generate_test_dict() -> Dict[str, any]:
     }
 
 
-def generate_tree_status_summary_dict() -> Dict[str]:
+def generate_build_summary_typed() -> BuildSummary:
+    return BuildSummary(
+        status=BuildStatusCount(),
+        architectures={},
+        configs={},
+        issues=[],
+        unknown_issues=0,
+    )
+
+
+def generate_test_summary_typed() -> TestSummary:
+    return TestSummary(
+        status=TestStatusCount(),
+        architectures=[],
+        configs={},
+        issues=[],
+        unknown_issues=0,
+        fail_reasons={},
+        failed_platforms=[],
+    )
+
+
+def generate_tree_status_summary_dict() -> Dict[str, defaultdict[int]]:
     return {
         "builds": defaultdict(int),
         "boots": defaultdict(int),
@@ -399,14 +462,20 @@ def handle_tree_status_summary(
         tree_status_summary[tree_index]["builds"][build_status] += 1
 
 
-def handle_test_or_boot(record: Dict, task: Dict) -> None:
-    status = record["status"]
-
+def create_record_test_platform(*, record: Dict) -> str:
     environment_misc = handle_environment_misc(record["environment_misc"])
     test_platform = env_misc_value_or_default(environment_misc).get("platform")
-
     record["test_platform"] = test_platform
 
+    return test_platform
+
+
+# deprecated, use handle_test_history and handle_test_summary separately instead, with typing
+def handle_test_or_boot(record: Dict, task: Dict) -> None:
+    status = record["status"]
+
+    test_platform = create_record_test_platform(record=record)
+
     task["history"].append(get_history(record))
     task["statusSummary"][status] += 1
     task["configs"][record["build__config_name"]][status] += 1
@@ -437,6 +506,125 @@ def handle_test_or_boot(record: Dict, task: Dict) -> None:
     )
 
 
+def handle_test_history(
+    *,
+    record: Dict,
+    task: List[TestHistoryItem],
+) -> None:
+    create_record_test_platform(record=record)
+
+    test_history_item = TestHistoryItem(
+        id=record["id"],
+        status=record["status"],
+        duration=record["duration"],
+        path=record["path"],
+        start_time=record["start_time"],
+        environment_compatible=record["environment_compatible"],
+        config=record["build__config_name"],
+        log_url=record["log_url"],
+        architecture=record["build__architecture"],
+        compiler=record["build__compiler"],
+        misc=Misc(platform=record["test_platform"]),
+    )
+
+    task.append(test_history_item)
+
+
+def handle_test_summary(
+    *,
+    record: Dict,
+    task: TestSummary,
+    processed_issues: Dict,
+    processed_archs: Dict[str, TestArchSummaryItem],
+) -> None:
+    status = record["status"]
+
+    if status is None:
+        status = NULL_STATUS
+
+    setattr(task.status, status, getattr(task.status, status) + 1)
+
+    config_name = record["build__config_name"]
+    if task.configs.get(config_name) is None:
+        task.configs[config_name] = TestStatusCount()
+    setattr(
+        task.configs[config_name],
+        status,
+        getattr(task.configs[config_name], status) + 1,
+    )
+
+    environment_misc = handle_environment_misc(record["environment_misc"])
+    test_platform = env_misc_value_or_default(environment_misc).get("platform")
+    if task.platforms is None:
+        task.platforms = {}
+    if task.platforms.get(test_platform) is None:
+        task.platforms[test_platform] = TestStatusCount()
+    setattr(
+        task.platforms[test_platform],
+        status,
+        getattr(task.platforms[test_platform], status) + 1,
+    )
+
+    arch_key = f'{record["build__architecture"]}{record["build__compiler"]}'
+    arch_summary = processed_archs.get(arch_key)
+    if not arch_summary:
+        arch_summary = get_arch_summary_typed(record)
+        processed_archs[arch_key] = arch_summary
+    setattr(arch_summary.status, status, getattr(arch_summary.status, status) + 1)
+
+    process_issue(record=record, task_issues_dict=processed_issues, issue_from="test")
+
+
+def handle_build_summary(
+    *,
+    record: Dict,
+    builds_summary: BuildSummary,
+    processed_issues: Dict,
+    tree_index: int,
+) -> None:
+    build: HardwareBuildHistoryItem = get_build_typed(record, tree_idx=tree_index)
+
+    # TODO: use build_status_map values or BuildStatusCount keys
+    status_key: Literal["valid", "invalid", "null"] = build_status_map.get(build.valid)
+    setattr(
+        builds_summary.status,
+        status_key,
+        getattr(builds_summary.status, status_key) + 1,
+    )
+
+    if config := build.config_name:
+        build_config_summary = builds_summary.configs.get(config)
+        if not build_config_summary:
+            build_config_summary = BuildStatusCount()
+            builds_summary.configs[config] = build_config_summary
+        setattr(
+            builds_summary.configs[config],
+            status_key,
+            getattr(builds_summary.configs[config], status_key) + 1,
+        )
+
+    if arch := build.architecture:
+        build_arch_summary = builds_summary.architectures.get(arch)
+        if not build_arch_summary:
+            build_arch_summary = BuildArchitectures()
+            builds_summary.architectures[arch] = build_arch_summary
+        setattr(
+            builds_summary.architectures[arch],
+            status_key,
+            getattr(builds_summary.architectures[arch], status_key) + 1,
+        )
+
+        compiler = build.compiler
+        if (
+            compiler is not None
+            and compiler not in builds_summary.architectures.get(arch).compilers
+        ):
+            builds_summary.architectures[arch].compilers.append(compiler)
+
+    process_issue(record=record, task_issues_dict=processed_issues, issue_from="build")
+
+
+# deprecated, use handle_build_history and handle_build_summary separately instead, with typing
 def handle_build(*, instance, record: Dict, build: Dict) -> None:
     instance.builds["items"].append(build)
     update_issues(
@@ -452,6 +640,27 @@ def handle_build(*, instance, record: Dict, build: Dict) -> None:
     )
 
 
+def process_issue(
+    *, record, task_issues_dict: Dict, issue_from: Literal["build", "test"]
+) -> None:
+    if issue_from == "build":
+        is_failed_task = record["build__valid"] is not True
+    else:
+        is_failed_task = record["status"] == FAIL_STATUS
+
+    update_issues(
+        issue_id=record["incidents__issue__id"],
+        issue_version=record["incidents__issue__version"],
+        incident_test_id=record["incidents__test_id"],
+        build_valid=record["build__valid"],
+        issue_comment=record["incidents__issue__comment"],
+        issue_report_url=record["incidents__issue__report_url"],
+        is_failed_task=is_failed_task,
+        issue_from=issue_from,
+        task=task_issues_dict,
+    )
+
+
 # TODO unify with treeDetails
 def update_issues(
     *,
@@ -533,8 +742,12 @@ def decide_if_is_build_in_filter(
 
 
 def decide_if_is_test_in_filter(
-    *, instance, test_type: PossibleTestType, record: Dict
+    *, instance, test_type: PossibleTestType, record: Dict, processed_tests: Set[str]
 ) -> bool:
+    is_test_processed = record["id"] in processed_tests
+    if is_test_processed:
+        return False
+
     test_filter_pass = True
 
     status = record["status"]
@@ -624,3 +837,24 @@ def assign_default_record_values(record: Dict) -> None:
         record["build__incidents__issue__id"] = UNKNOWN_STRING
     if record["incidents__issue__id"] is None and record["status"] == FAIL_STATUS:
         record["incidents__issue__id"] = UNKNOWN_STRING
+
+
+def format_issue_summary_for_response(
+    *,
+    builds_summary: BuildSummary,
+    boots_summary: TestSummary,
+    tests_summary: TestSummary,
+    processed_issues: Dict,
+) -> None:
+    builds_summary.issues = convert_issues_dict_to_list_typed(
+        issues_dict=processed_issues["build"]["issues"]
+    )
+    boots_summary.issues = convert_issues_dict_to_list_typed(
+        issues_dict=processed_issues["boot"]["issues"]
+    )
+    tests_summary.issues = convert_issues_dict_to_list_typed(
+        issues_dict=processed_issues["test"]["issues"]
+    )
+    builds_summary.unknown_issues = processed_issues["build"]["failedWithUnknownIssues"]
+    boots_summary.unknown_issues = processed_issues["boot"]["failedWithUnknownIssues"]
+    tests_summary.unknown_issues = processed_issues["test"]["failedWithUnknownIssues"]

backend/kernelCI_app/typeModels/databases.py

@@ -6,6 +6,7 @@
 ERROR_STATUS = "ERROR"
 MISS_STATUS = "MISS"
 PASS_STATUS = "PASS"
+NULL_STATUS = "NULL"
 
 failure_status_list = [ERROR_STATUS, FAIL_STATUS, MISS_STATUS]
 

backend/kernelCI_app/typeModels/hardwareDetails.py

@@ -79,6 +79,14 @@ class HardwareDetailsFullResponse(BaseModel):
     summary: HardwareSummary
 
 
-type HardwareTreeList = List[Dict[str, str]]
+class HardwareDetailsSummaryResponse(BaseModel):
+    summary: HardwareSummary
+
 
 PossibleTestType = Literal["test", "boot"]
+
+
+class HardwareBuildHistoryItem(BuildHistoryItem):
+    tree_name: Optional[str]
+    issue_id: Optional[str]
+    issue_version: Optional[str]

backend/kernelCI_app/urls.py

@@ -81,6 +81,10 @@ def viewCache(view):
          viewCache(views.HardwareDetailsCommitHistoryView),
          name="hardwareDetailsCommitHistory"
          ),
+    path("hardware/<str:hardware_id>/summary",
+         views.HardwareDetailsSummary.as_view(),
+         name="hardwareDetailsSummary"
+         ),
     path("hardware/",
          viewCache(views.HardwareView),
          name="hardware"),