Refactor Validations

.changelog/4753.yml

@@ -0,0 +1,8 @@
+changes:
+- description: "Added IF109 to the new validation format. The validation Checks whether an incident or indicator field
+ has a valid required field value."
+  type: feature
+- description: "Added IF117 to the new validation format. The validation Checks whether an incident or indicator field
+ aliases have a valid marketplaces field value."
+  type: feature
+pr_number: 4753

demisto_sdk/commands/content_graph/objects/indicator_incident_field.py

@@ -19,6 +19,8 @@ class IndicatorIncidentField(ContentItem):
     group: int = Field(None, exclude=True)
     unsearchable: Optional[bool] = Field(None, exclude=True)
     version: Optional[int] = 0
+    required: bool = Field(None, alias="required")
+    associated_types: list = Field(None, alias="associatedTypes")
 
     def _upload(
         self,

demisto_sdk/commands/content_graph/parsers/incident_field.py

@@ -29,6 +29,8 @@ def __init__(
         self.content = self.json_data.get("content")
         self.system = self.json_data.get("system")
         self.group = self.json_data.get("group")
+        self.required = self.json_data.get("required")
+        self.associated_types = self.json_data.get("associatedTypes")
 
         self.connect_to_dependencies()
 

demisto_sdk/commands/content_graph/parsers/indicator_field.py

@@ -25,6 +25,8 @@ def __init__(
         super().__init__(path, pack_marketplaces, git_sha=git_sha)
         self.associated_to_all = self.json_data.get("associatedToAll")
         self.select_values = self.json_data.get("selectValues")
+        self.required = self.json_data.get("required")
+        self.associated_types = self.json_data.get("associatedTypes")
 
         self.connect_to_dependencies()
 

demisto_sdk/commands/validate/sdk_validation_config.toml

@@ -227,6 +227,7 @@ select = [
   "IF105",
   "IF106",
   "IF116",
+  "IF117",
   "IF118",
   "IF119",
   "IM101",
@@ -470,10 +471,12 @@ select = [
   "IF104",
   "IF105",
   "IF106",
+  "IF109",
   "IF111",
   "IF113",
   "IF115",
   "IF116",
+  "IF117",
   "IF118",
   "IF119",
   "IM100",

demisto_sdk/commands/validate/tests/IF_validators_test.py

@@ -2,10 +2,12 @@
 
 import pytest
 
+from demisto_sdk.commands.common.constants import GitStatuses, MarketplaceVersions
 from demisto_sdk.commands.content_graph.objects.incident_field import IncidentField
 from demisto_sdk.commands.validate.tests.test_tools import (
     REPO,
     create_incident_field_object,
+    create_incident_type_object,
     create_old_file_pointers,
 )
 from demisto_sdk.commands.validate.validators.IF_validators.IF100_is_valid_name_and_cli_name import (
@@ -32,6 +34,9 @@
     INCIDENT_PROHIBITED_CLI_NAMES,
     IsCliNameReservedWordValidator,
 )
+from demisto_sdk.commands.validate.validators.IF_validators.IF109_invalid_required_field import (
+    IsValidRequiredFieldValidator,
+)
 from demisto_sdk.commands.validate.validators.IF_validators.IF111_is_field_type_changed import (
     IsFieldTypeChangedValidator,
 )
@@ -45,6 +50,9 @@
 from demisto_sdk.commands.validate.validators.IF_validators.IF116_select_values_cannot_contain_empty_values_in_multi_select_types import (
     SelectValuesCannotContainEmptyValuesInMultiSelectTypesValidator,
 )
+from demisto_sdk.commands.validate.validators.IF_validators.IF117_invalid_marketplaces_in_alias import (
+    IsValidAliasMarketplaceValidator,
+)
 from demisto_sdk.commands.validate.validators.IF_validators.IF118_is_alias_inner_alias_valid import (
     IsAliasInnerAliasValidator,
 )
@@ -745,3 +753,118 @@ def test_IsAliasInnerAliasValidator():
         result[0].message
         == "The following aliases have inner aliases: alias_1, alias_2"
     )
+
+
+def test_IsValidAliasMarketplaceValidator(mocker):
+    """
+    Given:
+    - An incident field with aliases, one of them with invalid marketplaces field.
+    When:
+    - Running validate on an incident field.
+    Then:
+    - Validate that the incorrect alias is caught.
+    """
+    aliases_names = [
+        {
+            "cliName": "alias_1",
+            "type": "shortText",
+            "marketplaces": [MarketplaceVersions.XSOAR],
+        },
+        {
+            "cliName": "alias_2",
+            "type": "shortText",
+            "marketplaces": [MarketplaceVersions.MarketplaceV2],
+        },
+    ]
+    inc_field = create_incident_field_object(
+        ["Aliases"],
+        [aliases_names],
+    )
+    aliases = []
+    for item in aliases_names:
+        alias = create_incident_field_object(
+            paths=["cliName"], values=[item.get("cliName")]
+        )
+        alias.marketplaces = item.get("marketplaces")
+        aliases.append(alias)
+
+    mocker.patch.object(IsValidAliasMarketplaceValidator, "graph", return_value="graph")
+    mocker.patch.object(
+        IsValidAliasMarketplaceValidator,
+        "_get_incident_fields_by_aliases",
+        return_value=aliases,
+    )
+    result = IsValidAliasMarketplaceValidator().obtain_invalid_content_items(
+        [inc_field]
+    )
+
+    assert (
+        result[0].message
+        == "The following fields exist as aliases and have invalid 'marketplaces' key value: \nalias_2\n "
+        "the value of the 'marketplaces' key in these fields should be ['xsoar']."
+    )
+
+
+@pytest.mark.parametrize(
+    "items, expected_number_of_failures, expected_msgs",
+    [
+        (
+            {
+                create_incident_field_object(
+                    paths=["required", "associatedToAll"], values=["false", "true"]
+                ): GitStatuses.MODIFIED
+            },
+            0,
+            [],
+        ),  # inc field not required -> okay
+        (
+            {
+                create_incident_field_object(
+                    paths=["required", "associatedToAll"], values=["true", "true"]
+                ): GitStatuses.ADDED
+            },
+            1,
+            ["A required IncidentField should not be associated with all types."],
+        ),
+        (
+            {
+                create_incident_field_object(
+                    paths=["required", "associatedToAll", "associatedTypes"],
+                    values=["true", "false", ["New Type", "Old Type"]],
+                ): GitStatuses.ADDED,
+                create_incident_type_object(
+                    paths=["id"], values=["New Type"]
+                ): GitStatuses.ADDED,
+            },
+            1,
+            [
+                "An already existing Types like Old Type cannot be added to an IncidentField with required value equals true."
+            ],
+        ),
+    ],
+)
+def test_IsValidRequiredFieldValidator(
+    items, expected_number_of_failures, expected_msgs
+):
+    """
+    Given:
+    - Incident fields.
+    When:
+    - Running validate on an incident fields.
+    Then:
+    - Validate that the field has a valid required value.
+    """
+    content_items = []
+    for item, status in items.items():
+        item.git_status = status
+        item.old_base_content_object = item.copy(deep=True)
+        content_items.append(item)
+
+    result = IsValidRequiredFieldValidator().obtain_invalid_content_items(content_items)
+    assert len(result) == len(expected_msgs)
+    assert all(
+        [
+            result.message == expected_msg
+            for result, expected_msg in zip(result, expected_msgs)
+        ]
+    )

demisto_sdk/commands/validate/validators/IF_validators/IF109_invalid_required_field.py

@@ -0,0 +1,133 @@
+from __future__ import annotations
+
+from typing import Iterable, List, Union, cast
+
+from demisto_sdk.commands.common.constants import GitStatuses
+from demisto_sdk.commands.content_graph.objects.incident_field import IncidentField
+from demisto_sdk.commands.content_graph.objects.incident_type import IncidentType
+from demisto_sdk.commands.content_graph.objects.indicator_field import IndicatorField
+from demisto_sdk.commands.content_graph.objects.indicator_type import IndicatorType
+from demisto_sdk.commands.validate.validators.base_validator import (
+    BaseValidator,
+    ValidationResult,
+)
+
+ContentTypes = Union[IncidentField, IndicatorField, IncidentType, IndicatorType]
+
+
+class IsValidRequiredFieldValidator(BaseValidator[ContentTypes]):
+    error_code = "IF109"
+    description = "Checks if the incident field required field is valid."
+    rationale = "In case an incident field is required, newly added associated incident types should be new."
+    error_message = "{0}"
+    related_field = "required"
+    expected_git_statuses = [GitStatuses.MODIFIED, GitStatuses.ADDED]
+
+    def obtain_invalid_content_items(
+        self, content_items: Iterable[ContentTypes]
+    ) -> List[ValidationResult]:
+        fields_items, types_items = self.sort_content_items(content_items)
+        return [
+            ValidationResult(
+                validator=self,
+                message=self.error_message.format(error_res),
+                content_object=content_item,
+            )
+            for content_item in fields_items
+            if (error_res := self.is_invalid_required_field(content_item, types_items))
+        ]
+
+    @staticmethod
+    def sort_content_items(content_items: Iterable[ContentTypes]):
+        """
+        Sort Content Items into two lists of Incident/Indicator fields and Incident/Indicator types.
+
+        Args:
+            content_items (Iterable[ContentTypes]): The content items list
+
+        Returns:
+            fields_items: items of type Incident/Indicator fields.
+            types_items: items of type Incident/Indicator types.
+        """
+        types_items: list[str] = []
+        fields_items: list[Union[IncidentField, IndicatorField]] = []
+        for item in content_items:
+            if (
+                isinstance(item, IncidentType) or isinstance(item, IndicatorType)
+            ) and item.git_status == GitStatuses.ADDED:
+                types_items.append(item.object_id)
+            elif isinstance(item, IncidentField) or isinstance(item, IndicatorField):
+                fields_items.append(item)
+
+        return fields_items, types_items
+
+    @staticmethod
+    def is_invalid_required_field(
+        content_item: Union[IncidentField, IndicatorField], added_types: list[str]
+    ):
+        """
+        Get from the graph the actual fields for the given aliases
+
+        Args:
+            content_item (Union[IncidentField, IndicatorField]): The incident field or indicator field to check its required field.
+            added_types (list): incident or indicator types that were created/added in the same pull request.
+
+        Returns:
+            str or None: return none in case it's valid, the error message in case it's not.
+        """
+        # Required fields should not be associated to all
+        if content_item.required and content_item.associated_to_all:
+            return (
+                f"A required {content_item.content_type.value}"
+                f" should not be associated with all types."
+            )
+
+        if content_item.git_status == GitStatuses.MODIFIED:
+            old_file = cast(
+                Union[IncidentField, IndicatorField],
+                content_item.old_base_content_object,
+            )
+
+            # Required value for an already existing field cannot be changed
+            if content_item.required != old_file.required:
+                return (
+                    f"Required value should not be changed for an already existing"
+                    f" {content_item.content_type.value}."
+                )
+
+            # An already existing Incident/Indicator Type cannot be added to Incident/Indicator Field with required value true
+            if content_item.required and len(content_item.associated_types) > len(
+                old_file.associated_types
+            ):
+                new_types = list(
+                    filter(
+                        lambda x: x not in old_file.associated_types,
+                        content_item.associated_types,
+                    )
+                )
+                invalid_new_types = []
+                for new_type in new_types:
+                    if new_type not in added_types:
+                        invalid_new_types.append(new_type)
+                if invalid_new_types:
+                    return (
+                        f"An already existing Type like {', '.join(invalid_new_types)} cannot be added to an "
+                        f"{content_item.content_type.value} "
+                        f"with required value equals true."
+                    )
+
+        # new field
+        elif content_item.required:
+            associated_types = content_item.associated_types
+            invalid_associated_types = []
+            # An already existing Incident/Indicator Type cannot be added to Incident/Indicator Field with required value true
+            for associated_type in associated_types:
+                if associated_type not in added_types:
+                    invalid_associated_types.append(associated_type)
+            if invalid_associated_types:
+                return (
+                    f"An already existing Types like {', '.join(invalid_associated_types)} cannot be added to an "
+                    f"{content_item.content_type.value} "
+                    f"with required value equals true."
+                )
+        return