diff --git a/.changelog/4707.yml b/.changelog/4707.yml new file mode 100644 index 00000000000..da68e7fa6fa --- /dev/null +++ b/.changelog/4707.yml @@ -0,0 +1,4 @@ +changes: +- description: Fixed an issue where the created date would conflict with the new XSOAR system field with the same name, by introducing a new field named `firstCreated`. + type: fix +pr_number: 4707 diff --git a/TestSuite/repo.py b/TestSuite/repo.py index b19a53702c7..899435c63c7 100644 --- a/TestSuite/repo.py +++ b/TestSuite/repo.py @@ -2,6 +2,7 @@ import shutil from pathlib import Path from typing import List, Optional +from unittest.mock import MagicMock from demisto_sdk.commands.common.constants import DEMISTO_GIT_PRIMARY_BRANCH from demisto_sdk.commands.common.git_util import GitUtil @@ -320,6 +321,7 @@ def create_graph(self, output_path: Path = None): return self.graph_interface def create_pack(self, name: Optional[str] = None): + GitUtil.get_file_creation_date = MagicMock(return_value="2024-12-19T11:49:45Z") # type: ignore[assignment] if name is None: name = f"pack_{len(self.packs)}" pack = Pack(self._packs_path, name, repo=self) diff --git a/demisto_sdk/commands/common/git_util.py b/demisto_sdk/commands/common/git_util.py index ffb2c6e45aa..621dbdad20f 100644 --- a/demisto_sdk/commands/common/git_util.py +++ b/demisto_sdk/commands/common/git_util.py @@ -1,5 +1,6 @@ import os import re +from datetime import datetime from functools import lru_cache from pathlib import Path from typing import List, Optional, Sequence, Set, Tuple, Union @@ -18,6 +19,7 @@ from demisto_sdk.commands.common.constants import ( DEMISTO_GIT_PRIMARY_BRANCH, DEMISTO_GIT_UPSTREAM, + ISO_TIMESTAMP_FORMAT, PACKS_FOLDER, ) from demisto_sdk.commands.common.logger import logger @@ -1194,3 +1196,9 @@ def stage_file(self, file_path: Union[Path, str]): logger.debug(f"Staged file '{file_path}'") else: logger.error(f"File '{file_path}' doesn't exist. Not adding.") + + def get_file_creation_date(self, file_path: Path) -> str: + if commits := list(self.repo.iter_commits(paths=file_path)): + first_commit = commits[-1] + return first_commit.authored_datetime.strftime(ISO_TIMESTAMP_FORMAT) + return datetime.now().strftime(ISO_TIMESTAMP_FORMAT) diff --git a/demisto_sdk/commands/common/tests/git_util_test.py b/demisto_sdk/commands/common/tests/git_util_test.py index 9442d4b1c47..d0f81b34faf 100644 --- a/demisto_sdk/commands/common/tests/git_util_test.py +++ b/demisto_sdk/commands/common/tests/git_util_test.py @@ -1,8 +1,10 @@ import stat +from datetime import datetime from pathlib import Path from git import Blob +from demisto_sdk.commands.common.constants import ISO_TIMESTAMP_FORMAT from TestSuite.repo import Repo @@ -144,3 +146,23 @@ def test_git_util_with_repo(): git_util = GitUtil(repo) assert git_util.repo is not None assert git_util.repo.working_dir == repo.working_dir + + +def test_get_file_creation_date(git_repo: Repo): + """ + Given: + - A git repo and a file in it. + + When: + - Retrieving the creation time of the given file. + + Then: + - The creation time of the file is returned. + """ + file = Path("pack_metadata.json") + git_repo.make_file(str(file), "{}") + git_repo.git_util.commit_files(f"added {file}", str(file)) + + file_creation_date = git_repo.git_util.get_file_creation_date(file) + + datetime.strptime(file_creation_date, ISO_TIMESTAMP_FORMAT) # raises if invalid diff --git a/demisto_sdk/commands/content_graph/objects/pack_metadata.py b/demisto_sdk/commands/content_graph/objects/pack_metadata.py index 179c6456aea..3f3a1dc4416 100644 --- a/demisto_sdk/commands/content_graph/objects/pack_metadata.py +++ b/demisto_sdk/commands/content_graph/objects/pack_metadata.py @@ -33,7 +33,7 @@ class PackMetadata(BaseModel): name: str display_name: str description: Optional[str] - created: Optional[str] + created: Optional[str] = Field(alias="firstCreated") updated: Optional[str] = Field("") legacy: Optional[bool] support: str = Field("") diff --git a/demisto_sdk/commands/content_graph/parsers/pack.py b/demisto_sdk/commands/content_graph/parsers/pack.py index 79a6c17644e..a744d6b80f5 100644 --- a/demisto_sdk/commands/content_graph/parsers/pack.py +++ b/demisto_sdk/commands/content_graph/parsers/pack.py @@ -140,7 +140,9 @@ def __init__(self, path: Path, metadata: Dict[str, Any]) -> None: self.display_name: str = metadata.get("name", "") self.description: str = metadata.get("description", "") self.support: str = metadata.get("support", "") - self.created: str = metadata.get("created") or NOW + self.created = metadata.get("firstCreated") or metadata.get("created") + if not self.created: + self.created = GitUtil().get_file_creation_date(file_path=path) self.updated: str = metadata.get("updated") or NOW self.legacy: bool = metadata.get( "legacy", metadata.get("partnerId") is None @@ -377,7 +379,7 @@ def field_mapping(self): return { "name": "name", "description": "description", - "created": "created", + "created": "firstCreated", "support": "support", "email": "email", "price": "price", diff --git a/demisto_sdk/commands/init/initiator.py b/demisto_sdk/commands/init/initiator.py index 33097425618..16614525c00 100644 --- a/demisto_sdk/commands/init/initiator.py +++ b/demisto_sdk/commands/init/initiator.py @@ -2,6 +2,7 @@ import os import re import shutil +from datetime import datetime from distutils.dir_util import copy_tree from pathlib import Path from typing import Dict, List, Optional, Set, Tuple @@ -29,6 +30,7 @@ INTEGRATION_CATEGORIES, INTEGRATIONS_DIR, INTEGRATIONS_DIR_REGEX, + ISO_TIMESTAMP_FORMAT, JOBS_DIR, LAYOUTS_DIR, MARKETPLACE_LIVE_DISCUSSIONS, @@ -68,6 +70,7 @@ from demisto_sdk.commands.secrets.secrets import SecretsValidator ANALYTICS_AND_SIEM_CATEGORY = "Analytics & SIEM" +NOW = datetime.now().strftime(ISO_TIMESTAMP_FORMAT) def extract_values_from_nested_dict_to_a_set(given_dictionary: dict, return_set: set): @@ -683,6 +686,7 @@ def create_metadata(fill_manually: bool, data: Dict = None) -> Dict: "useCases": [], "keywords": [], "marketplaces": MARKETPLACES, + "firstCreated": NOW, } if data: diff --git a/demisto_sdk/commands/init/tests/initiator_test.py b/demisto_sdk/commands/init/tests/initiator_test.py index dbd17813799..ecaeb46dc74 100644 --- a/demisto_sdk/commands/init/tests/initiator_test.py +++ b/demisto_sdk/commands/init/tests/initiator_test.py @@ -29,7 +29,11 @@ from demisto_sdk.commands.common.handlers import DEFAULT_JSON_HANDLER as json from demisto_sdk.commands.common.handlers import DEFAULT_YAML_HANDLER as yaml from demisto_sdk.commands.common.tools import get_file -from demisto_sdk.commands.init.initiator import ANALYTICS_AND_SIEM_CATEGORY, Initiator +from demisto_sdk.commands.init.initiator import ( + ANALYTICS_AND_SIEM_CATEGORY, + NOW, + Initiator, +) from TestSuite.test_tools import ChangeCWD DIR_NAME = "DirName" @@ -129,6 +133,7 @@ def test_create_metadata_non_filled_manually(self, initiator): "useCases": [], "keywords": [], "marketplaces": MARKETPLACES, + "firstCreated": NOW, } def test_create_metadata_non_filled_manually_with_data(self, initiator): @@ -162,6 +167,7 @@ def test_create_metadata_non_filled_manually_with_data(self, initiator): "useCases": [], "keywords": [], "marketplaces": MARKETPLACES, + "firstCreated": NOW, } def test_create_metadata_partner(self, monkeypatch, initiator): @@ -214,6 +220,7 @@ def test_create_metadata_partner(self, monkeypatch, initiator): "useCases": [], "githubUser": [], "marketplaces": MARKETPLACES, + "firstCreated": NOW, } def test_create_metadata_partner_wrong_url(self, monkeypatch, initiator): @@ -267,6 +274,7 @@ def test_create_metadata_partner_wrong_url(self, monkeypatch, initiator): "useCases": [], "githubUser": [], "marketplaces": ["xsoar"], + "firstCreated": NOW, } def test_create_metadata_community(self, monkeypatch, initiator): @@ -317,6 +325,7 @@ def test_create_metadata_community(self, monkeypatch, initiator): "useCases": [], "githubUser": [], "marketplaces": ["marketplacev2"], + "firstCreated": NOW, } diff --git a/demisto_sdk/commands/prepare_content/tests/yml_unifier_test.py b/demisto_sdk/commands/prepare_content/tests/yml_unifier_test.py index a91fd73c532..379f152cc2e 100644 --- a/demisto_sdk/commands/prepare_content/tests/yml_unifier_test.py +++ b/demisto_sdk/commands/prepare_content/tests/yml_unifier_test.py @@ -1188,6 +1188,7 @@ def test_unify_default_output_script(self, mocker): "tags": [], "useCases": [], "keywords": [], + "created": "2020-06-15T11:49:45Z", } ) PACK_METADATA_PARTNER_EMAIL_LIST = json.dumps( @@ -1203,6 +1204,7 @@ def test_unify_default_output_script(self, mocker): "tags": [], "useCases": [], "keywords": [], + "created": "2020-06-15T11:49:45Z", } ) PACK_METADATA_STRINGS_EMAIL_LIST = json.dumps( @@ -1218,6 +1220,7 @@ def test_unify_default_output_script(self, mocker): "tags": [], "useCases": [], "keywords": [], + "created": "2020-06-15T11:49:45Z", } ) PACK_METADATA_PARTNER_NO_EMAIL = json.dumps( @@ -1233,6 +1236,7 @@ def test_unify_default_output_script(self, mocker): "tags": [], "useCases": [], "keywords": [], + "created": "2020-06-15T11:49:45Z", } ) PACK_METADATA_PARTNER_NO_URL = json.dumps( @@ -1248,6 +1252,7 @@ def test_unify_default_output_script(self, mocker): "tags": [], "useCases": [], "keywords": [], + "created": "2020-06-15T11:49:45Z", } ) PACK_METADATA_XSOAR = json.dumps( @@ -1263,6 +1268,7 @@ def test_unify_default_output_script(self, mocker): "tags": [], "useCases": [], "keywords": [], + "created": "2020-06-15T11:49:45Z", } ) @@ -1279,6 +1285,7 @@ def test_unify_default_output_script(self, mocker): "tags": [], "useCases": [], "keywords": [], + "created": "2020-06-15T11:49:45Z", } ) @@ -1331,7 +1338,6 @@ def test_unify_partner_contributed_pack(mocker, repo): Then - Ensure unify create unified file with partner support notes. """ - pack = repo.create_pack("PackName") integration = pack.create_integration("integration", "bla", INTEGRATION_YAML) pack.pack_metadata.write_json(PACK_METADATA_PARTNER) @@ -1495,7 +1501,6 @@ def test_unify_partner_contributed_pack_no_url(mocker, repo): Then - Ensure unify create unified file with partner support notes. """ - pack = repo.create_pack("PackName") integration = pack.create_integration("integration", "bla", INTEGRATION_YAML) pack.pack_metadata.write_json(PACK_METADATA_PARTNER_NO_URL) @@ -1604,7 +1609,6 @@ def test_unify_community_contributed(mocker, repo): Then - Ensure unify create unified file with community detailed description. """ - pack = repo.create_pack("PackName") integration = pack.create_integration("integration", "bla", INTEGRATION_YAML) pack.pack_metadata.write_json(PACK_METADATA_COMMUNITY) diff --git a/demisto_sdk/tests/integration_tests/prepare_content_test.py b/demisto_sdk/tests/integration_tests/prepare_content_test.py index 43553f93ace..bd6a24582af 100644 --- a/demisto_sdk/tests/integration_tests/prepare_content_test.py +++ b/demisto_sdk/tests/integration_tests/prepare_content_test.py @@ -96,7 +96,9 @@ def test_unify_integration__detailed_description_partner_collector( """ name = "test" description = f"this is an integration {name}" - pack.pack_metadata.update({"support": "partner"}) + pack.pack_metadata.update( + {"support": "partner", "created": "2023-10-24T11:49:45Z"} + ) yml = { "commonfields": {"id": name, "version": -1}, "name": name, diff --git a/demisto_sdk/tests/integration_tests/unify_integration_test.py b/demisto_sdk/tests/integration_tests/unify_integration_test.py index 7977949c763..2e593c4a9ac 100644 --- a/demisto_sdk/tests/integration_tests/unify_integration_test.py +++ b/demisto_sdk/tests/integration_tests/unify_integration_test.py @@ -184,6 +184,7 @@ def test_add_custom_section_flag_integration(self, mocker, repo, flag): - make sure the nativeImage was key was added with the native-images. """ pack = repo.create_pack("PackName") + pack.pack_metadata.update({"created": "2023-10-24T11:49:45Z"}) integration = pack.create_integration("dummy-integration") integration.create_default_integration() @@ -242,6 +243,7 @@ def test_add_custom_section_flag(self, repo): - make sure the nativeimage was key was added with the native-images. """ pack = repo.create_pack("PackName") + pack.pack_metadata.update({"created": "2023-10-24T11:49:45Z"}) script = pack.create_script("dummy-script") script.create_default_script() @@ -267,6 +269,7 @@ def test_ignore_native_image_integration(self, monkeypatch, repo): - make sure the nativeimage key is not added to the integration unified yml. """ pack = repo.create_pack("PackName") + pack.pack_metadata.update({"created": "2023-10-24T11:49:45Z"}) integration = pack.create_integration("dummy-integration") integration.create_default_integration() @@ -295,6 +298,7 @@ def test_ignore_native_image_script(self, repo): - make sure the nativeImage key is not added to the script unified yml. """ pack = repo.create_pack("PackName") + pack.pack_metadata.update({"created": "2023-10-24T11:49:45Z"}) script = pack.create_script("dummy-script") script.create_default_script()