From a7d0799100ed5d2e6a9ed0891b3dc4e5e1922f9a Mon Sep 17 00:00:00 2001 From: Tobias Grigo <56518487+hstct@users.noreply.github.com> Date: Mon, 20 Nov 2023 13:43:30 +0100 Subject: [PATCH] Merge pull request #920 from ATIX-AG/test-pulp2pulp Add pulp to pulp verification tests (cherry picked from commit 3a872659fcb25f813938d94c671c3bc362393168) --- CHANGES/919.misc | 1 + .../tests/functional/api/test_pulp_to_pulp.py | 207 ++++++++++++++++++ pulp_deb/tests/functional/conftest.py | 7 +- pulp_deb/tests/functional/constants.py | 65 ++++++ 4 files changed, 279 insertions(+), 1 deletion(-) create mode 100644 CHANGES/919.misc create mode 100644 pulp_deb/tests/functional/api/test_pulp_to_pulp.py diff --git a/CHANGES/919.misc b/CHANGES/919.misc new file mode 100644 index 000000000..4c7462fe0 --- /dev/null +++ b/CHANGES/919.misc @@ -0,0 +1 @@ +Added tests that verify the download of content served by ``pulp_deb``. diff --git a/pulp_deb/tests/functional/api/test_pulp_to_pulp.py b/pulp_deb/tests/functional/api/test_pulp_to_pulp.py new file mode 100644 index 000000000..71839152e --- /dev/null +++ b/pulp_deb/tests/functional/api/test_pulp_to_pulp.py @@ -0,0 +1,207 @@ +"""Tests that verify download of content served by Pulp.""" +import pytest + +from pulp_deb.tests.functional.constants import ( + DEB_P2P_SIMPLE_THEN_STRUCTURED, + DEB_PUBLICATION_ARGS_ONLY_SIMPLE, + DEB_PUBLICATION_ARGS_ONLY_STRUCTURED, + DEB_PUBLICATION_ARGS_SIMPLE_AND_STRUCTURED, + DEB_P2P_ONLY_SIMPLE, + DEB_P2P_ONLY_STRUCTURED, + DEB_P2P_SIMPLE_AND_STRUCTURED, + DEB_P2P_REMOTE_ARGS_SIMPLE, + DEB_P2P_REMOTE_ARGS_STRUCTURED, + DEB_P2P_REMOTE_ARGS_BOTH, + DEB_P2P_REMOTE_ARGS_VERBATIM, +) +from pulp_deb.tests.functional.utils import get_counts_from_content_summary + +pulp_to_pulp_test_data = [ + pytest.param( + "immediate", + False, + DEB_PUBLICATION_ARGS_ONLY_SIMPLE, + DEB_P2P_REMOTE_ARGS_SIMPLE, + DEB_P2P_ONLY_SIMPLE, + id="p2p-immediate-simple", + ), + pytest.param( + "immediate", + False, + DEB_PUBLICATION_ARGS_ONLY_STRUCTURED, + DEB_P2P_REMOTE_ARGS_STRUCTURED, + DEB_P2P_ONLY_STRUCTURED, + id="p2p-immediate-structured", + ), + pytest.param( + "immediate", + False, + DEB_PUBLICATION_ARGS_SIMPLE_AND_STRUCTURED, + DEB_P2P_REMOTE_ARGS_BOTH, + DEB_P2P_SIMPLE_AND_STRUCTURED, + id="p2p-immediate-both", + ), + pytest.param( + "immediate", True, {}, DEB_P2P_REMOTE_ARGS_VERBATIM, {}, id="p2p-immediate-verbatim" + ), + pytest.param( + "streamed", + False, + DEB_PUBLICATION_ARGS_ONLY_SIMPLE, + DEB_P2P_REMOTE_ARGS_SIMPLE, + DEB_P2P_ONLY_SIMPLE, + id="p2p-streamed-simple", + ), + pytest.param( + "streamed", + False, + DEB_PUBLICATION_ARGS_ONLY_STRUCTURED, + DEB_P2P_REMOTE_ARGS_STRUCTURED, + DEB_P2P_ONLY_STRUCTURED, + id="p2p-streamed-structured", + ), + pytest.param( + "streamed", + False, + DEB_PUBLICATION_ARGS_SIMPLE_AND_STRUCTURED, + DEB_P2P_REMOTE_ARGS_BOTH, + DEB_P2P_SIMPLE_AND_STRUCTURED, + id="p2p-streamed-both", + ), + pytest.param( + "streamed", True, {}, DEB_P2P_REMOTE_ARGS_VERBATIM, {}, id="p2p-streamed-verbatim" + ), + pytest.param( + "on_demand", + False, + DEB_PUBLICATION_ARGS_ONLY_SIMPLE, + DEB_P2P_REMOTE_ARGS_SIMPLE, + DEB_P2P_ONLY_SIMPLE, + id="p2p-on_demand-simple", + ), + pytest.param( + "on_demand", + False, + DEB_PUBLICATION_ARGS_ONLY_STRUCTURED, + DEB_P2P_REMOTE_ARGS_STRUCTURED, + DEB_P2P_ONLY_STRUCTURED, + id="p2p-on_demand-structured", + ), + pytest.param( + "on_demand", + False, + DEB_PUBLICATION_ARGS_SIMPLE_AND_STRUCTURED, + DEB_P2P_REMOTE_ARGS_BOTH, + DEB_P2P_SIMPLE_AND_STRUCTURED, + id="p2p-on_demand-both", + ), + pytest.param( + "on_demand", True, {}, DEB_P2P_REMOTE_ARGS_VERBATIM, {}, id="p2p-on_demand-verbatim" + ), +] + + +@pytest.mark.parametrize( + "policy, is_verbatim, publication_args, remote_args_new, expected_value", pulp_to_pulp_test_data +) +def test_pulp_to_pulp_sync( + deb_get_content_summary, + deb_init_and_sync, + deb_distribution_factory, + is_verbatim, + policy, + publication_args, + request, + remote_args_new, + expected_value, + delete_orphans_pre, +): + """Verify whether content served by Pulp can be synced. + + The initial sync to Pulp is one of many different download policies, the second sync is + immediate in order to exercise downloading all of the files. + + Multiple test cases are covered here which are a combination of different policies and + different publish methods: + + * `Test policy=immediate and publish=simple` + * `Test policy=immediate and publish=structured` + * `Test policy=immediate and publish=simple+structured` + * `Test policy=immediate and publish=verbatim` + * `Test policy=streamed and publish=simple` + * `Test policy=streamed and publish=structured` + * `Test policy=streamed and publish=simple+structured` + * `Test policy=streamed and publish=verbatim` + * `Test policy=on_demand and publish=simple` + * `Test policy=on_demand and publish=structured` + * `Test policy=on_demand and publish=simple+structured` + * `Test policy=on_demand and publish=verbatim` + """ + remote_args = {"policy": policy} + publication_factory = ( + request.getfixturevalue("deb_verbatim_publication_factory") + if is_verbatim + else request.getfixturevalue("deb_publication_factory") + ) + + # Create and sync a repository. Then publish and distribute it. + repo, _ = deb_init_and_sync(remote_args=remote_args) + publication = publication_factory(repo, **publication_args) + distribution = deb_distribution_factory(publication) + + # Create and sync a new repository with the published repo as the remote + repo_new, _ = deb_init_and_sync(url=distribution.base_url, remote_args=remote_args_new) + + # Assert whether the present content matches the expected data + summary_new = deb_get_content_summary(repo_new) + summary_orig = deb_get_content_summary(repo) + present_new = get_counts_from_content_summary(summary_new.present) + if is_verbatim: + present_orig = get_counts_from_content_summary(summary_orig.present) + assert present_orig == present_new + else: + assert present_new == expected_value + + # Assert whether the added content matches the expected data + added_new = get_counts_from_content_summary(summary_new.added) + if is_verbatim: + added_orig = get_counts_from_content_summary(summary_orig.added) + assert added_orig == added_new + else: + assert added_new == expected_value + + +def test_pulp_to_pulp_sync_simple_to_structured( + deb_init_and_sync, + deb_publication_factory, + deb_distribution_factory, + deb_get_content_summary, + delete_orphans_pre, +): + """Verify whether a repository served by Pulp can sync its simple content first + and in a concurrent sync simple and structured content without loss of data. + """ + # Create and sync a repository. Then publish and distribute it. + repo, _ = deb_init_and_sync(remote_args={"policy": "immediate"}) + publication = deb_publication_factory(repo, **DEB_PUBLICATION_ARGS_SIMPLE_AND_STRUCTURED) + distribution = deb_distribution_factory(publication) + + # Create and sync a new repository with published repo as the remote and simple distribution + repo_simple, _ = deb_init_and_sync( + url=distribution.base_url, remote_args=DEB_P2P_REMOTE_ARGS_SIMPLE + ) + summary_simple = deb_get_content_summary(repo_simple) + present_simple = get_counts_from_content_summary(summary_simple.present) + added_simple = get_counts_from_content_summary(summary_simple.added) + assert present_simple == DEB_P2P_ONLY_SIMPLE + assert added_simple == DEB_P2P_ONLY_SIMPLE + + # Use the same respository and sync again with published repo as remote but both distributions + repo_both, _ = deb_init_and_sync( + repository=repo_simple, url=distribution.base_url, remote_args=DEB_P2P_REMOTE_ARGS_BOTH + ) + summary_both = deb_get_content_summary(repo_both) + present_both = get_counts_from_content_summary(summary_both.present) + added_both = get_counts_from_content_summary(summary_both.added) + assert added_both == DEB_P2P_SIMPLE_THEN_STRUCTURED + assert present_both == DEB_P2P_SIMPLE_AND_STRUCTURED diff --git a/pulp_deb/tests/functional/conftest.py b/pulp_deb/tests/functional/conftest.py index 284df1a95..d252aaf18 100644 --- a/pulp_deb/tests/functional/conftest.py +++ b/pulp_deb/tests/functional/conftest.py @@ -603,7 +603,12 @@ def _deb_init_and_sync( :param return_task: Whether to include the sync task to the return value. Default: False. :returns: A tuple containing the repository and remote and optionally the sync task. """ - url = deb_get_fixture_server_url() if url is None else deb_get_fixture_server_url(url) + if url is None: + url = deb_get_fixture_server_url() + elif url.startswith("http"): + url = url + else: + url = deb_get_fixture_server_url(url) if repository is None: repository = deb_repository_factory(**repo_args) if remote is None: diff --git a/pulp_deb/tests/functional/constants.py b/pulp_deb/tests/functional/constants.py index 276c9f889..bee49ad12 100644 --- a/pulp_deb/tests/functional/constants.py +++ b/pulp_deb/tests/functional/constants.py @@ -43,6 +43,11 @@ def _clean_dict(d): DEB_PUBLICATION_ARGS_SIMPLE_AND_STRUCTURED = {"simple": True, "structured": True} DEB_PUBLICATION_ARGS_ALL = {"simple": True, "structured": True, "signing_service": ""} +DEB_P2P_REMOTE_ARGS_SIMPLE = {"distributions": "default", "policy": "immediate"} +DEB_P2P_REMOTE_ARGS_STRUCTURED = {"distributions": "ragnarok nosuite", "policy": "immediate"} +DEB_P2P_REMOTE_ARGS_BOTH = {"distributions": "ragnarok nosuite default", "policy": "immediate"} +DEB_P2P_REMOTE_ARGS_VERBATIM = {"distributions": "ragnarok nosuite", "policy": "immediate"} + DEB_FIXTURE_SUMMARY = _clean_dict( { DEB_RELEASE_NAME: 2, @@ -103,6 +108,66 @@ def _clean_dict(d): } ) +DEB_P2P_ONLY_SIMPLE = _clean_dict( + { + DEB_RELEASE_NAME: 1, + DEB_RELEASE_ARCHITECTURE_NAME: 3, + DEB_RELEASE_COMPONENT_NAME: 1, + DEB_RELEASE_FILE_NAME: 1, + DEB_PACKAGE_INDEX_NAME: 3, + DEB_PACKAGE_RELEASE_COMPONENT_NAME: 4, + DEB_INSTALLER_FILE_INDEX_NAME: 0, + DEB_PACKAGE_NAME: 4, + DEB_INSTALLER_PACKAGE_NAME: 0, + DEB_GENERIC_CONTENT_NAME: 0, + } +) + +DEB_P2P_ONLY_STRUCTURED = _clean_dict( + { + DEB_RELEASE_NAME: 2, + DEB_RELEASE_ARCHITECTURE_NAME: 5, + DEB_RELEASE_COMPONENT_NAME: 3, + DEB_RELEASE_FILE_NAME: 2, + DEB_PACKAGE_INDEX_NAME: 8, + DEB_PACKAGE_RELEASE_COMPONENT_NAME: 7, + DEB_INSTALLER_FILE_INDEX_NAME: 0, + DEB_PACKAGE_NAME: 4, + DEB_INSTALLER_PACKAGE_NAME: 0, + DEB_GENERIC_CONTENT_NAME: 0, + } +) + +DEB_P2P_SIMPLE_AND_STRUCTURED = _clean_dict( + { + DEB_RELEASE_NAME: 3, + DEB_RELEASE_ARCHITECTURE_NAME: 8, + DEB_RELEASE_COMPONENT_NAME: 4, + DEB_RELEASE_FILE_NAME: 3, + DEB_PACKAGE_INDEX_NAME: 11, + DEB_PACKAGE_RELEASE_COMPONENT_NAME: 11, + DEB_INSTALLER_FILE_INDEX_NAME: 0, + DEB_PACKAGE_NAME: 8, + DEB_INSTALLER_PACKAGE_NAME: 0, + DEB_GENERIC_CONTENT_NAME: 0, + } +) + +DEB_P2P_SIMPLE_THEN_STRUCTURED = _clean_dict( + { + DEB_RELEASE_NAME: 2, + DEB_RELEASE_ARCHITECTURE_NAME: 5, + DEB_RELEASE_COMPONENT_NAME: 3, + DEB_RELEASE_FILE_NAME: 2, + DEB_PACKAGE_INDEX_NAME: 8, + DEB_PACKAGE_RELEASE_COMPONENT_NAME: 7, + DEB_INSTALLER_FILE_INDEX_NAME: 0, + DEB_PACKAGE_NAME: 4, + DEB_INSTALLER_PACKAGE_NAME: 0, + DEB_GENERIC_CONTENT_NAME: 0, + } +) + DEB_FIXTURE_PACKAGE_COUNT = DEB_FIXTURE_SUMMARY.get(DEB_PACKAGE_NAME, 0) DEB_REPORT_CODE_SKIP_RELEASE = "sync.release_file.was_skipped"