Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Trim some cloud tests. #176

Merged
merged 2 commits into from
Dec 1, 2023
Merged

Trim some cloud tests. #176

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4901402
Trim some cloud tests.
delucchi-cmu Nov 30, 2023
0f1806b
Expanding docstring
delucchi-cmu Nov 30, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .pre-commit-config.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,11 +75,11 @@ repos:
- repo: local
hooks:
- id: pylint
name: pylint (python files in tests/ and benchmarks/)
name: pylint (test-ish python files)
entry: pylint
language: system
types: [python]
files: ^(tests|benchmarks)/
files: ^(tests|benchmarks|cloud_tests)/
args:
[
"-rn", # Only display messages
Expand Down
2 changes: 1 addition & 1 deletion cloud_tests/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,5 +59,5 @@ def example_cloud_storage_options(cloud):
return {}
```

* 3.) Finally, you will need to copy the entire `/tests/data/` directory into your newly created bucket. This can be accomplished by running the `copy_data_to_fs.py` script in the `cloud_tests/` directory.
* 3.) Finally, you will need to copy several `/tests/data/` directories into your newly created bucket. This can be accomplished by running the `copy_data_to_fs.py` script in the `cloud_tests/` directory.
* 4.) Before running the tests, you will need to export your `valid_storage_option_param` into the environment.
202 changes: 7 additions & 195 deletions cloud_tests/conftest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,25 +1,17 @@
import dataclasses
import os
import os.path
from typing import List

import pandas as pd
import pytest

from hipscat.catalog import PartitionInfo
from hipscat.catalog.association_catalog.association_catalog_info import AssociationCatalogInfo
from hipscat.catalog.association_catalog.partition_join_info import PartitionJoinInfo
from hipscat.catalog.catalog_info import CatalogInfo
from hipscat.catalog.dataset.base_catalog_info import BaseCatalogInfo
from hipscat.inspection.almanac import Almanac
from hipscat.pixel_math import HealpixPixel

DATA_DIR_NAME = "data"
ALMANAC_DIR_NAME = "almanac"
SMALL_SKY_DIR_NAME = "small_sky"
SMALL_SKY_ORDER1_DIR_NAME = "small_sky_order1"
SMALL_SKY_TO_SMALL_SKY_ORDER1_DIR_NAME = "small_sky_to_small_sky_order1"
TEST_DIR = os.path.dirname(__file__)

# pylint: disable=missing-function-docstring, redefined-outer-name

Expand All @@ -39,10 +31,9 @@ def pytest_generate_tests(metafunc):
@pytest.fixture
def example_cloud_path(cloud):
if cloud == "abfs":
return "abfs:///hipscat/pytests/hipscat"
return "abfs://hipscat/pytests/hipscat"

else:
raise NotImplementedError("Cloud format not implemented for hipscat tests!")
raise NotImplementedError("Cloud format not implemented for hipscat tests!")


@pytest.fixture
Expand Down Expand Up @@ -83,68 +74,8 @@ def small_sky_order1_dir_cloud(test_data_dir_cloud):


@pytest.fixture
def small_sky_to_small_sky_order1_dir_cloud(test_data_dir_cloud):
return os.path.join(test_data_dir_cloud, SMALL_SKY_TO_SMALL_SKY_ORDER1_DIR_NAME)


@pytest.fixture
def catalog_pixels() -> List[HealpixPixel]:
return [HealpixPixel(1, 0), HealpixPixel(1, 1), HealpixPixel(2, 8)]


@pytest.fixture
def association_catalog_path_cloud(test_data_dir_cloud) -> str:
return os.path.join(test_data_dir_cloud, "small_sky_to_small_sky_order1")


@pytest.fixture
def association_catalog_info_file_cloud(association_catalog_path_cloud) -> str:
return os.path.join(association_catalog_path_cloud, "catalog_info.json")


@pytest.fixture
def index_catalog_info_file_cloud(test_data_dir_cloud) -> str:
return os.path.join(test_data_dir_cloud, "index_catalog", "catalog_info.json")


@pytest.fixture
def margin_cache_catalog_info_file_cloud(test_data_dir_cloud) -> str:
return os.path.join(test_data_dir_cloud, "margin_cache", "catalog_info.json")


@pytest.fixture
def source_catalog_info_file_cloud(test_data_dir_cloud) -> str:
return os.path.join(test_data_dir_cloud, "small_sky_source", "catalog_info.json")


@pytest.fixture
def association_catalog_info(association_catalog_info_data) -> AssociationCatalogInfo:
return AssociationCatalogInfo(**association_catalog_info_data)


@pytest.fixture
def association_catalog_partition_join_file_cloud(association_catalog_path_cloud) -> str:
return os.path.join(association_catalog_path_cloud, "partition_join_info.csv")


@pytest.fixture
def dataset_path_cloud(test_data_dir_cloud) -> str:
return os.path.join(test_data_dir_cloud, "dataset")


@pytest.fixture
def base_catalog_info_file_cloud(dataset_path_cloud) -> str:
return os.path.join(dataset_path_cloud, "catalog_info.json")


@pytest.fixture
def base_catalog_info(base_catalog_info_data) -> BaseCatalogInfo:
return BaseCatalogInfo(**base_catalog_info_data)


@pytest.fixture
def catalog_path_cloud(test_data_dir_cloud) -> str:
return os.path.join(test_data_dir_cloud, "catalog")
def base_catalog_info_file_cloud(test_data_dir_cloud) -> str:
return os.path.join(test_data_dir_cloud, "dataset", "catalog_info.json")


@pytest.fixture
Expand All @@ -153,18 +84,9 @@ def catalog_info_file_cloud(catalog_path_cloud) -> str:


@pytest.fixture
def test_data_dir():
return os.path.join(TEST_DIR, DATA_DIR_NAME)


@pytest.fixture
def small_sky_dir_local(test_data_dir):
return os.path.join(test_data_dir, SMALL_SKY_DIR_NAME)


@pytest.fixture
def small_sky_order1_dir_local(test_data_dir):
return os.path.join(test_data_dir, SMALL_SKY_ORDER1_DIR_NAME)
def small_sky_dir_local():
cloud_test_path = os.path.dirname(__file__)
return os.path.join(cloud_test_path, "..", "tests", "data", SMALL_SKY_DIR_NAME)


@pytest.fixture
Expand All @@ -179,15 +101,6 @@ def assert_match(catalog_info: BaseCatalogInfo, dictionary: dict):
return assert_match


@pytest.fixture
def base_catalog_info_data() -> dict:
return {
"catalog_name": "test_name",
"catalog_type": "object",
"total_rows": 10,
}


@pytest.fixture
def catalog_info_data() -> dict:
return {
Expand All @@ -200,107 +113,6 @@ def catalog_info_data() -> dict:
}


@pytest.fixture
def association_catalog_info_data() -> dict:
return {
"catalog_name": "test_name",
"catalog_type": "association",
"total_rows": 10,
"primary_catalog": "small_sky",
"primary_column": "id",
"join_catalog": "small_sky_order1",
"join_column": "id",
}


@pytest.fixture
def source_catalog_info() -> dict:
return {
"catalog_name": "test_source",
"catalog_type": "source",
"total_rows": 100,
"epoch": "J2000",
"ra_column": "source_ra",
"dec_column": "source_dec",
}


@pytest.fixture
def source_catalog_info_with_extra() -> dict:
return {
"catalog_name": "test_source",
"catalog_type": "source",
"total_rows": 100,
"epoch": "J2000",
"ra_column": "source_ra",
"dec_column": "source_dec",
"primary_catalog": "test_name",
"mjd_column": "mjd",
"band_column": "band",
"mag_column": "mag",
"mag_err_column": "",
}


@pytest.fixture
def margin_cache_catalog_info() -> dict:
return {
"catalog_name": "test_margin",
"catalog_type": "margin",
"total_rows": 100,
"primary_catalog": "test_name",
"margin_threshold": 0.5,
}


@pytest.fixture
def index_catalog_info() -> dict:
return {
"catalog_name": "test_index",
"catalog_type": "index",
"total_rows": 100,
"primary_catalog": "test_name",
"indexing_column": "id",
}


@pytest.fixture
def index_catalog_info_with_extra() -> dict:
return {
"catalog_name": "test_index",
"catalog_type": "index",
"total_rows": 100,
"primary_catalog": "test_name",
"indexing_column": "id",
"extra_columns": ["foo", "bar"],
}


@pytest.fixture
def catalog_info(catalog_info_data) -> CatalogInfo:
return CatalogInfo(**catalog_info_data)


@pytest.fixture
def association_catalog_join_pixels() -> pd.DataFrame:
return pd.DataFrame.from_dict(
{
PartitionJoinInfo.PRIMARY_ORDER_COLUMN_NAME: [0, 0, 0, 0],
PartitionJoinInfo.PRIMARY_PIXEL_COLUMN_NAME: [11, 11, 11, 11],
PartitionJoinInfo.JOIN_ORDER_COLUMN_NAME: [1, 1, 1, 1],
PartitionJoinInfo.JOIN_PIXEL_COLUMN_NAME: [44, 45, 46, 47],
}
)


@pytest.fixture
def default_almanac_cloud(example_cloud_path, example_cloud_storage_options):
"""Set up default environment variables and fetch default almanac data."""

test_data_dir = os.path.join(example_cloud_path, "data")
almanac_dir = os.path.join(example_cloud_path, "data", "almanac")

os.environ["HIPSCAT_ALMANAC_DIR"] = almanac_dir
os.environ["HIPSCAT_DEFAULT_DIR"] = test_data_dir

return Almanac(storage_options=example_cloud_storage_options)
8 changes: 6 additions & 2 deletions cloud_tests/copy_data_to_fs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,12 +64,16 @@ def copy_dir(source_fs, source_fp, destination_fs, desintation_fp, verbose=False


if __name__ == "__main__":
# pylint: disable=invalid-name,duplicate-key

source_pw = f"{os.getcwd()}/../tests/data"
target_pw = "new_protocol:///path/to/pytest/hipscat"

target_so = {
"valid_storage_option_param1": os.environ.get("NEW_PROTOCOL_PARAM1"),
"valid_storage_option_param1": os.environ.get("NEW_PROTOCOL_PARAM2"),
"valid_storage_option_param2": os.environ.get("NEW_PROTOCOL_PARAM2"),
}
copy_tree_fs_to_fs(source_pw, target_pw, {}, target_so, verbose=True)

for data_dir in ["almanac", "dataset", "small_sky", "small_sky_order1"]:
source_dir = os.path.join(source_pw, data_dir)
copy_tree_fs_to_fs(source_dir, target_pw, {}, target_so, verbose=True)
77 changes: 0 additions & 77 deletions cloud_tests/hipscat/catalog/association_catalog/test_association_catalog_cloud.py
View file
Open in desktop

This file was deleted.

Loading