diff --git a/src/hats/catalog/__init__.py b/src/hats/catalog/__init__.py index ee69f91f..5897d917 100644 --- a/src/hats/catalog/__init__.py +++ b/src/hats/catalog/__init__.py @@ -6,5 +6,6 @@ from .catalog_type import CatalogType from .dataset.dataset import Dataset from .dataset.table_properties import TableProperties +from .map.map_catalog import MapCatalog from .margin_cache.margin_catalog import MarginCatalog from .partition_info import PartitionInfo diff --git a/src/hats/catalog/catalog_type.py b/src/hats/catalog/catalog_type.py index 61ac8fb3..be36757a 100644 --- a/src/hats/catalog/catalog_type.py +++ b/src/hats/catalog/catalog_type.py @@ -9,6 +9,7 @@ class CatalogType(str, Enum): ASSOCIATION = "association" INDEX = "index" MARGIN = "margin" + MAP = "map" @classmethod def all_types(cls): diff --git a/src/hats/catalog/dataset/table_properties.py b/src/hats/catalog/dataset/table_properties.py index 94542eed..008f9500 100644 --- a/src/hats/catalog/dataset/table_properties.py +++ b/src/hats/catalog/dataset/table_properties.py @@ -25,6 +25,7 @@ ], CatalogType.INDEX: ["primary_catalog", "indexing_column", "extra_columns"], CatalogType.MARGIN: ["primary_catalog", "margin_threshold", "ra_column", "dec_column", "default_columns"], + CatalogType.MAP: ["default_columns"], } ## catalog_name, catalog_type, and total_rows are required for ALL types @@ -40,6 +41,7 @@ ], CatalogType.INDEX: ["primary_catalog", "indexing_column"], CatalogType.MARGIN: ["primary_catalog", "margin_threshold"], + CatalogType.MAP: [], } # All additional properties in the HATS recommendation. diff --git a/src/hats/catalog/map/__init__.py b/src/hats/catalog/map/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/hats/catalog/map/map_catalog.py b/src/hats/catalog/map/map_catalog.py new file mode 100644 index 00000000..32cb6f15 --- /dev/null +++ b/src/hats/catalog/map/map_catalog.py @@ -0,0 +1,7 @@ +from __future__ import annotations + +from hats.catalog.healpix_dataset.healpix_dataset import HealpixDataset + + +class MapCatalog(HealpixDataset): + """A HATS table to represent non-point-source data in a continuous map.""" diff --git a/src/hats/loaders/read_hats.py b/src/hats/loaders/read_hats.py index f071e2d2..532f35cb 100644 --- a/src/hats/loaders/read_hats.py +++ b/src/hats/loaders/read_hats.py @@ -9,7 +9,7 @@ from upath import UPath import hats.pixel_math.healpix_shim as hp -from hats.catalog import AssociationCatalog, Catalog, CatalogType, Dataset, MarginCatalog +from hats.catalog import AssociationCatalog, Catalog, CatalogType, Dataset, MapCatalog, MarginCatalog from hats.catalog.association_catalog.partition_join_info import PartitionJoinInfo from hats.catalog.dataset.table_properties import TableProperties from hats.catalog.index.index_catalog import IndexCatalog @@ -23,6 +23,7 @@ CatalogType.ASSOCIATION: AssociationCatalog, CatalogType.INDEX: IndexCatalog, CatalogType.MARGIN: MarginCatalog, + CatalogType.MAP: MapCatalog, } @@ -63,6 +64,7 @@ def _is_healpix_dataset(dataset_type): CatalogType.SOURCE, CatalogType.ASSOCIATION, CatalogType.MARGIN, + CatalogType.MAP, ) diff --git a/tests/data/generate_data.ipynb b/tests/data/generate_data.ipynb index 242e4bc7..3e1c9032 100644 --- a/tests/data/generate_data.ipynb +++ b/tests/data/generate_data.ipynb @@ -297,6 +297,49 @@ " runner.pipeline_with_client(args, client)" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "from hats.pixel_math.spatial_index import healpix_to_spatial_index\n", + "\n", + "target_pixels = np.arange(0, 12)\n", + "\n", + "healpix_29 = healpix_to_spatial_index(0, target_pixels)\n", + "\n", + "square_vals = target_pixels * target_pixels\n", + "value_frame = pd.DataFrame({\"_healpix_29\": healpix_29, \"star_count\": square_vals})" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "with tempfile.TemporaryDirectory() as pipeline_tmp:\n", + " csv_file = Path(pipeline_tmp) / \"square_map.csv\"\n", + " value_frame.to_csv(csv_file, index=False)\n", + " args = ImportArguments(\n", + " constant_healpix_order=1,\n", + " catalog_type=\"map\",\n", + " use_healpix_29=True,\n", + " ra_column=None,\n", + " dec_column=None,\n", + " file_reader=\"csv\",\n", + " input_file_list=[csv_file],\n", + " output_artifact_name=\"square_map\",\n", + " output_path=\".\",\n", + " tmp_dir=pipeline_tmp,\n", + " )\n", + "\n", + " runner.pipeline_with_client(args, client)" + ] + }, { "cell_type": "code", "execution_count": null, @@ -317,7 +360,7 @@ ], "metadata": { "kernelspec": { - "display_name": "env", + "display_name": "demo", "language": "python", "name": "python3" }, @@ -331,7 +374,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.12" + "version": "3.12.3" } }, "nbformat": 4, diff --git a/tests/data/square_map/dataset/Norder=1/Dir=0/Npix=0.parquet b/tests/data/square_map/dataset/Norder=1/Dir=0/Npix=0.parquet new file mode 100644 index 00000000..d51cc17a Binary files /dev/null and b/tests/data/square_map/dataset/Norder=1/Dir=0/Npix=0.parquet differ diff --git a/tests/data/square_map/dataset/Norder=1/Dir=0/Npix=12.parquet b/tests/data/square_map/dataset/Norder=1/Dir=0/Npix=12.parquet new file mode 100644 index 00000000..657359aa Binary files /dev/null and b/tests/data/square_map/dataset/Norder=1/Dir=0/Npix=12.parquet differ diff --git a/tests/data/square_map/dataset/Norder=1/Dir=0/Npix=16.parquet b/tests/data/square_map/dataset/Norder=1/Dir=0/Npix=16.parquet new file mode 100644 index 00000000..ccb88d01 Binary files /dev/null and b/tests/data/square_map/dataset/Norder=1/Dir=0/Npix=16.parquet differ diff --git a/tests/data/square_map/dataset/Norder=1/Dir=0/Npix=20.parquet b/tests/data/square_map/dataset/Norder=1/Dir=0/Npix=20.parquet new file mode 100644 index 00000000..73ee4190 Binary files /dev/null and b/tests/data/square_map/dataset/Norder=1/Dir=0/Npix=20.parquet differ diff --git a/tests/data/square_map/dataset/Norder=1/Dir=0/Npix=24.parquet b/tests/data/square_map/dataset/Norder=1/Dir=0/Npix=24.parquet new file mode 100644 index 00000000..e98d36a6 Binary files /dev/null and b/tests/data/square_map/dataset/Norder=1/Dir=0/Npix=24.parquet differ diff --git a/tests/data/square_map/dataset/Norder=1/Dir=0/Npix=28.parquet b/tests/data/square_map/dataset/Norder=1/Dir=0/Npix=28.parquet new file mode 100644 index 00000000..61faf754 Binary files /dev/null and b/tests/data/square_map/dataset/Norder=1/Dir=0/Npix=28.parquet differ diff --git a/tests/data/square_map/dataset/Norder=1/Dir=0/Npix=32.parquet b/tests/data/square_map/dataset/Norder=1/Dir=0/Npix=32.parquet new file mode 100644 index 00000000..fc36e5ea Binary files /dev/null and b/tests/data/square_map/dataset/Norder=1/Dir=0/Npix=32.parquet differ diff --git a/tests/data/square_map/dataset/Norder=1/Dir=0/Npix=36.parquet b/tests/data/square_map/dataset/Norder=1/Dir=0/Npix=36.parquet new file mode 100644 index 00000000..c8c5cba2 Binary files /dev/null and b/tests/data/square_map/dataset/Norder=1/Dir=0/Npix=36.parquet differ diff --git a/tests/data/square_map/dataset/Norder=1/Dir=0/Npix=4.parquet b/tests/data/square_map/dataset/Norder=1/Dir=0/Npix=4.parquet new file mode 100644 index 00000000..9ece3d2b Binary files /dev/null and b/tests/data/square_map/dataset/Norder=1/Dir=0/Npix=4.parquet differ diff --git a/tests/data/square_map/dataset/Norder=1/Dir=0/Npix=40.parquet b/tests/data/square_map/dataset/Norder=1/Dir=0/Npix=40.parquet new file mode 100644 index 00000000..6f8c8f77 Binary files /dev/null and b/tests/data/square_map/dataset/Norder=1/Dir=0/Npix=40.parquet differ diff --git a/tests/data/square_map/dataset/Norder=1/Dir=0/Npix=44.parquet b/tests/data/square_map/dataset/Norder=1/Dir=0/Npix=44.parquet new file mode 100644 index 00000000..3c64d2fe Binary files /dev/null and b/tests/data/square_map/dataset/Norder=1/Dir=0/Npix=44.parquet differ diff --git a/tests/data/square_map/dataset/Norder=1/Dir=0/Npix=8.parquet b/tests/data/square_map/dataset/Norder=1/Dir=0/Npix=8.parquet new file mode 100644 index 00000000..21421910 Binary files /dev/null and b/tests/data/square_map/dataset/Norder=1/Dir=0/Npix=8.parquet differ diff --git a/tests/data/square_map/dataset/_common_metadata b/tests/data/square_map/dataset/_common_metadata new file mode 100644 index 00000000..71be4790 Binary files /dev/null and b/tests/data/square_map/dataset/_common_metadata differ diff --git a/tests/data/square_map/dataset/_metadata b/tests/data/square_map/dataset/_metadata new file mode 100644 index 00000000..fc765808 Binary files /dev/null and b/tests/data/square_map/dataset/_metadata differ diff --git a/tests/data/square_map/partition_info.csv b/tests/data/square_map/partition_info.csv new file mode 100644 index 00000000..798921b7 --- /dev/null +++ b/tests/data/square_map/partition_info.csv @@ -0,0 +1,13 @@ +Norder,Npix +1,0 +1,4 +1,8 +1,12 +1,16 +1,20 +1,24 +1,28 +1,32 +1,36 +1,40 +1,44 diff --git a/tests/data/square_map/point_map.fits b/tests/data/square_map/point_map.fits new file mode 100644 index 00000000..3f1bd5f0 Binary files /dev/null and b/tests/data/square_map/point_map.fits differ diff --git a/tests/data/square_map/properties b/tests/data/square_map/properties new file mode 100644 index 00000000..c28a0585 --- /dev/null +++ b/tests/data/square_map/properties @@ -0,0 +1,12 @@ +#HATS catalog +obs_collection=square_map +dataproduct_type=map +hats_nrows=12 +hats_max_rows=1000000 +hats_order=1 +moc_sky_fraction=0.25000 +hats_builder=hats-import v0.4.1 +hats_creation_date=2024-11-20T19\:32UTC +hats_estsize=52 +hats_release_date=2024-09-18 +hats_version=v0.1 diff --git a/tests/hats/catalog/loaders/test_read_hats.py b/tests/hats/catalog/loaders/test_read_hats.py index fd8511e3..ad37b683 100644 --- a/tests/hats/catalog/loaders/test_read_hats.py +++ b/tests/hats/catalog/loaders/test_read_hats.py @@ -8,6 +8,7 @@ def test_read_hats_branches( small_sky_source_object_index_dir, margin_catalog_path, small_sky_source_dir, + test_data_dir, ): read_hats(small_sky_dir) read_hats(small_sky_order1_dir) @@ -15,3 +16,4 @@ def test_read_hats_branches( read_hats(small_sky_source_object_index_dir) read_hats(margin_catalog_path) read_hats(small_sky_source_dir) + read_hats(test_data_dir / "square_map")