diff --git a/src/.pylintrc b/src/.pylintrc index 39b138d9..39dac7b5 100644 --- a/src/.pylintrc +++ b/src/.pylintrc @@ -62,7 +62,7 @@ ignore-patterns=_version.py # (useful for modules/projects where namespaces are manipulated during runtime # and thus existing member attributes cannot be deduced by static analysis). It # supports qualified module names, as well as Unix pattern matching. -ignored-modules= +ignored-modules=astropy.units # Python code to execute, usually for sys.path manipulation such as # pygtk.require(). diff --git a/src/hipscat/catalog/catalog.py b/src/hipscat/catalog/catalog.py index 1ff90abb..5b0dbacb 100644 --- a/src/hipscat/catalog/catalog.py +++ b/src/hipscat/catalog/catalog.py @@ -13,13 +13,9 @@ from hipscat.catalog.catalog_type import CatalogType from hipscat.catalog.healpix_dataset.healpix_dataset import HealpixDataset, PixelInputTypes from hipscat.pixel_math import HealpixPixel -from hipscat.pixel_math.box_filter import filter_pixels_by_box, wrap_ra_angles -from hipscat.pixel_math.cone_filter import filter_pixels_by_cone -from hipscat.pixel_math.polygon_filter import ( - CartesianCoordinates, - SphericalCoordinates, - filter_pixels_by_polygon, -) +from hipscat.pixel_math.box_filter import generate_box_moc, wrap_ra_angles +from hipscat.pixel_math.cone_filter import generate_cone_moc +from hipscat.pixel_math.polygon_filter import CartesianCoordinates, SphericalCoordinates, generate_polygon_moc from hipscat.pixel_math.validators import ( validate_box_search, validate_declination_values, @@ -85,7 +81,7 @@ def filter_by_cone(self, ra: float, dec: float, radius_arcsec: float) -> Catalog """ validate_radius(radius_arcsec) validate_declination_values(dec) - return self.filter_from_pixel_list(filter_pixels_by_cone(self.pixel_tree, ra, dec, radius_arcsec)) + return self.filter_by_moc(generate_cone_moc(ra, dec, radius_arcsec, self.get_max_coverage_order())) def filter_by_box( self, ra: Tuple[float, float] | None = None, dec: Tuple[float, float] | None = None @@ -103,7 +99,7 @@ def filter_by_box( """ ra = tuple(wrap_ra_angles(ra)) if ra else None validate_box_search(ra, dec) - return self.filter_from_pixel_list(filter_pixels_by_box(self.pixel_tree, ra, dec)) + return self.filter_by_moc(generate_box_moc(ra, dec, self.get_max_coverage_order())) def filter_by_polygon(self, vertices: List[SphericalCoordinates] | List[CartesianCoordinates]) -> Catalog: """Filter the pixels in the catalog to only include the pixels that overlap @@ -122,9 +118,11 @@ def filter_by_polygon(self, vertices: List[SphericalCoordinates] | List[Cartesia validate_declination_values(dec) # Get the coordinates vector on the unit sphere if we were provided # with polygon spherical coordinates of ra and dec - vertices = hp.ang2vec(ra, dec, lonlat=True) - validate_polygon(vertices) - return self.filter_from_pixel_list(filter_pixels_by_polygon(self.pixel_tree, vertices)) + cart_vertices = hp.ang2vec(ra, dec, lonlat=True) + else: + cart_vertices = vertices + validate_polygon(cart_vertices) + return self.filter_by_moc(generate_polygon_moc(cart_vertices, self.get_max_coverage_order())) def generate_negative_tree_pixels(self) -> List[HealpixPixel]: """Get the leaf nodes at each healpix order that have zero catalog data. diff --git a/src/hipscat/catalog/healpix_dataset/healpix_dataset.py b/src/hipscat/catalog/healpix_dataset/healpix_dataset.py index 635731c2..7315330d 100644 --- a/src/hipscat/catalog/healpix_dataset/healpix_dataset.py +++ b/src/hipscat/catalog/healpix_dataset/healpix_dataset.py @@ -14,6 +14,7 @@ from hipscat.io import FilePointer, file_io, paths from hipscat.pixel_math import HealpixPixel from hipscat.pixel_tree import PixelAlignment, PixelAlignmentType +from hipscat.pixel_tree.moc_filter import filter_by_moc from hipscat.pixel_tree.pixel_alignment import align_with_mocs from hipscat.pixel_tree.pixel_tree import PixelTree @@ -131,18 +132,45 @@ def _check_files_exist(cls, catalog_base_dir: FilePointer, storage_options: dict f"_metadata or partition info file is required in catalog directory {catalog_base_dir}" ) + def get_max_coverage_order(self) -> int: + """Gets the maximum HEALPix order for which the coverage of the catalog is known from the pixel + tree and moc if it exists""" + max_order = ( + max(self.moc.max_order, self.pixel_tree.get_max_depth()) + if self.moc is not None + else self.pixel_tree.get_max_depth() + ) + return max_order + def filter_from_pixel_list(self, pixels: List[HealpixPixel]) -> Self: - """Filter the pixels in the catalog to only include the requested pixels. + """Filter the pixels in the catalog to only include any that overlap with the requested pixels. Args: pixels (List[HealpixPixels]): the pixels to include Returns: - A new catalog with only those pixels. Note that we reset the total_rows - to None, instead of performing a scan over the new pixel sizes. + A new catalog with only the pixels that overlap with the given pixels. Note that we reset the + total_rows to None, as updating would require a scan over the new pixel sizes. """ + orders = np.array([p.order for p in pixels]) + pixel_inds = np.array([p.pixel for p in pixels]) + max_order = np.max(orders) if len(orders) > 0 else 0 + moc = MOC.from_healpix_cells(ipix=pixel_inds, depth=orders, max_depth=max_order) + return self.filter_by_moc(moc) + + def filter_by_moc(self, moc: MOC) -> Self: + """Filter the pixels in the catalog to only include the pixels that overlap with the moc provided. + + Args: + moc (mocpy.MOC): the moc to filter by + + Returns: + A new catalog with only the pixels that overlap with the moc. Note that we reset the total_rows + to None, as updating would require a scan over the new pixel sizes.""" + filtered_tree = filter_by_moc(self.pixel_tree, moc) + filtered_moc = self.moc.intersection(moc) if self.moc is not None else None filtered_catalog_info = dataclasses.replace(self.catalog_info, total_rows=None) - return self.__class__(filtered_catalog_info, pixels) + return self.__class__(filtered_catalog_info, filtered_tree, moc=filtered_moc) def align( self, other_cat: Self, alignment_type: PixelAlignmentType = PixelAlignmentType.INNER diff --git a/src/hipscat/catalog/margin_cache/margin_catalog.py b/src/hipscat/catalog/margin_cache/margin_catalog.py index 0b65bc25..5bbc8f07 100644 --- a/src/hipscat/catalog/margin_cache/margin_catalog.py +++ b/src/hipscat/catalog/margin_cache/margin_catalog.py @@ -1,11 +1,13 @@ from __future__ import annotations +import healpy as hp from mocpy import MOC -from typing_extensions import TypeAlias +from typing_extensions import Self, TypeAlias from hipscat.catalog.catalog_type import CatalogType from hipscat.catalog.healpix_dataset.healpix_dataset import HealpixDataset, PixelInputTypes from hipscat.catalog.margin_cache import MarginCacheCatalogInfo +from hipscat.pixel_tree.moc_utils import copy_moc class MarginCatalog(HealpixDataset): @@ -46,3 +48,27 @@ def __init__( super().__init__( catalog_info, pixels, catalog_path=catalog_path, moc=moc, storage_options=storage_options ) + + def filter_by_moc(self, moc: MOC) -> Self: + """Filter the pixels in the margin catalog to only include the margin pixels that overlap with the moc + + For the case of margin pixels, this includes any pixels whose margin areas may overlap with the moc. + This is not always done with a high accuracy, but always includes any pixels that will overlap, + and may include extra partitions that do not. + + Args: + moc (mocpy.MOC): the moc to filter by + + Returns: + A new margin catalog with only the pixels that overlap or that have margin area that overlap with + the moc. Note that we reset the total_rows to None, as updating would require a scan over the new + pixel sizes.""" + max_order = moc.max_order + max_order_size = hp.nside2resol(2**max_order, arcmin=True) + if self.catalog_info.margin_threshold > max_order_size * 60: + raise ValueError( + f"Cannot Filter Margin: Margin size {self.catalog_info.margin_threshold} is " + f"greater than the size of a pixel at the highest order {max_order}." + ) + search_moc = copy_moc(moc).add_neighbours() + return super().filter_by_moc(search_moc) diff --git a/src/hipscat/io/file_io/file_io.py b/src/hipscat/io/file_io/file_io.py index 51ea0686..d465fc7b 100644 --- a/src/hipscat/io/file_io/file_io.py +++ b/src/hipscat/io/file_io/file_io.py @@ -2,6 +2,7 @@ import json import tempfile +import warnings from typing import Any, Dict, Tuple, Union import healpy as hp @@ -278,8 +279,17 @@ def read_fits_image(map_file_pointer: FilePointer, storage_options: Union[Dict[A with file_system.open(map_file_pointer, "rb") as _map_file: map_data = _map_file.read() _tmp_file.write(map_data) - map_fits_image = hp.read_map(_tmp_file.name) - return map_fits_image + map_fits_image = hp.read_map(_tmp_file.name, nest=True, h=True) + header_dict = dict(map_fits_image[1]) + if header_dict["ORDERING"] != "NESTED": + warnings.warn( + "point_map.fits file written in RING ordering, due to " + "https://github.com/astronomy-commons/hipscat/issues/271. " + "Converting to NESTED." + ) + map_fits_image = hp.read_map(_tmp_file.name) + return map_fits_image + return map_fits_image[0] def write_fits_image( @@ -296,7 +306,7 @@ def write_fits_image( file_system, map_file_pointer = get_fs(file_pointer=map_file_pointer, storage_options=storage_options) with tempfile.NamedTemporaryFile() as _tmp_file: with file_system.open(map_file_pointer, "wb") as _map_file: - hp.write_map(_tmp_file.name, histogram, overwrite=True, dtype=np.int64) + hp.write_map(_tmp_file.name, histogram, overwrite=True, dtype=np.int64, nest=True) _map_file.write(_tmp_file.read()) diff --git a/src/hipscat/io/write_metadata.py b/src/hipscat/io/write_metadata.py index 1e925ecd..2afda08d 100644 --- a/src/hipscat/io/write_metadata.py +++ b/src/hipscat/io/write_metadata.py @@ -12,6 +12,7 @@ from hipscat.io import file_io, paths from hipscat.io.parquet_metadata import write_parquet_metadata as wpm +from hipscat.pixel_math.healpix_pixel import HealpixPixel class HipscatEncoder(json.JSONEncoder): @@ -26,6 +27,12 @@ def default(self, o): return str(o) if isinstance(o, (type, np.dtype, pd.core.dtypes.base.ExtensionDtype)): return str(o) + if isinstance(o, HealpixPixel): + return str(o) + if np.issubdtype(o.dtype, np.integer): + return int(o) + if isinstance(o, np.ndarray): + return o.tolist() return super().default(o) # pragma: no cover diff --git a/src/hipscat/pixel_math/box_filter.py b/src/hipscat/pixel_math/box_filter.py index 0042ccce..2dd9d9f9 100644 --- a/src/hipscat/pixel_math/box_filter.py +++ b/src/hipscat/pixel_math/box_filter.py @@ -4,48 +4,34 @@ import healpy as hp import numpy as np +from mocpy import MOC -from hipscat.pixel_math import HealpixPixel -from hipscat.pixel_math.filter import get_filtered_pixel_list from hipscat.pixel_math.polygon_filter import SphericalCoordinates -from hipscat.pixel_tree import PixelAlignmentType, align_trees -from hipscat.pixel_tree.pixel_tree import PixelTree -def filter_pixels_by_box( - pixel_tree: PixelTree, ra: Tuple[float, float] | None, dec: Tuple[float, float] | None -) -> List[HealpixPixel]: - """Filter the leaf pixels in a pixel tree to return a partition_info dataframe - with the pixels that overlap with a right ascension and/or declination region. +def generate_box_moc(ra: Tuple[float, float] | None, dec: Tuple[float, float] | None, order: int) -> MOC: + """Generates a MOC object that covers the specified box area Args: - pixel_tree (PixelTree): The catalog tree to filter pixels from ra (Tuple[float, float]): Right ascension range, in [0,360] degrees dec (Tuple[float, float]): Declination range, in [-90,90] degrees + order (int): Maximum order of the moc to generate the box at Returns: - List of HEALPix representing only the pixels that overlap with the right - ascension and/or declination region. + a MOC object that covers the specified box """ - max_order = pixel_tree.get_max_depth() - - filter_tree = None - ra_search_tree, dec_search_tree = None, None + filter_moc = None + ra_search_moc, dec_search_moc = None, None if ra is not None: - ra_search_tree = _generate_ra_strip_pixel_tree(ra, max_order) - filter_tree = ra_search_tree + ra_search_moc = _generate_ra_strip_moc(ra, order) + filter_moc = ra_search_moc if dec is not None: - dec_search_tree = _generate_dec_strip_pixel_tree(dec, max_order) - filter_tree = dec_search_tree - if ra_search_tree is not None and dec_search_tree is not None: - filter_tree = align_trees( - ra_search_tree, dec_search_tree, alignment_type=PixelAlignmentType.INNER - ).pixel_tree - - result_pixels = get_filtered_pixel_list(pixel_tree, filter_tree) - - return result_pixels + dec_search_moc = _generate_dec_strip_moc(dec, order) + filter_moc = dec_search_moc + if ra_search_moc is not None and dec_search_moc is not None: + filter_moc = ra_search_moc.intersection(dec_search_moc) + return filter_moc def wrap_ra_angles(ra: np.ndarray | Iterable | int | float) -> np.ndarray: @@ -60,7 +46,7 @@ def wrap_ra_angles(ra: np.ndarray | Iterable | int | float) -> np.ndarray: return np.asarray(ra, dtype=float) % 360 -def _generate_ra_strip_pixel_tree(ra_range: Tuple[float, float], order: int) -> PixelTree: +def _generate_ra_strip_moc(ra_range: Tuple[float, float], order: int) -> MOC: """Generates a pixel_tree filled with leaf nodes at a given order that overlap with the ra region.""" # Subdivide polygons (if needed) in two smaller polygons of at most 180 degrees division_ra = _get_division_ra(ra_range) @@ -90,11 +76,11 @@ def _generate_ra_strip_pixel_tree(ra_range: Tuple[float, float], order: int) -> order, ) - pixel_list = [HealpixPixel(order, polygon_pixel) for polygon_pixel in pixels_in_range] - return PixelTree.from_healpix(pixel_list) + orders = np.full(pixels_in_range.shape, fill_value=order) + return MOC.from_healpix_cells(ipix=pixels_in_range, depth=orders, max_depth=order) -def _generate_dec_strip_pixel_tree(dec_range: Tuple[float, float], order: int) -> PixelTree: +def _generate_dec_strip_moc(dec_range: Tuple[float, float], order: int) -> MOC: """Generates a pixel_tree filled with leaf nodes at a given order that overlap with the dec region.""" nside = hp.order2nside(order) # Convert declination values to colatitudes, in radians, and revert their order @@ -103,8 +89,8 @@ def _generate_dec_strip_pixel_tree(dec_range: Tuple[float, float], order: int) - pixels_in_range = hp.ring2nest( nside, hp.query_strip(nside, theta1=colat_rad[0], theta2=colat_rad[1], inclusive=True) ) - pixel_list = [HealpixPixel(order, polygon_pixel) for polygon_pixel in pixels_in_range] - return PixelTree.from_healpix(pixel_list) + orders = np.full(pixels_in_range.shape, fill_value=order) + return MOC.from_healpix_cells(ipix=pixels_in_range, depth=orders, max_depth=order) def _get_division_ra(ra_range: Tuple[float, float]) -> float | None: diff --git a/src/hipscat/pixel_math/cone_filter.py b/src/hipscat/pixel_math/cone_filter.py index d02b0ea3..168bb36e 100644 --- a/src/hipscat/pixel_math/cone_filter.py +++ b/src/hipscat/pixel_math/cone_filter.py @@ -1,37 +1,7 @@ -from typing import List +import astropy.units as u +from mocpy import MOC -import healpy as hp -import numpy as np -from hipscat.pixel_math import HealpixPixel -from hipscat.pixel_math.filter import get_filtered_pixel_list -from hipscat.pixel_tree.pixel_tree import PixelTree - - -def filter_pixels_by_cone( - pixel_tree: PixelTree, ra: float, dec: float, radius_arcsec: float -) -> List[HealpixPixel]: - """Filter the leaf pixels in a pixel tree to return a partition_info dataframe with the pixels - that overlap with a cone. - - Args: - ra (float): Right Ascension of the center of the cone in degrees - dec (float): Declination of the center of the cone in degrees - radius_arcsec (float): Radius of the cone in arcseconds - - Returns: - List of HealpixPixels representing only the pixels that overlap with the specified cone. - """ - max_order = pixel_tree.get_max_depth() - cone_tree = _generate_cone_pixel_tree(ra, dec, radius_arcsec, max_order) - return get_filtered_pixel_list(pixel_tree, cone_tree) - - -def _generate_cone_pixel_tree(ra: float, dec: float, radius_arcsec: float, order: int): - """Generates a pixel_tree filled with leaf nodes at a given order that overlap with a cone""" - n_side = hp.order2nside(order) - center_vec = hp.ang2vec(ra, dec, lonlat=True) - radius_radians = np.radians(radius_arcsec / 3600.0) - cone_pixels = hp.query_disc(n_side, center_vec, radius_radians, inclusive=True, nest=True) - pixel_list = [HealpixPixel(order, cone_pixel) for cone_pixel in cone_pixels] - return PixelTree.from_healpix(pixel_list) +def generate_cone_moc(ra: float, dec: float, radius_arcsec: float, order: int) -> MOC: + """Generate a MOC object that covers the cone""" + return MOC.from_cone(lon=ra * u.deg, lat=dec * u.deg, radius=radius_arcsec * u.arcsec, max_depth=order) diff --git a/src/hipscat/pixel_math/polygon_filter.py b/src/hipscat/pixel_math/polygon_filter.py index 059f9d31..64a4d7d5 100644 --- a/src/hipscat/pixel_math/polygon_filter.py +++ b/src/hipscat/pixel_math/polygon_filter.py @@ -1,15 +1,12 @@ from __future__ import annotations -from typing import List, Tuple +from typing import Tuple import healpy as hp import numpy as np +from mocpy import MOC from typing_extensions import TypeAlias -from hipscat.pixel_math import HealpixPixel -from hipscat.pixel_math.filter import get_filtered_pixel_list -from hipscat.pixel_tree.pixel_tree import PixelTree - # Pair of spherical sky coordinates (ra, dec) SphericalCoordinates: TypeAlias = Tuple[float, float] @@ -17,30 +14,10 @@ CartesianCoordinates: TypeAlias = Tuple[float, float, float] -def filter_pixels_by_polygon( - pixel_tree: PixelTree, vertices: List[CartesianCoordinates] -) -> List[HealpixPixel]: - """Filter the leaf pixels in a pixel tree to return a list of healpix pixels that - overlap with a polygonal region. - - Args: - pixel_tree (PixelTree): The catalog tree to filter pixels from. - vertices (List[CartesianCoordinates]): The vertices of the polygon to filter points - with, in lists of (x,y,z) points on the unit sphere. - - Returns: - List of HealpixPixel, representing only the pixels that overlap - with the specified polygonal region, and the maximum pixel order. - """ - max_order = pixel_tree.get_max_depth() - polygon_tree = _generate_polygon_pixel_tree(vertices, max_order) - return get_filtered_pixel_list(pixel_tree, polygon_tree) - - -def _generate_polygon_pixel_tree(vertices: np.array, order: int) -> PixelTree: - """Generates a pixel_tree filled with leaf nodes at a given order that overlap within - a polygon. Vertices is an array of 3D coordinates, in cartesian representation (x,y,z) +def generate_polygon_moc(vertices: np.array, order: int) -> MOC: + """Generates a moc filled with leaf nodes at a given order that overlap within + a polygon. Vertices is an array of cartesian coordinates, in representation (x,y,z) and shape (Num vertices, 3), representing the vertices of the polygon.""" polygon_pixels = hp.query_polygon(hp.order2nside(order), vertices, inclusive=True, nest=True) - pixel_list = [HealpixPixel(order, polygon_pixel) for polygon_pixel in polygon_pixels] - return PixelTree.from_healpix(pixel_list) + polygon_orders = np.full(len(polygon_pixels), fill_value=order) + return MOC.from_healpix_cells(ipix=polygon_pixels, depth=polygon_orders, max_depth=order) diff --git a/src/hipscat/pixel_tree/moc_utils.py b/src/hipscat/pixel_tree/moc_utils.py new file mode 100644 index 00000000..42cb9ce6 --- /dev/null +++ b/src/hipscat/pixel_tree/moc_utils.py @@ -0,0 +1,6 @@ +from mocpy import MOC + + +def copy_moc(moc: MOC) -> MOC: + """Returns a copy of a given MOC object""" + return MOC.from_depth29_ranges(max_depth=moc.max_order, ranges=moc.to_depth29_ranges) diff --git a/tests/.pylintrc b/tests/.pylintrc index 75f20f1e..cf331269 100644 --- a/tests/.pylintrc +++ b/tests/.pylintrc @@ -62,7 +62,7 @@ ignore-patterns=_version.py # (useful for modules/projects where namespaces are manipulated during runtime # and thus existing member attributes cannot be deduced by static analysis). It # supports qualified module names, as well as Unix pattern matching. -ignored-modules= +ignored-modules=astropy.units # Python code to execute, usually for sys.path manipulation such as # pygtk.require(). diff --git a/tests/data/generate_data.ipynb b/tests/data/generate_data.ipynb index 3aae325d..ba8c6b53 100644 --- a/tests/data/generate_data.ipynb +++ b/tests/data/generate_data.ipynb @@ -31,11 +31,13 @@ "from hipscat_import.margin_cache.margin_cache_arguments import MarginCacheArguments\n", "import tempfile\n", "from pathlib import Path\n", + "from dask.distributed import Client\n", "\n", "tmp_path = tempfile.TemporaryDirectory()\n", "tmp_dir = tmp_path.name\n", "\n", - "hipscat_import_dir = \"../../../hipscat-import/tests/hipscat_import/data/\"" + "hipscat_import_dir = \"../../../hipscat-import/tests/hipscat_import/data/\"\n", + "client = Client(n_workers=1, threads_per_worker=1, local_directory=tmp_dir)" ] }, { @@ -63,10 +65,9 @@ " output_path=\".\",\n", " file_reader=\"csv\",\n", " output_artifact_name=\"small_sky\",\n", - " overwrite=True,\n", " tmp_dir=tmp_dir,\n", ")\n", - "runner.pipeline(args)" + "runner.pipeline_with_client(args, client)" ] }, { @@ -100,10 +101,9 @@ " file_reader=\"csv\",\n", " output_artifact_name=\"small_sky_order1\",\n", " constant_healpix_order=1,\n", - " overwrite=True,\n", " tmp_dir=tmp_dir,\n", ")\n", - "runner.pipeline(args)" + "runner.pipeline_with_client(args, client)" ] }, { @@ -130,10 +130,9 @@ " output_artifact_name=\"small_sky_order1_id_index\",\n", " include_hipscat_index=False,\n", " compute_partition_size=200_000,\n", - " overwrite=True,\n", " tmp_dir=tmp_dir,\n", ")\n", - "runner.pipeline(args)" + "runner.pipeline_with_client(args, client)" ] }, { @@ -167,10 +166,9 @@ " input_catalog_path=\"small_sky_order1\",\n", " output_path=\".\",\n", " output_artifact_name=\"small_sky_order1_margin\",\n", - " overwrite=True,\n", " tmp_dir=tmp_dir,\n", ")\n", - "runner.pipeline(margin_args)" + "runner.pipeline_with_client(args, client)" ] }, { @@ -241,10 +239,9 @@ " catalog_type=\"source\",\n", " pixel_threshold=3000,\n", " output_artifact_name=\"small_sky_source\",\n", - " overwrite=True,\n", " tmp_dir=tmp_dir,\n", ")\n", - "runner.pipeline(args)" + "runner.pipeline_with_client(args, client)" ] }, { @@ -282,10 +279,9 @@ " output_artifact_name=\"small_sky_source_object_index\",\n", " include_hipscat_index=False,\n", " compute_partition_size=200_000,\n", - " overwrite=True,\n", " tmp_dir=tmp_dir,\n", ")\n", - "runner.pipeline(args)" + "runner.pipeline_with_client(args, client)" ] }, { @@ -294,6 +290,7 @@ "metadata": {}, "outputs": [], "source": [ + "client.close()\n", "tmp_path.cleanup()" ] } @@ -314,7 +311,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.13" + "version": "3.10.14" } }, "nbformat": 4, diff --git a/tests/data/small_sky/Norder=0/Dir=0/Npix=11.parquet b/tests/data/small_sky/Norder=0/Dir=0/Npix=11.parquet index e0cb8d94..18ddf0c2 100644 Binary files a/tests/data/small_sky/Norder=0/Dir=0/Npix=11.parquet and b/tests/data/small_sky/Norder=0/Dir=0/Npix=11.parquet differ diff --git a/tests/data/small_sky/_common_metadata b/tests/data/small_sky/_common_metadata index 4cf7a744..31b324b6 100644 Binary files a/tests/data/small_sky/_common_metadata and b/tests/data/small_sky/_common_metadata differ diff --git a/tests/data/small_sky/_metadata b/tests/data/small_sky/_metadata index 26df207b..02a7f7b7 100644 Binary files a/tests/data/small_sky/_metadata and b/tests/data/small_sky/_metadata differ diff --git a/tests/data/small_sky/point_map.fits b/tests/data/small_sky/point_map.fits index 1971966f..66769c9c 100644 Binary files a/tests/data/small_sky/point_map.fits and b/tests/data/small_sky/point_map.fits differ diff --git a/tests/data/small_sky/provenance_info.json b/tests/data/small_sky/provenance_info.json index 2b7c8cf0..a700a07f 100644 --- a/tests/data/small_sky/provenance_info.json +++ b/tests/data/small_sky/provenance_info.json @@ -5,35 +5,34 @@ "epoch": "J2000", "ra_column": "ra", "dec_column": "dec", - "version": "0.2.1", - "generation_date": "2024.01.09", + "version": "0.3.4.dev17+g922a4b7.d20240529", + "generation_date": "2024.05.29", "tool_args": { "tool_name": "hipscat_import", - "version": "0.2.1", + "version": "0.3.3.dev10+gd573bcd", "runtime_args": { "catalog_name": "small_sky", - "output_path": "/home/delucchi/git/hipscat/tests/data/", + "output_path": ".", "output_artifact_name": "small_sky", - "tmp_dir": "", - "overwrite": true, + "tmp_dir": "/tmp/tmpgasth6x_", "dask_tmp": "", "dask_n_workers": 1, "dask_threads_per_worker": 1, - "catalog_path": "/home/delucchi/git/hipscat/tests/data/small_sky", - "tmp_path": "/home/delucchi/git/hipscat/tests/data/small_sky/intermediate", + "catalog_path": "./small_sky", + "tmp_path": "/tmp/tmpgasth6x_/small_sky/intermediate", "epoch": "J2000", "catalog_type": "object", - "input_path": "/home/delucchi/git/hipscat-import/tests/hipscat_import/data/small_sky", + "input_path": "../../../hipscat-import/tests/hipscat_import/data/small_sky", "input_paths": [ - "file:///home/delucchi/git/hipscat-import/tests/hipscat_import/data/small_sky/catalog.csv" + "file:///home/delucchi/git/fits/tests/data/../../../hipscat-import/tests/hipscat_import/data/small_sky/catalog.csv" ], - "input_format": "csv", "input_file_list": [], "ra_column": "ra", "dec_column": "dec", "use_hipscat_index": false, "sort_columns": null, "constant_healpix_order": -1, + "lowest_healpix_order": 0, "highest_healpix_order": 7, "pixel_threshold": 1000000, "mapping_healpix_order": 7, @@ -43,9 +42,10 @@ "chunksize": 500000, "header": "infer", "schema_file": null, - "separator": ",", "column_names": null, - "type_map": {} + "type_map": null, + "parquet_kwargs": null, + "kwargs": {} } } } diff --git a/tests/data/small_sky_order1/Norder=1/Dir=0/Npix=44.parquet b/tests/data/small_sky_order1/Norder=1/Dir=0/Npix=44.parquet index fb1e07a7..65ea6d91 100644 Binary files a/tests/data/small_sky_order1/Norder=1/Dir=0/Npix=44.parquet and b/tests/data/small_sky_order1/Norder=1/Dir=0/Npix=44.parquet differ diff --git a/tests/data/small_sky_order1/Norder=1/Dir=0/Npix=45.parquet b/tests/data/small_sky_order1/Norder=1/Dir=0/Npix=45.parquet index 0f691583..3621116a 100644 Binary files a/tests/data/small_sky_order1/Norder=1/Dir=0/Npix=45.parquet and b/tests/data/small_sky_order1/Norder=1/Dir=0/Npix=45.parquet differ diff --git a/tests/data/small_sky_order1/Norder=1/Dir=0/Npix=46.parquet b/tests/data/small_sky_order1/Norder=1/Dir=0/Npix=46.parquet index fcef8363..8c9a3ad4 100644 Binary files a/tests/data/small_sky_order1/Norder=1/Dir=0/Npix=46.parquet and b/tests/data/small_sky_order1/Norder=1/Dir=0/Npix=46.parquet differ diff --git a/tests/data/small_sky_order1/Norder=1/Dir=0/Npix=47.parquet b/tests/data/small_sky_order1/Norder=1/Dir=0/Npix=47.parquet index 527d15d0..e04d97c8 100644 Binary files a/tests/data/small_sky_order1/Norder=1/Dir=0/Npix=47.parquet and b/tests/data/small_sky_order1/Norder=1/Dir=0/Npix=47.parquet differ diff --git a/tests/data/small_sky_order1/_common_metadata b/tests/data/small_sky_order1/_common_metadata index 4cf7a744..31b324b6 100644 Binary files a/tests/data/small_sky_order1/_common_metadata and b/tests/data/small_sky_order1/_common_metadata differ diff --git a/tests/data/small_sky_order1/_metadata b/tests/data/small_sky_order1/_metadata index 3ec7ff01..810fded3 100644 Binary files a/tests/data/small_sky_order1/_metadata and b/tests/data/small_sky_order1/_metadata differ diff --git a/tests/data/small_sky_order1/point_map.fits b/tests/data/small_sky_order1/point_map.fits index e7287c9f..09493254 100644 Binary files a/tests/data/small_sky_order1/point_map.fits and b/tests/data/small_sky_order1/point_map.fits differ diff --git a/tests/data/small_sky_order1/provenance_info.json b/tests/data/small_sky_order1/provenance_info.json index 416efca8..c82e530a 100644 --- a/tests/data/small_sky_order1/provenance_info.json +++ b/tests/data/small_sky_order1/provenance_info.json @@ -5,35 +5,34 @@ "epoch": "J2000", "ra_column": "ra", "dec_column": "dec", - "version": "0.2.1", - "generation_date": "2024.01.09", + "version": "0.3.4.dev17+g922a4b7.d20240529", + "generation_date": "2024.05.29", "tool_args": { "tool_name": "hipscat_import", - "version": "0.2.1", + "version": "0.3.3.dev10+gd573bcd", "runtime_args": { "catalog_name": "small_sky_order1", - "output_path": "tests/data", + "output_path": ".", "output_artifact_name": "small_sky_order1", - "tmp_dir": "", - "overwrite": true, + "tmp_dir": "/tmp/tmpgasth6x_", "dask_tmp": "", "dask_n_workers": 1, "dask_threads_per_worker": 1, - "catalog_path": "tests/data/small_sky_order1", - "tmp_path": "tests/data/small_sky_order1/intermediate", + "catalog_path": "./small_sky_order1", + "tmp_path": "/tmp/tmpgasth6x_/small_sky_order1/intermediate", "epoch": "J2000", "catalog_type": "object", - "input_path": "/home/delucchi/git/hipscat-import/tests/hipscat_import/data/small_sky", + "input_path": "../../../hipscat-import/tests/hipscat_import/data/small_sky", "input_paths": [ - "file:///home/delucchi/git/hipscat-import/tests/hipscat_import/data/small_sky/catalog.csv" + "file:///home/delucchi/git/fits/tests/data/../../../hipscat-import/tests/hipscat_import/data/small_sky/catalog.csv" ], - "input_format": "csv", "input_file_list": [], "ra_column": "ra", "dec_column": "dec", "use_hipscat_index": false, "sort_columns": null, "constant_healpix_order": 1, + "lowest_healpix_order": 0, "highest_healpix_order": 7, "pixel_threshold": 1000000, "mapping_healpix_order": 1, @@ -43,9 +42,10 @@ "chunksize": 500000, "header": "infer", "schema_file": null, - "separator": ",", "column_names": null, - "type_map": {} + "type_map": null, + "parquet_kwargs": null, + "kwargs": {} } } } diff --git a/tests/data/small_sky_order1_id_index/_common_metadata b/tests/data/small_sky_order1_id_index/_common_metadata index b4e35f47..82e7b9bf 100644 Binary files a/tests/data/small_sky_order1_id_index/_common_metadata and b/tests/data/small_sky_order1_id_index/_common_metadata differ diff --git a/tests/data/small_sky_order1_id_index/_metadata b/tests/data/small_sky_order1_id_index/_metadata index 1a527df3..b6b1278f 100644 Binary files a/tests/data/small_sky_order1_id_index/_metadata and b/tests/data/small_sky_order1_id_index/_metadata differ diff --git a/tests/data/small_sky_order1_id_index/index/part.0.parquet b/tests/data/small_sky_order1_id_index/index/part.0.parquet index d82342d0..124f8c89 100644 Binary files a/tests/data/small_sky_order1_id_index/index/part.0.parquet and b/tests/data/small_sky_order1_id_index/index/part.0.parquet differ diff --git a/tests/data/small_sky_order1_id_index/provenance_info.json b/tests/data/small_sky_order1_id_index/provenance_info.json index 63ee3652..676fe968 100644 --- a/tests/data/small_sky_order1_id_index/provenance_info.json +++ b/tests/data/small_sky_order1_id_index/provenance_info.json @@ -2,29 +2,28 @@ "catalog_name": "small_sky_order1_id_index", "catalog_type": "index", "total_rows": 131, - "primary_catalog": "/home/delucchi/git/hipscat/tests/data/small_sky_order1", + "primary_catalog": "small_sky", "indexing_column": "id", "extra_columns": [], - "version": "0.2.1", - "generation_date": "2024.01.09", + "version": "0.3.4.dev17+g922a4b7.d20240529", + "generation_date": "2024.05.29", "tool_args": { "tool_name": "hipscat_import", - "version": "0.2.1", + "version": "0.3.3.dev10+gd573bcd", "runtime_args": { "catalog_name": "small_sky_order1_id_index", - "output_path": "/home/delucchi/git/hipscat/tests/data/", + "output_path": ".", "output_artifact_name": "small_sky_order1_id_index", - "tmp_dir": "", - "overwrite": true, + "tmp_dir": "/tmp/tmpgasth6x_", "dask_tmp": "", "dask_n_workers": 1, "dask_threads_per_worker": 1, - "catalog_path": "/home/delucchi/git/hipscat/tests/data/small_sky_order1_id_index", - "tmp_path": "/home/delucchi/git/hipscat/tests/data/small_sky_order1_id_index/intermediate", - "input_catalog_path": "/home/delucchi/git/hipscat/tests/data/small_sky_order1", + "catalog_path": "./small_sky_order1_id_index", + "tmp_path": "/tmp/tmpgasth6x_/small_sky_order1_id_index/intermediate", + "input_catalog_path": "small_sky", "indexing_column": "id", "extra_columns": [], - "include_hipscat_index": "False", + "include_hipscat_index": false, "include_order_pixel": true } } diff --git a/tests/data/small_sky_order1_margin/Norder=0/Dir=0/Npix=4.parquet b/tests/data/small_sky_order1_margin/Norder=0/Dir=0/Npix=4.parquet index c5d85839..96682560 100644 Binary files a/tests/data/small_sky_order1_margin/Norder=0/Dir=0/Npix=4.parquet and b/tests/data/small_sky_order1_margin/Norder=0/Dir=0/Npix=4.parquet differ diff --git a/tests/data/small_sky_order1_margin/Norder=1/Dir=0/Npix=44.parquet b/tests/data/small_sky_order1_margin/Norder=1/Dir=0/Npix=44.parquet index e08db717..2ac09418 100644 Binary files a/tests/data/small_sky_order1_margin/Norder=1/Dir=0/Npix=44.parquet and b/tests/data/small_sky_order1_margin/Norder=1/Dir=0/Npix=44.parquet differ diff --git a/tests/data/small_sky_order1_margin/Norder=1/Dir=0/Npix=45.parquet b/tests/data/small_sky_order1_margin/Norder=1/Dir=0/Npix=45.parquet index debb4560..6fb1e6bf 100644 Binary files a/tests/data/small_sky_order1_margin/Norder=1/Dir=0/Npix=45.parquet and b/tests/data/small_sky_order1_margin/Norder=1/Dir=0/Npix=45.parquet differ diff --git a/tests/data/small_sky_order1_margin/Norder=1/Dir=0/Npix=46.parquet b/tests/data/small_sky_order1_margin/Norder=1/Dir=0/Npix=46.parquet index aec24647..c17c7089 100644 Binary files a/tests/data/small_sky_order1_margin/Norder=1/Dir=0/Npix=46.parquet and b/tests/data/small_sky_order1_margin/Norder=1/Dir=0/Npix=46.parquet differ diff --git a/tests/data/small_sky_order1_margin/Norder=1/Dir=0/Npix=47.parquet b/tests/data/small_sky_order1_margin/Norder=1/Dir=0/Npix=47.parquet index 74f6b7d9..9cad481d 100644 Binary files a/tests/data/small_sky_order1_margin/Norder=1/Dir=0/Npix=47.parquet and b/tests/data/small_sky_order1_margin/Norder=1/Dir=0/Npix=47.parquet differ diff --git a/tests/data/small_sky_order1_margin/_common_metadata b/tests/data/small_sky_order1_margin/_common_metadata index c5c1c64b..d27a9c83 100644 Binary files a/tests/data/small_sky_order1_margin/_common_metadata and b/tests/data/small_sky_order1_margin/_common_metadata differ diff --git a/tests/data/small_sky_order1_margin/_metadata b/tests/data/small_sky_order1_margin/_metadata index defa58c6..2b547411 100644 Binary files a/tests/data/small_sky_order1_margin/_metadata and b/tests/data/small_sky_order1_margin/_metadata differ diff --git a/tests/data/small_sky_order1_margin/provenance_info.json b/tests/data/small_sky_order1_margin/provenance_info.json index cb4a3dac..26f9cd0a 100644 --- a/tests/data/small_sky_order1_margin/provenance_info.json +++ b/tests/data/small_sky_order1_margin/provenance_info.json @@ -2,25 +2,24 @@ "catalog_name": "small_sky_order1_margin", "catalog_type": "margin", "total_rows": 28, - "primary_catalog": "data/small_sky_order1", + "primary_catalog": "small_sky_order1", "margin_threshold": 7200, - "version": "0.2.3", - "generation_date": "2024.01.30", + "version": "0.3.4.dev17+g922a4b7.d20240529", + "generation_date": "2024.05.29", "tool_args": { "tool_name": "hipscat_import", - "version": "0.2.2", + "version": "0.3.3.dev10+gd573bcd", "runtime_args": { "catalog_name": "small_sky_order1_margin", - "output_path": "data/", + "output_path": ".", "output_artifact_name": "small_sky_order1_margin", - "tmp_dir": "", - "overwrite": false, + "tmp_dir": "/tmp/tmp8j6vsyq2", "dask_tmp": "", "dask_n_workers": 1, "dask_threads_per_worker": 1, - "catalog_path": "data/small_sky_order1_margin", - "tmp_path": "data/small_sky_order1_margin/intermediate", - "input_catalog_path": "data/small_sky_order1", + "catalog_path": "./small_sky_order1_margin", + "tmp_path": "/tmp/tmp8j6vsyq2/small_sky_order1_margin/intermediate", + "input_catalog_path": "small_sky_order1", "margin_threshold": 7200, "margin_order": 2 } diff --git a/tests/data/small_sky_source/Norder=0/Dir=0/Npix=4.parquet b/tests/data/small_sky_source/Norder=0/Dir=0/Npix=4.parquet index 30b43a52..ef0f15c2 100644 Binary files a/tests/data/small_sky_source/Norder=0/Dir=0/Npix=4.parquet and b/tests/data/small_sky_source/Norder=0/Dir=0/Npix=4.parquet differ diff --git a/tests/data/small_sky_source/Norder=1/Dir=0/Npix=47.parquet b/tests/data/small_sky_source/Norder=1/Dir=0/Npix=47.parquet index bb20f73d..7fccea01 100644 Binary files a/tests/data/small_sky_source/Norder=1/Dir=0/Npix=47.parquet and b/tests/data/small_sky_source/Norder=1/Dir=0/Npix=47.parquet differ diff --git a/tests/data/small_sky_source/Norder=2/Dir=0/Npix=176.parquet b/tests/data/small_sky_source/Norder=2/Dir=0/Npix=176.parquet index 5186c58f..5dd191ea 100644 Binary files a/tests/data/small_sky_source/Norder=2/Dir=0/Npix=176.parquet and b/tests/data/small_sky_source/Norder=2/Dir=0/Npix=176.parquet differ diff --git a/tests/data/small_sky_source/Norder=2/Dir=0/Npix=177.parquet b/tests/data/small_sky_source/Norder=2/Dir=0/Npix=177.parquet index 9ce5e005..b8c7073c 100644 Binary files a/tests/data/small_sky_source/Norder=2/Dir=0/Npix=177.parquet and b/tests/data/small_sky_source/Norder=2/Dir=0/Npix=177.parquet differ diff --git a/tests/data/small_sky_source/Norder=2/Dir=0/Npix=178.parquet b/tests/data/small_sky_source/Norder=2/Dir=0/Npix=178.parquet index 133f899e..15f5bdac 100644 Binary files a/tests/data/small_sky_source/Norder=2/Dir=0/Npix=178.parquet and b/tests/data/small_sky_source/Norder=2/Dir=0/Npix=178.parquet differ diff --git a/tests/data/small_sky_source/Norder=2/Dir=0/Npix=179.parquet b/tests/data/small_sky_source/Norder=2/Dir=0/Npix=179.parquet index 8454ed03..6a730dbe 100644 Binary files a/tests/data/small_sky_source/Norder=2/Dir=0/Npix=179.parquet and b/tests/data/small_sky_source/Norder=2/Dir=0/Npix=179.parquet differ diff --git a/tests/data/small_sky_source/Norder=2/Dir=0/Npix=180.parquet b/tests/data/small_sky_source/Norder=2/Dir=0/Npix=180.parquet index 83b2806b..ae4c4ae4 100644 Binary files a/tests/data/small_sky_source/Norder=2/Dir=0/Npix=180.parquet and b/tests/data/small_sky_source/Norder=2/Dir=0/Npix=180.parquet differ diff --git a/tests/data/small_sky_source/Norder=2/Dir=0/Npix=181.parquet b/tests/data/small_sky_source/Norder=2/Dir=0/Npix=181.parquet index 3af86fb2..e331a85e 100644 Binary files a/tests/data/small_sky_source/Norder=2/Dir=0/Npix=181.parquet and b/tests/data/small_sky_source/Norder=2/Dir=0/Npix=181.parquet differ diff --git a/tests/data/small_sky_source/Norder=2/Dir=0/Npix=182.parquet b/tests/data/small_sky_source/Norder=2/Dir=0/Npix=182.parquet index 66606f31..0c30651a 100644 Binary files a/tests/data/small_sky_source/Norder=2/Dir=0/Npix=182.parquet and b/tests/data/small_sky_source/Norder=2/Dir=0/Npix=182.parquet differ diff --git a/tests/data/small_sky_source/Norder=2/Dir=0/Npix=183.parquet b/tests/data/small_sky_source/Norder=2/Dir=0/Npix=183.parquet index 33b3d8e8..ba150b64 100644 Binary files a/tests/data/small_sky_source/Norder=2/Dir=0/Npix=183.parquet and b/tests/data/small_sky_source/Norder=2/Dir=0/Npix=183.parquet differ diff --git a/tests/data/small_sky_source/Norder=2/Dir=0/Npix=184.parquet b/tests/data/small_sky_source/Norder=2/Dir=0/Npix=184.parquet index 0382d917..fb798bce 100644 Binary files a/tests/data/small_sky_source/Norder=2/Dir=0/Npix=184.parquet and b/tests/data/small_sky_source/Norder=2/Dir=0/Npix=184.parquet differ diff --git a/tests/data/small_sky_source/Norder=2/Dir=0/Npix=185.parquet b/tests/data/small_sky_source/Norder=2/Dir=0/Npix=185.parquet index c3e7edce..5d338aa9 100644 Binary files a/tests/data/small_sky_source/Norder=2/Dir=0/Npix=185.parquet and b/tests/data/small_sky_source/Norder=2/Dir=0/Npix=185.parquet differ diff --git a/tests/data/small_sky_source/Norder=2/Dir=0/Npix=186.parquet b/tests/data/small_sky_source/Norder=2/Dir=0/Npix=186.parquet index a53f3490..9c0cd57e 100644 Binary files a/tests/data/small_sky_source/Norder=2/Dir=0/Npix=186.parquet and b/tests/data/small_sky_source/Norder=2/Dir=0/Npix=186.parquet differ diff --git a/tests/data/small_sky_source/Norder=2/Dir=0/Npix=187.parquet b/tests/data/small_sky_source/Norder=2/Dir=0/Npix=187.parquet index 9eacc4ec..53a7fb12 100644 Binary files a/tests/data/small_sky_source/Norder=2/Dir=0/Npix=187.parquet and b/tests/data/small_sky_source/Norder=2/Dir=0/Npix=187.parquet differ diff --git a/tests/data/small_sky_source/_common_metadata b/tests/data/small_sky_source/_common_metadata index 3f78df69..28747fc8 100644 Binary files a/tests/data/small_sky_source/_common_metadata and b/tests/data/small_sky_source/_common_metadata differ diff --git a/tests/data/small_sky_source/_metadata b/tests/data/small_sky_source/_metadata index 9cc2015e..8d2e4ef0 100644 Binary files a/tests/data/small_sky_source/_metadata and b/tests/data/small_sky_source/_metadata differ diff --git a/tests/data/small_sky_source/point_map.fits b/tests/data/small_sky_source/point_map.fits index 53bf901d..44dc0171 100644 Binary files a/tests/data/small_sky_source/point_map.fits and b/tests/data/small_sky_source/point_map.fits differ diff --git a/tests/data/small_sky_source/provenance_info.json b/tests/data/small_sky_source/provenance_info.json index 9cf4fb8b..aa93b408 100644 --- a/tests/data/small_sky_source/provenance_info.json +++ b/tests/data/small_sky_source/provenance_info.json @@ -5,27 +5,26 @@ "epoch": "J2000", "ra_column": "source_ra", "dec_column": "source_dec", - "version": "0.2.7.dev5+gde86705", - "generation_date": "2024.03.01", + "version": "0.3.4.dev17+g922a4b7.d20240529", + "generation_date": "2024.05.29", "tool_args": { "tool_name": "hipscat_import", - "version": "0.2.5.dev5+g0733afb", + "version": "0.3.3.dev10+gd573bcd", "runtime_args": { "catalog_name": "small_sky_source", "output_path": ".", "output_artifact_name": "small_sky_source", - "tmp_dir": "/tmp/user/11115/tmp2vluzdw5", - "overwrite": true, + "tmp_dir": "/tmp/tmpgasth6x_", "dask_tmp": "", "dask_n_workers": 1, "dask_threads_per_worker": 1, "catalog_path": "./small_sky_source", - "tmp_path": "/tmp/user/11115/tmp2vluzdw5/small_sky_source/intermediate", + "tmp_path": "/tmp/tmpgasth6x_/small_sky_source/intermediate", "epoch": "J2000", "catalog_type": "source", "input_path": "../../../hipscat-import/tests/hipscat_import/data/small_sky_source", "input_paths": [ - "file:///home/delucchi/git/hipscat/tests/data/../../../hipscat-import/tests/hipscat_import/data/small_sky_source/small_sky_source.csv" + "file:///home/delucchi/git/fits/tests/data/../../../hipscat-import/tests/hipscat_import/data/small_sky_source/small_sky_source.csv" ], "input_file_list": [], "ra_column": "source_ra", @@ -33,6 +32,7 @@ "use_hipscat_index": false, "sort_columns": null, "constant_healpix_order": -1, + "lowest_healpix_order": 0, "highest_healpix_order": 7, "pixel_threshold": 3000, "mapping_healpix_order": 7, @@ -42,9 +42,10 @@ "chunksize": 500000, "header": "infer", "schema_file": null, - "separator": ",", "column_names": null, - "type_map": {} + "type_map": null, + "parquet_kwargs": null, + "kwargs": {} } } } diff --git a/tests/data/small_sky_source_object_index/_common_metadata b/tests/data/small_sky_source_object_index/_common_metadata index 455e38b3..796247f5 100644 Binary files a/tests/data/small_sky_source_object_index/_common_metadata and b/tests/data/small_sky_source_object_index/_common_metadata differ diff --git a/tests/data/small_sky_source_object_index/_metadata b/tests/data/small_sky_source_object_index/_metadata index a65a2b7e..52bfed08 100644 Binary files a/tests/data/small_sky_source_object_index/_metadata and b/tests/data/small_sky_source_object_index/_metadata differ diff --git a/tests/data/small_sky_source_object_index/index/part.0.parquet b/tests/data/small_sky_source_object_index/index/part.0.parquet index 2c339aef..da8af08c 100644 Binary files a/tests/data/small_sky_source_object_index/index/part.0.parquet and b/tests/data/small_sky_source_object_index/index/part.0.parquet differ diff --git a/tests/data/small_sky_source_object_index/provenance_info.json b/tests/data/small_sky_source_object_index/provenance_info.json index d42b6028..6773b0d2 100644 --- a/tests/data/small_sky_source_object_index/provenance_info.json +++ b/tests/data/small_sky_source_object_index/provenance_info.json @@ -5,22 +5,21 @@ "primary_catalog": "small_sky_source", "indexing_column": "object_id", "extra_columns": [], - "version": "0.2.7.dev5+gde86705", - "generation_date": "2024.03.01", + "version": "0.3.4.dev17+g922a4b7.d20240529", + "generation_date": "2024.05.29", "tool_args": { "tool_name": "hipscat_import", - "version": "0.2.5.dev5+g0733afb", + "version": "0.3.3.dev10+gd573bcd", "runtime_args": { "catalog_name": "small_sky_source_object_index", "output_path": ".", "output_artifact_name": "small_sky_source_object_index", - "tmp_dir": "/tmp/user/11115/tmp2vluzdw5", - "overwrite": true, + "tmp_dir": "/tmp/tmpgasth6x_", "dask_tmp": "", "dask_n_workers": 1, "dask_threads_per_worker": 1, "catalog_path": "./small_sky_source_object_index", - "tmp_path": "/tmp/user/11115/tmp2vluzdw5/small_sky_source_object_index/intermediate", + "tmp_path": "/tmp/tmpgasth6x_/small_sky_source_object_index/intermediate", "input_catalog_path": "small_sky_source", "indexing_column": "object_id", "extra_columns": [], diff --git a/tests/data/small_sky_to_small_sky_order1/_common_metadata b/tests/data/small_sky_to_small_sky_order1/_common_metadata index 45e3dbab..eb61105d 100644 Binary files a/tests/data/small_sky_to_small_sky_order1/_common_metadata and b/tests/data/small_sky_to_small_sky_order1/_common_metadata differ diff --git a/tests/data/small_sky_to_small_sky_order1/_metadata b/tests/data/small_sky_to_small_sky_order1/_metadata index e05295c3..ecc423c2 100644 Binary files a/tests/data/small_sky_to_small_sky_order1/_metadata and b/tests/data/small_sky_to_small_sky_order1/_metadata differ diff --git a/tests/hipscat/catalog/margin_cache/test_margin_catalog.py b/tests/hipscat/catalog/margin_cache/test_margin_catalog.py index 4cd5ff66..ffe7e037 100644 --- a/tests/hipscat/catalog/margin_cache/test_margin_catalog.py +++ b/tests/hipscat/catalog/margin_cache/test_margin_catalog.py @@ -5,6 +5,7 @@ from hipscat.catalog import CatalogType, MarginCatalog, PartitionInfo from hipscat.loaders import read_from_hipscat +from hipscat.pixel_math import HealpixPixel def test_init_catalog(margin_catalog_info, margin_catalog_pixels): @@ -76,3 +77,23 @@ def test_empty_directory(tmp_path, margin_cache_catalog_info_data, margin_catalo with pytest.warns(UserWarning, match="slow"): catalog = read_from_hipscat(catalog_path) assert catalog.catalog_name == margin_cache_catalog_info_data["catalog_name"] + + +def test_margin_filter(margin_catalog_info, margin_catalog_pixels): + catalog = MarginCatalog(margin_catalog_info, margin_catalog_pixels) + pixels = [HealpixPixel(1, 44)] + filtered_catalog = catalog.filter_from_pixel_list(pixels) + assert filtered_catalog.get_healpix_pixels() == [ + HealpixPixel(1, 44), + HealpixPixel(1, 45), + HealpixPixel(1, 46), + HealpixPixel(1, 47), + ] + + +def test_margin_filter_invalid_size(margin_catalog_info, margin_catalog_pixels): + margin_catalog_info.margin_threshold = 10000000 + catalog = MarginCatalog(margin_catalog_info, margin_catalog_pixels) + pixels = [HealpixPixel(1, 44)] + with pytest.raises(ValueError, match="greater than the size of a pixel"): + catalog.filter_from_pixel_list(pixels) diff --git a/tests/hipscat/catalog/test_catalog.py b/tests/hipscat/catalog/test_catalog.py index ffbebfa1..72b77415 100644 --- a/tests/hipscat/catalog/test_catalog.py +++ b/tests/hipscat/catalog/test_catalog.py @@ -2,16 +2,18 @@ import os +import astropy.units as u import healpy as hp import numpy as np import pytest +from mocpy import MOC from hipscat.catalog import Catalog, CatalogType, PartitionInfo from hipscat.io import paths from hipscat.io.file_io import read_fits_image from hipscat.loaders import read_from_hipscat from hipscat.pixel_math import HealpixPixel -from hipscat.pixel_math.box_filter import _generate_ra_strip_pixel_tree +from hipscat.pixel_math.box_filter import _generate_ra_strip_moc, generate_box_moc from hipscat.pixel_math.validators import ValidatorsErrors from hipscat.pixel_tree.pixel_tree import PixelTree @@ -116,8 +118,31 @@ def test_load_catalog_small_sky_source(small_sky_source_dir): assert len(cat.get_healpix_pixels()) == 14 +def test_max_coverage_order(small_sky_order1_catalog): + assert small_sky_order1_catalog.get_max_coverage_order() >= small_sky_order1_catalog.moc.max_order + assert ( + small_sky_order1_catalog.get_max_coverage_order() + >= small_sky_order1_catalog.pixel_tree.get_max_depth() + ) + high_moc_order = 8 + test_moc = MOC.from_depth29_ranges( + max_depth=high_moc_order, ranges=small_sky_order1_catalog.moc.to_depth29_ranges + ) + small_sky_order1_catalog.moc = test_moc + assert small_sky_order1_catalog.get_max_coverage_order() == high_moc_order + small_sky_order1_catalog.moc = None + assert ( + small_sky_order1_catalog.get_max_coverage_order() + == small_sky_order1_catalog.pixel_tree.get_max_depth() + ) + + def test_cone_filter(small_sky_order1_catalog): - filtered_catalog = small_sky_order1_catalog.filter_by_cone(315, -66.443, 0.1) + ra = 315 + dec = -66.443 + radius = 0.1 + + filtered_catalog = small_sky_order1_catalog.filter_by_cone(ra, dec, radius) filtered_pixels = filtered_catalog.get_healpix_pixels() assert len(filtered_pixels) == 1 @@ -126,6 +151,15 @@ def test_cone_filter(small_sky_order1_catalog): assert (1, 44) in filtered_catalog.pixel_tree assert filtered_catalog.catalog_info.total_rows is None + assert filtered_catalog.moc is not None + cone_moc = MOC.from_cone( + lon=ra * u.deg, + lat=dec * u.deg, + radius=radius * u.arcsec, + max_depth=small_sky_order1_catalog.get_max_coverage_order(), + ) + assert filtered_catalog.moc == cone_moc.intersection(small_sky_order1_catalog.moc) + def test_cone_filter_big(small_sky_order1_catalog): filtered_catalog = small_sky_order1_catalog.filter_by_cone(315, -66.443, 30 * 3600) @@ -171,6 +205,14 @@ def test_polygonal_filter(small_sky_order1_catalog): assert filtered_pixels == [HealpixPixel(1, 46)] assert (1, 46) in filtered_catalog.pixel_tree assert filtered_catalog.catalog_info.total_rows is None + assert filtered_catalog.moc is not None + ra, dec = np.array(polygon_vertices).T + polygon_moc = MOC.from_polygon( + lon=ra * u.deg, + lat=dec * u.deg, + max_depth=small_sky_order1_catalog.get_max_coverage_order(), + ) + assert filtered_catalog.moc == polygon_moc.intersection(small_sky_order1_catalog.moc) def test_polygonal_filter_with_cartesian_coordinates(small_sky_order1_catalog): @@ -241,8 +283,9 @@ def test_polygonal_filter_invalid_polygon(small_sky_order1_catalog): def test_box_filter_ra(small_sky_order1_catalog): + ra = (280, 290) # The catalog pixels are distributed around the [270,0] degree range. - filtered_catalog = small_sky_order1_catalog.filter_by_box(ra=(280, 290)) + filtered_catalog = small_sky_order1_catalog.filter_by_box(ra=ra) filtered_pixels = filtered_catalog.get_healpix_pixels() @@ -254,6 +297,10 @@ def test_box_filter_ra(small_sky_order1_catalog): assert len(filtered_catalog.pixel_tree.pixels[1]) == 2 assert filtered_catalog.catalog_info.total_rows is None + assert filtered_catalog.moc is not None + box_moc = generate_box_moc(ra=ra, dec=None, order=small_sky_order1_catalog.get_max_coverage_order()) + assert filtered_catalog.moc == box_moc.intersection(small_sky_order1_catalog.moc) + def test_box_filter_wrapped_ra(small_sky_order1_catalog): # The catalog pixels are distributed around the [270,0] degree range. @@ -308,25 +355,25 @@ def assert_is_subset_of(catalog, catalog_complement): def test_box_filter_ra_pixel_tree_generation(): """This method tests the pixel tree generation for the ra filter""" # The catalog pixels are distributed around the [270,0] degree range. - pixel_tree = _generate_ra_strip_pixel_tree(ra_range=(0, 180), order=1) - pixel_tree_complement = _generate_ra_strip_pixel_tree(ra_range=(180, 0), order=1) - assert len(pixel_tree) == len(pixel_tree_complement) + moc = _generate_ra_strip_moc(ra_range=(0, 180), order=1) + moc_complement = _generate_ra_strip_moc(ra_range=(180, 0), order=1) + assert len(moc.flatten()) == len(moc_complement.flatten()) - pixel_tree = _generate_ra_strip_pixel_tree(ra_range=(10, 50), order=1) - pixel_tree_complement = _generate_ra_strip_pixel_tree(ra_range=(50, 10), order=1) - assert len(pixel_tree) < len(pixel_tree_complement) + moc = _generate_ra_strip_moc(ra_range=(10, 50), order=1) + moc_complement = _generate_ra_strip_moc(ra_range=(50, 10), order=1) + assert len(moc.flatten()) < len(moc_complement.flatten()) - pixel_tree = _generate_ra_strip_pixel_tree(ra_range=(10, 220), order=1) - pixel_tree_complement = _generate_ra_strip_pixel_tree(ra_range=(220, 10), order=1) - assert len(pixel_tree_complement) < len(pixel_tree) + moc = _generate_ra_strip_moc(ra_range=(10, 220), order=1) + moc_complement = _generate_ra_strip_moc(ra_range=(220, 10), order=1) + assert len(moc_complement.flatten()) < len(moc.flatten()) - pixel_tree = _generate_ra_strip_pixel_tree(ra_range=(200, 350), order=1) - pixel_tree_complement = _generate_ra_strip_pixel_tree(ra_range=(350, 200), order=1) - assert len(pixel_tree) < len(pixel_tree_complement) + moc = _generate_ra_strip_moc(ra_range=(200, 350), order=1) + moc_complement = _generate_ra_strip_moc(ra_range=(350, 200), order=1) + assert len(moc.flatten()) < len(moc_complement.flatten()) - pixel_tree = _generate_ra_strip_pixel_tree(ra_range=(200, 50), order=1) - pixel_tree_complement = _generate_ra_strip_pixel_tree(ra_range=(50, 200), order=1) - assert len(pixel_tree_complement) < len(pixel_tree) + moc = _generate_ra_strip_moc(ra_range=(200, 50), order=1) + moc_complement = _generate_ra_strip_moc(ra_range=(50, 200), order=1) + assert len(moc_complement.flatten()) < len(moc.flatten()) def test_box_filter_dec(small_sky_order1_catalog): diff --git a/tests/hipscat/io/test_write_metadata.py b/tests/hipscat/io/test_write_metadata.py index 37f2a920..3af12a00 100644 --- a/tests/hipscat/io/test_write_metadata.py +++ b/tests/hipscat/io/test_write_metadata.py @@ -4,7 +4,10 @@ import shutil from pathlib import Path +import healpy as hp +import numpy as np import numpy.testing as npt +import pytest import hipscat.io.write_metadata as io import hipscat.pixel_math as hist @@ -27,7 +30,14 @@ def test_write_json_file(assert_text_file_matches, tmp_path): " 3,", " 5", " ]", - r' "integer_type": ""', + r' "integer_type": "",', + ' "np_int": 5000000,', + ' "np_float": 1.618,', + ' "pixel": "Order: 5, Pixel: 9000",', + r' "pixels": \[', + ' "Order: 5, Pixel: 9000",', + ' "Order: 5, Pixel: 9001"', + " ]", "}", ] @@ -37,6 +47,10 @@ def test_write_json_file(assert_text_file_matches, tmp_path): dictionary["first_number"] = 1 dictionary["first_five_fib"] = [1, 1, 2, 3, 5] dictionary["integer_type"] = int + dictionary["np_int"] = np.uint64(5_000_000) + dictionary["np_float"] = np.float64(1.618) + dictionary["pixel"] = HealpixPixel(5, 9_000) + dictionary["pixels"] = np.array([HealpixPixel(5, 9_000), HealpixPixel(5, 9_001)]) json_filename = os.path.join(tmp_path, "dictionary.json") io.write_json_file(dictionary, json_filename) @@ -204,3 +218,26 @@ def test_read_write_fits_point_map(tmp_path): output = file_io.read_fits_image(output_file) npt.assert_array_equal(output, initial_histogram) + + # Check the metadata of the fits file: + map_fits_image = hp.read_map(output_file, nest=True, h=True) + + header_dict = dict(map_fits_image[1]) + assert header_dict["ORDERING"] == "NESTED" + assert header_dict["PIXTYPE"] == "HEALPIX" + assert header_dict["NSIDE"] == 2 + + npt.assert_array_equal(initial_histogram, map_fits_image[0]) + + +def test_read_ring_fits_point_map(tmp_path): + """Check that we write and can read a FITS file for spatial distribution.""" + output_file = os.path.join(tmp_path, "point_map.fits") + initial_histogram = hist.empty_histogram(1) + filled_pixels = [51, 29, 51, 0] + initial_histogram[44:] = filled_pixels[:] + hp.write_map(output_file, initial_histogram, dtype=np.int64) + + with pytest.warns(UserWarning, match="/hipscat/issues/271"): + output = file_io.read_fits_image(output_file) + npt.assert_array_equal(output, initial_histogram)