Use nest ordering for point_map.fits file. (#273)

* Use nest ordering for point_map.fits file.

* Handle older format, and update all test data files.

* Add test for new warning.

src/hipscat/io/file_io/file_io.py

@@ -2,6 +2,7 @@
 
 import json
 import tempfile
+import warnings
 from typing import Any, Dict, Tuple, Union
 
 import healpy as hp
@@ -278,8 +279,17 @@ def read_fits_image(map_file_pointer: FilePointer, storage_options: Union[Dict[A
         with file_system.open(map_file_pointer, "rb") as _map_file:
             map_data = _map_file.read()
             _tmp_file.write(map_data)
-            map_fits_image = hp.read_map(_tmp_file.name)
-    return map_fits_image
+            map_fits_image = hp.read_map(_tmp_file.name, nest=True, h=True)
+            header_dict = dict(map_fits_image[1])
+            if header_dict["ORDERING"] != "NESTED":
+                warnings.warn(
+                    "point_map.fits file written in RING ordering, due to "
+                    "https://github.com/astronomy-commons/hipscat/issues/271. "
+                    "Converting to NESTED."
+                )
+                map_fits_image = hp.read_map(_tmp_file.name)
+                return map_fits_image
+            return map_fits_image[0]
 
 
 def write_fits_image(
@@ -296,7 +306,7 @@ def write_fits_image(
     file_system, map_file_pointer = get_fs(file_pointer=map_file_pointer, storage_options=storage_options)
     with tempfile.NamedTemporaryFile() as _tmp_file:
         with file_system.open(map_file_pointer, "wb") as _map_file:
-            hp.write_map(_tmp_file.name, histogram, overwrite=True, dtype=np.int64)
+            hp.write_map(_tmp_file.name, histogram, overwrite=True, dtype=np.int64, nest=True)
             _map_file.write(_tmp_file.read())
 
 

tests/data/generate_data.ipynb

@@ -31,11 +31,13 @@
     "from hipscat_import.margin_cache.margin_cache_arguments import MarginCacheArguments\n",
     "import tempfile\n",
     "from pathlib import Path\n",
+    "from dask.distributed import Client\n",
     "\n",
     "tmp_path = tempfile.TemporaryDirectory()\n",
     "tmp_dir = tmp_path.name\n",
     "\n",
-    "hipscat_import_dir = \"../../../hipscat-import/tests/hipscat_import/data/\""
+    "hipscat_import_dir = \"../../../hipscat-import/tests/hipscat_import/data/\"\n",
+    "client = Client(n_workers=1, threads_per_worker=1, local_directory=tmp_dir)"
    ]
   },
   {
@@ -63,10 +65,9 @@
     "    output_path=\".\",\n",
     "    file_reader=\"csv\",\n",
     "    output_artifact_name=\"small_sky\",\n",
-    "    overwrite=True,\n",
     "    tmp_dir=tmp_dir,\n",
     ")\n",
-    "runner.pipeline(args)"
+    "runner.pipeline_with_client(args, client)"
    ]
   },
   {
@@ -100,10 +101,9 @@
     "    file_reader=\"csv\",\n",
     "    output_artifact_name=\"small_sky_order1\",\n",
     "    constant_healpix_order=1,\n",
-    "    overwrite=True,\n",
     "    tmp_dir=tmp_dir,\n",
     ")\n",
-    "runner.pipeline(args)"
+    "runner.pipeline_with_client(args, client)"
    ]
   },
   {
@@ -130,10 +130,9 @@
     "    output_artifact_name=\"small_sky_order1_id_index\",\n",
     "    include_hipscat_index=False,\n",
     "    compute_partition_size=200_000,\n",
-    "    overwrite=True,\n",
     "    tmp_dir=tmp_dir,\n",
     ")\n",
-    "runner.pipeline(args)"
+    "runner.pipeline_with_client(args, client)"
    ]
   },
   {
@@ -167,10 +166,9 @@
     "    input_catalog_path=\"small_sky_order1\",\n",
     "    output_path=\".\",\n",
     "    output_artifact_name=\"small_sky_order1_margin\",\n",
-    "    overwrite=True,\n",
     "    tmp_dir=tmp_dir,\n",
     ")\n",
-    "runner.pipeline(margin_args)"
+    "runner.pipeline_with_client(args, client)"
    ]
   },
   {
@@ -241,10 +239,9 @@
     "    catalog_type=\"source\",\n",
     "    pixel_threshold=3000,\n",
     "    output_artifact_name=\"small_sky_source\",\n",
-    "    overwrite=True,\n",
     "    tmp_dir=tmp_dir,\n",
     ")\n",
-    "runner.pipeline(args)"
+    "runner.pipeline_with_client(args, client)"
    ]
   },
   {
@@ -282,10 +279,9 @@
     "    output_artifact_name=\"small_sky_source_object_index\",\n",
     "    include_hipscat_index=False,\n",
     "    compute_partition_size=200_000,\n",
-    "    overwrite=True,\n",
     "    tmp_dir=tmp_dir,\n",
     ")\n",
-    "runner.pipeline(args)"
+    "runner.pipeline_with_client(args, client)"
    ]
   },
   {
@@ -294,6 +290,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "client.close()\n",
     "tmp_path.cleanup()"
    ]
   }
@@ -314,7 +311,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.13"
+   "version": "3.10.14"
   }
  },
  "nbformat": 4,

tests/data/small_sky/provenance_info.json

@@ -5,35 +5,34 @@
     "epoch": "J2000",
     "ra_column": "ra",
     "dec_column": "dec",
-    "version": "0.2.1",
-    "generation_date": "2024.01.09",
+    "version": "0.3.4.dev17+g922a4b7.d20240529",
+    "generation_date": "2024.05.29",
     "tool_args": {
         "tool_name": "hipscat_import",
-        "version": "0.2.1",
+        "version": "0.3.3.dev10+gd573bcd",
         "runtime_args": {
             "catalog_name": "small_sky",
-            "output_path": "/home/delucchi/git/hipscat/tests/data/",
+            "output_path": ".",
             "output_artifact_name": "small_sky",
-            "tmp_dir": "",
-            "overwrite": true,
+            "tmp_dir": "/tmp/tmpgasth6x_",
             "dask_tmp": "",
             "dask_n_workers": 1,
             "dask_threads_per_worker": 1,
-            "catalog_path": "/home/delucchi/git/hipscat/tests/data/small_sky",
-            "tmp_path": "/home/delucchi/git/hipscat/tests/data/small_sky/intermediate",
+            "catalog_path": "./small_sky",
+            "tmp_path": "/tmp/tmpgasth6x_/small_sky/intermediate",
             "epoch": "J2000",
             "catalog_type": "object",
-            "input_path": "/home/delucchi/git/hipscat-import/tests/hipscat_import/data/small_sky",
+            "input_path": "../../../hipscat-import/tests/hipscat_import/data/small_sky",
             "input_paths": [
-                "file:///home/delucchi/git/hipscat-import/tests/hipscat_import/data/small_sky/catalog.csv"
+                "file:///home/delucchi/git/fits/tests/data/../../../hipscat-import/tests/hipscat_import/data/small_sky/catalog.csv"
             ],
-            "input_format": "csv",
             "input_file_list": [],
             "ra_column": "ra",
             "dec_column": "dec",
             "use_hipscat_index": false,
             "sort_columns": null,
             "constant_healpix_order": -1,
+            "lowest_healpix_order": 0,
             "highest_healpix_order": 7,
             "pixel_threshold": 1000000,
             "mapping_healpix_order": 7,
@@ -43,9 +42,10 @@
                 "chunksize": 500000,
                 "header": "infer",
                 "schema_file": null,
-                "separator": ",",
                 "column_names": null,
-                "type_map": {}
+                "type_map": null,
+                "parquet_kwargs": null,
+                "kwargs": {}
             }
         }
     }

tests/data/small_sky_order1/provenance_info.json

@@ -5,35 +5,34 @@
     "epoch": "J2000",
     "ra_column": "ra",
     "dec_column": "dec",
-    "version": "0.2.1",
-    "generation_date": "2024.01.09",
+    "version": "0.3.4.dev17+g922a4b7.d20240529",
+    "generation_date": "2024.05.29",
     "tool_args": {
         "tool_name": "hipscat_import",
-        "version": "0.2.1",
+        "version": "0.3.3.dev10+gd573bcd",
         "runtime_args": {
             "catalog_name": "small_sky_order1",
-            "output_path": "tests/data",
+            "output_path": ".",
             "output_artifact_name": "small_sky_order1",
-            "tmp_dir": "",
-            "overwrite": true,
+            "tmp_dir": "/tmp/tmpgasth6x_",
             "dask_tmp": "",
             "dask_n_workers": 1,
             "dask_threads_per_worker": 1,
-            "catalog_path": "tests/data/small_sky_order1",
-            "tmp_path": "tests/data/small_sky_order1/intermediate",
+            "catalog_path": "./small_sky_order1",
+            "tmp_path": "/tmp/tmpgasth6x_/small_sky_order1/intermediate",
             "epoch": "J2000",
             "catalog_type": "object",
-            "input_path": "/home/delucchi/git/hipscat-import/tests/hipscat_import/data/small_sky",
+            "input_path": "../../../hipscat-import/tests/hipscat_import/data/small_sky",
             "input_paths": [
-                "file:///home/delucchi/git/hipscat-import/tests/hipscat_import/data/small_sky/catalog.csv"
+                "file:///home/delucchi/git/fits/tests/data/../../../hipscat-import/tests/hipscat_import/data/small_sky/catalog.csv"
             ],
-            "input_format": "csv",
             "input_file_list": [],
             "ra_column": "ra",
             "dec_column": "dec",
             "use_hipscat_index": false,
             "sort_columns": null,
             "constant_healpix_order": 1,
+            "lowest_healpix_order": 0,
             "highest_healpix_order": 7,
             "pixel_threshold": 1000000,
             "mapping_healpix_order": 1,
@@ -43,9 +42,10 @@
                 "chunksize": 500000,
                 "header": "infer",
                 "schema_file": null,
-                "separator": ",",
                 "column_names": null,
-                "type_map": {}
+                "type_map": null,
+                "parquet_kwargs": null,
+                "kwargs": {}
             }
         }
     }

tests/data/small_sky_order1_id_index/provenance_info.json

@@ -2,29 +2,28 @@
     "catalog_name": "small_sky_order1_id_index",
     "catalog_type": "index",
     "total_rows": 131,
-    "primary_catalog": "/home/delucchi/git/hipscat/tests/data/small_sky_order1",
+    "primary_catalog": "small_sky",
     "indexing_column": "id",
     "extra_columns": [],
-    "version": "0.2.1",
-    "generation_date": "2024.01.09",
+    "version": "0.3.4.dev17+g922a4b7.d20240529",
+    "generation_date": "2024.05.29",
     "tool_args": {
         "tool_name": "hipscat_import",
-        "version": "0.2.1",
+        "version": "0.3.3.dev10+gd573bcd",
         "runtime_args": {
             "catalog_name": "small_sky_order1_id_index",
-            "output_path": "/home/delucchi/git/hipscat/tests/data/",
+            "output_path": ".",
             "output_artifact_name": "small_sky_order1_id_index",
-            "tmp_dir": "",
-            "overwrite": true,
+            "tmp_dir": "/tmp/tmpgasth6x_",
             "dask_tmp": "",
             "dask_n_workers": 1,
             "dask_threads_per_worker": 1,
-            "catalog_path": "/home/delucchi/git/hipscat/tests/data/small_sky_order1_id_index",
-            "tmp_path": "/home/delucchi/git/hipscat/tests/data/small_sky_order1_id_index/intermediate",
-            "input_catalog_path": "/home/delucchi/git/hipscat/tests/data/small_sky_order1",
+            "catalog_path": "./small_sky_order1_id_index",
+            "tmp_path": "/tmp/tmpgasth6x_/small_sky_order1_id_index/intermediate",
+            "input_catalog_path": "small_sky",
             "indexing_column": "id",
             "extra_columns": [],
-            "include_hipscat_index": "False",
+            "include_hipscat_index": false,
             "include_order_pixel": true
         }
     }

tests/data/small_sky_order1_margin/provenance_info.json

@@ -2,25 +2,24 @@
     "catalog_name": "small_sky_order1_margin",
     "catalog_type": "margin",
     "total_rows": 28,
-    "primary_catalog": "data/small_sky_order1",
+    "primary_catalog": "small_sky_order1",
     "margin_threshold": 7200,
-    "version": "0.2.3",
-    "generation_date": "2024.01.30",
+    "version": "0.3.4.dev17+g922a4b7.d20240529",
+    "generation_date": "2024.05.29",
     "tool_args": {
         "tool_name": "hipscat_import",
-        "version": "0.2.2",
+        "version": "0.3.3.dev10+gd573bcd",
         "runtime_args": {
             "catalog_name": "small_sky_order1_margin",
-            "output_path": "data/",
+            "output_path": ".",
             "output_artifact_name": "small_sky_order1_margin",
-            "tmp_dir": "",
-            "overwrite": false,
+            "tmp_dir": "/tmp/tmp8j6vsyq2",
             "dask_tmp": "",
             "dask_n_workers": 1,
             "dask_threads_per_worker": 1,
-            "catalog_path": "data/small_sky_order1_margin",
-            "tmp_path": "data/small_sky_order1_margin/intermediate",
-            "input_catalog_path": "data/small_sky_order1",
+            "catalog_path": "./small_sky_order1_margin",
+            "tmp_path": "/tmp/tmp8j6vsyq2/small_sky_order1_margin/intermediate",
+            "input_catalog_path": "small_sky_order1",
             "margin_threshold": 7200,
             "margin_order": 2
         }

tests/data/small_sky_source/provenance_info.json

@@ -5,34 +5,34 @@
     "epoch": "J2000",
     "ra_column": "source_ra",
     "dec_column": "source_dec",
-    "version": "0.2.7.dev5+gde86705",
-    "generation_date": "2024.03.01",
+    "version": "0.3.4.dev17+g922a4b7.d20240529",
+    "generation_date": "2024.05.29",
     "tool_args": {
         "tool_name": "hipscat_import",
-        "version": "0.2.5.dev5+g0733afb",
+        "version": "0.3.3.dev10+gd573bcd",
         "runtime_args": {
             "catalog_name": "small_sky_source",
             "output_path": ".",
             "output_artifact_name": "small_sky_source",
-            "tmp_dir": "/tmp/user/11115/tmp2vluzdw5",
-            "overwrite": true,
+            "tmp_dir": "/tmp/tmpgasth6x_",
             "dask_tmp": "",
             "dask_n_workers": 1,
             "dask_threads_per_worker": 1,
             "catalog_path": "./small_sky_source",
-            "tmp_path": "/tmp/user/11115/tmp2vluzdw5/small_sky_source/intermediate",
+            "tmp_path": "/tmp/tmpgasth6x_/small_sky_source/intermediate",
             "epoch": "J2000",
             "catalog_type": "source",
             "input_path": "../../../hipscat-import/tests/hipscat_import/data/small_sky_source",
             "input_paths": [
-                "file:///home/delucchi/git/hipscat/tests/data/../../../hipscat-import/tests/hipscat_import/data/small_sky_source/small_sky_source.csv"
+                "file:///home/delucchi/git/fits/tests/data/../../../hipscat-import/tests/hipscat_import/data/small_sky_source/small_sky_source.csv"
             ],
             "input_file_list": [],
             "ra_column": "source_ra",
             "dec_column": "source_dec",
             "use_hipscat_index": false,
             "sort_columns": null,
             "constant_healpix_order": -1,
+            "lowest_healpix_order": 0,
             "highest_healpix_order": 7,
             "pixel_threshold": 3000,
             "mapping_healpix_order": 7,
@@ -42,9 +42,10 @@
                 "chunksize": 500000,
                 "header": "infer",
                 "schema_file": null,
-                "separator": ",",
                 "column_names": null,
-                "type_map": {}
+                "type_map": null,
+                "parquet_kwargs": null,
+                "kwargs": {}
             }
         }
     }