Skip to content

Commit

Permalink
added tests for pixel_catalog_files with url
Browse files Browse the repository at this point in the history
  • Loading branch information
Schwarzam committed May 29, 2024
1 parent 29d0266 commit 81e532f
Show file tree
Hide file tree
Showing 2 changed files with 26 additions and 14 deletions.
31 changes: 17 additions & 14 deletions src/hipscat/io/paths.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,14 @@
from __future__ import annotations

import re
import fsspec
from typing import Dict, List
from urllib.parse import parse_qs, urlencode, urlparse, urlunparse

from fsspec.implementations.http import HTTPFileSystem

from hipscat.io.file_io.file_pointer import FilePointer, append_paths_to_pointer, get_fs
from hipscat.pixel_math.healpix_pixel import INVALID_PIXEL, HealpixPixel

from urllib.parse import urlparse, urlunparse, parse_qs, urlencode

ORDER_DIRECTORY_PREFIX = "Norder"
DIR_DIRECTORY_PREFIX = "Dir"
PIXEL_DIRECTORY_PREFIX = "Npix"
Expand Down Expand Up @@ -90,7 +90,10 @@ def get_healpix_from_path(path: str) -> HealpixPixel:


def pixel_catalog_files(
catalog_base_dir: FilePointer, pixels: List[HealpixPixel], storage_options: Dict | None = None, query_params: dict = None
catalog_base_dir: FilePointer,
pixels: List[HealpixPixel],
storage_options: Dict | None = None,
query_params: dict = None,
) -> List[FilePointer]:
"""Create a list of path *pointers* for pixel catalog files. This will not create the directory
or files.
Expand All @@ -112,12 +115,11 @@ def pixel_catalog_files(
A list of paths to the pixels, in the same order as the input pixel list.
"""
fs, _ = get_fs(catalog_base_dir, storage_options)


url_params = ''
if isinstance(fs, fsspec.implementations.http.HTTPFileSystem) and query_params:

url_params = ""
if isinstance(fs, HTTPFileSystem) and query_params:
url_params = dict_to_query_urlparams(query_params)

base_path_stripped = catalog_base_dir.removesuffix(fs.sep)
return [
fs.sep.join(
Expand All @@ -131,28 +133,29 @@ def pixel_catalog_files(
for pixel in pixels
]


def dict_to_query_urlparams(query_params: dict) -> str:
"""Converts a dictionary to a url query parameter string
Args:
query_params (dict): dictionary of query parameters
Returns:
query parameter string to append to a url
"""
base_url = ""

for key, value in query_params.items():
if isinstance(value, list):
value = ",".join(value)
query_params[key] = value

url_parts = list(urlparse(base_url))
query = dict(parse_qs(url_parts[4]))
query.update(query_params)
url_parts[4] = urlencode(query, doseq=True)
return urlunparse(url_parts)


def pixel_catalog_file(catalog_base_dir: FilePointer, pixel_order: int, pixel_number: int) -> FilePointer:
"""Create path *pointer* for a pixel catalog file. This will not create the directory
Expand Down
9 changes: 9 additions & 0 deletions tests/hipscat/io/test_paths.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,15 @@ def test_pixel_catalog_files():
assert expected == result


def test_pixel_catalog_files_url():
expected = [
"https://foo/Norder=0/Dir=0/Npix=5.parquet?columns=ID%2CRA%2CDEC%2Cr_auto&filters=r_auto%3C13"
]
query_params = {"columns": ["ID", "RA", "DEC", "r_auto"], "filters": ["r_auto<13"]}
result = paths.pixel_catalog_files("https://foo", [HealpixPixel(0, 5)], query_params=query_params)
assert expected == result


def test_get_healpix_from_path():
expected = HealpixPixel(5, 34)

Expand Down

0 comments on commit 81e532f

Please sign in to comment.