Uses LargeString to help avoid offset overflow errors (#147)

moeyensj · Jan 4, 2024 · 15b3517 · 15b3517
1 parent d84f1c0
commit 15b3517
Show file tree

Hide file tree

Showing 15 changed files with 42 additions and 34 deletions.
diff --git a/requirements.txt b/requirements.txt
@@ -1,4 +1,4 @@
-adam-core @ git+https://github.com/B612-Asteroid-Institute/adam_core@main
+adam-core @ git+https://github.com/B612-Asteroid-Institute/adam_core@ef8ee48976dbf9c70580c166de4cc4fd6195fa36
 quivr >= 0.7.1
 astropy >= 5.3.1
 astroquery

diff --git a/setup.cfg b/setup.cfg
@@ -31,7 +31,7 @@ setup_requires =
     wheel
     setuptools_scm >= 6.0
 install_requires =
-    adam-core @ git+https://github.com/B612-Asteroid-Institute/adam_core@16b5d1ed2c3649998b826687e89a406eb3c8ccdf#egg=adam_core
+    adam-core @ git+https://github.com/B612-Asteroid-Institute/adam_core@ef8ee48976dbf9c70580c166de4cc4fd6195fa36#egg=adam_core
     astropy >= 5.3.1
     astroquery
     difi

diff --git a/thor/clusters.py b/thor/clusters.py
@@ -98,16 +98,16 @@ def drop_duplicate_clusters(
 
 
 class Clusters(qv.Table):
-    cluster_id = qv.StringColumn(default=lambda: uuid.uuid4().hex)
+    cluster_id = qv.LargeStringColumn(default=lambda: uuid.uuid4().hex)
     vtheta_x = qv.Float64Column()
     vtheta_y = qv.Float64Column()
     arc_length = qv.Float64Column()
     num_obs = qv.Int64Column()
 
 
 class ClusterMembers(qv.Table):
-    cluster_id = qv.StringColumn()
-    obs_id = qv.StringColumn()
+    cluster_id = qv.LargeStringColumn()
+    obs_id = qv.LargeStringColumn()
 
 
 def find_clusters(points, eps, min_samples, alg="hotspot_2d"):
@@ -547,8 +547,8 @@ def cluster_velocity(
         )
 
         cluster_members = ClusterMembers.from_kwargs(
-            cluster_id=np.concatenate(cluster_members_cluster_ids),
-            obs_id=np.concatenate(cluster_members_obs_ids),
+            cluster_id=np.concatenate(cluster_members_cluster_ids).tolist(),
+            obs_id=np.concatenate(cluster_members_obs_ids).tolist(),
         )
 
     return clusters, cluster_members

diff --git a/thor/observations/observations.py b/thor/observations/observations.py
@@ -24,8 +24,8 @@ class ObserversWithStates(qv.Table):
 
 
 class InputObservations(qv.Table):
-    id = qv.StringColumn()
-    exposure_id = qv.StringColumn()
+    id = qv.LargeStringColumn()
+    exposure_id = qv.LargeStringColumn()
     time = Timestamp.as_column()
     ra = qv.Float64Column()
     dec = qv.Float64Column()
@@ -34,13 +34,13 @@ class InputObservations(qv.Table):
     ra_dec_cov = qv.Float64Column(nullable=True)
     mag = qv.Float64Column()
     mag_sigma = qv.Float64Column(nullable=True)
-    filter = qv.StringColumn()
-    observatory_code = qv.StringColumn()
+    filter = qv.LargeStringColumn()
+    observatory_code = qv.LargeStringColumn()
 
 
 class Observations(qv.Table):
-    id = qv.StringColumn()
-    exposure_id = qv.StringColumn()
+    id = qv.LargeStringColumn()
+    exposure_id = qv.LargeStringColumn()
     coordinates = SphericalCoordinates.as_column()
     photometry = Photometry.as_column()
     state_id = qv.Int64Column()

diff --git a/thor/observations/photometry.py b/thor/observations/photometry.py
@@ -7,4 +7,4 @@ class Photometry(qv.Table):
 
     mag = qv.Float64Column()
     mag_sigma = qv.Float64Column(nullable=True)
-    filter = qv.StringColumn()
+    filter = qv.LargeStringColumn()
diff --git a/thor/orbit.py b/thor/orbit.py
@@ -39,8 +39,8 @@
 
 
 class RangedPointSourceDetections(qv.Table):
-    id = qv.StringColumn()
-    exposure_id = qv.StringColumn()
+    id = qv.LargeStringColumn()
+    exposure_id = qv.LargeStringColumn()
     coordinates = SphericalCoordinates.as_column()
     state_id = qv.Int64Column()
 
@@ -96,8 +96,8 @@ def range_observations_worker(
 
 
 class TestOrbits(qv.Table):
-    orbit_id = qv.StringColumn(default=lambda: uuid.uuid4().hex)
-    object_id = qv.StringColumn(nullable=True)
+    orbit_id = qv.LargeStringColumn(default=lambda: uuid.uuid4().hex)
+    object_id = qv.LargeStringColumn(nullable=True)
     bundle_id = qv.Int64Column(nullable=True)
     coordinates = CartesianCoordinates.as_column()
 

diff --git a/thor/orbit_determination/fitted_orbits.py b/thor/orbit_determination/fitted_orbits.py
@@ -136,8 +136,8 @@ def drop_duplicate_orbits(
 
 class FittedOrbits(qv.Table):
 
-    orbit_id = qv.StringColumn(default=lambda: uuid.uuid4().hex)
-    object_id = qv.StringColumn(nullable=True)
+    orbit_id = qv.LargeStringColumn(default=lambda: uuid.uuid4().hex)
+    object_id = qv.LargeStringColumn(nullable=True)
     coordinates = CartesianCoordinates.as_column()
     arc_length = qv.Float64Column()
     num_obs = qv.Int64Column()
@@ -164,8 +164,8 @@ def to_orbits(self) -> Orbits:
 
 class FittedOrbitMembers(qv.Table):
 
-    orbit_id = qv.StringColumn()
-    obs_id = qv.StringColumn()
+    orbit_id = qv.LargeStringColumn()
+    obs_id = qv.LargeStringColumn()
     residuals = Residuals.as_column(nullable=True)
     solution = qv.BooleanColumn(nullable=True)
     outlier = qv.BooleanColumn(nullable=True)

diff --git a/thor/orbits/attribution.py b/thor/orbits/attribution.py
@@ -34,8 +34,8 @@
 
 
 class Attributions(qv.Table):
-    orbit_id = qv.StringColumn()
-    obs_id = qv.StringColumn()
+    orbit_id = qv.LargeStringColumn()
+    obs_id = qv.LargeStringColumn()
     residuals = Residuals.as_column(nullable=True)
     distance = qv.Float64Column(nullable=True)
 

diff --git a/thor/orbits/gauss.py b/thor/orbits/gauss.py
@@ -203,7 +203,9 @@ def gaussIOD(
         SphericalCoordinates.from_kwargs(
             lon=coords[:, 0],
             lat=coords[:, 1],
-            origin=Origin.from_kwargs(code=np.full(len(coords), "Unknown")),
+            origin=Origin.from_kwargs(
+                code=np.full(len(coords), "Unknown", dtype="object")
+            ),
             frame="equatorial",
         ).to_unit_sphere(),
         representation_out=CartesianCoordinates,
@@ -344,7 +346,9 @@ def gaussIOD(
                 vy=orbits[:, 4],
                 vz=orbits[:, 5],
                 time=Timestamp.from_mjd(epochs, scale="utc"),
-                origin=Origin.from_kwargs(code=np.full(len(orbits), "SUN")),
+                origin=Origin.from_kwargs(
+                    code=np.full(len(orbits), "SUN", dtype="object")
+                ),
                 frame="ecliptic",
             )
         )

diff --git a/thor/orbits/iod.py b/thor/orbits/iod.py
@@ -445,7 +445,9 @@ def iod(
         )
 
         orbit_members = FittedOrbitMembers.from_kwargs(
-            orbit_id=np.full(len(obs_ids_all), orbit_sol.orbit_id[0].as_py()),
+            orbit_id=np.full(
+                len(obs_ids_all), orbit_sol.orbit_id[0].as_py(), dtype="object"
+            ),
             obs_id=obs_ids_all,
             residuals=residuals_sol,
             solution=np.isin(obs_ids_all, obs_ids_sol),

diff --git a/thor/orbits/od.py b/thor/orbits/od.py
@@ -524,7 +524,9 @@ def od(
         # od_orbit["improved"] = improved
 
         od_orbit_members = FittedOrbitMembers.from_kwargs(
-            orbit_id=np.full(len(obs_ids_all), orbit_prev.orbit_id[0].as_py()),
+            orbit_id=np.full(
+                len(obs_ids_all), orbit_prev.orbit_id[0].as_py(), dtype="object"
+            ),
             obs_id=obs_ids_all,
             residuals=residuals_prev,
             solution=np.isin(obs_ids_all, obs_id_outlier, invert=True),

diff --git a/thor/range_and_transform.py b/thor/range_and_transform.py
@@ -27,7 +27,7 @@
 
 
 class TransformedDetections(qv.Table):
-    id = qv.StringColumn()
+    id = qv.LargeStringColumn()
     coordinates = GnomonicCoordinates.as_column()
     state_id = qv.Int64Column()
 

diff --git a/thor/tests/test_clusters.py b/thor/tests/test_clusters.py
@@ -378,7 +378,7 @@ def test_Clusters_drop_duplicates():
         num_obs=np.full(len(cluster_ids), 5),
     )
     cluster_members = ClusterMembers.from_kwargs(
-        cluster_id=np.repeat(cluster_ids, 5),
+        cluster_id=list(np.repeat(cluster_ids, 5)),
         obs_id=[
             obs for cluster_members_i in obs_ids_duplicated for obs in cluster_members_i
         ],

diff --git a/thor/tests/test_orbit_selection.py b/thor/tests/test_orbit_selection.py
@@ -20,7 +20,7 @@ def test_select_average_within_region():
         ap=np.random.rand(5) * 360.0,
         M=np.random.rand(5) * 360.0,
         time=Timestamp.from_mjd([59000.0 for _ in range(5)], scale="tdb"),
-        origin=Origin.from_kwargs(code=np.full(5, "SUN")),
+        origin=Origin.from_kwargs(code=np.full(5, "SUN", dtype="object")),
         frame="ecliptic",
     )
 

diff --git a/thor/utils/tests/test_linkages.py b/thor/utils/tests/test_linkages.py
@@ -12,12 +12,12 @@
 
 
 class Linkages(qv.Table):
-    linkage_id = qv.StringColumn(default=lambda: uuid.uuid4().hex)
+    linkage_id = qv.LargeStringColumn(default=lambda: uuid.uuid4().hex)
 
 
 class LinkageMembers(qv.Table):
-    linkage_id = qv.StringColumn(nullable=True)
-    obs_id = qv.StringColumn(nullable=True)
+    linkage_id = qv.LargeStringColumn(nullable=True)
+    obs_id = qv.LargeStringColumn(nullable=True)
 
 
 def test_sort_by_id_and_time():