From 5fc1c05a94fb73329572153e4f9b8a250d7e7b6b Mon Sep 17 00:00:00 2001 From: Erin Hoops Date: Fri, 31 May 2024 15:26:18 -0700 Subject: [PATCH 01/20] add parent_track_id to zarr --- tools/convert_tracks_csv_to_sparse_zarr.py | 29 +++++++++++++++++++--- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/tools/convert_tracks_csv_to_sparse_zarr.py b/tools/convert_tracks_csv_to_sparse_zarr.py index c6e33d4f..a0dc2b84 100644 --- a/tools/convert_tracks_csv_to_sparse_zarr.py +++ b/tools/convert_tracks_csv_to_sparse_zarr.py @@ -34,9 +34,17 @@ points_to_tracks = lil_matrix((timepoints * max_points_in_timepoint, tracks), dtype=np.int32) tracks_to_children = lil_matrix((tracks, tracks), dtype=np.int32) tracks_to_parents = lil_matrix((tracks, tracks), dtype=np.int32) + +# create a map of trackIds to parent trackIds +direct_parent_map = {} +for point in points: + track_id, t, z, y, x, parent_track_id, n = point # n is the nth point in this timepoint + if direct_parent_map.get(track_id) is None: + direct_parent_map[track_id] = parent_track_id + for point in points: - track_id, t, z, y, x, parent_track_id, n = point - point_id = t * max_points_in_timepoint + n + track_id, t, z, y, x, parent_track_id, n = point # n is the nth point in this timepoint + point_id = t * max_points_in_timepoint + n # creates a sequential ID for each point, but there is no guarantee that the points close together in space points_array[t, 3 * n:3 * (n + 1)] = [z, y, x] @@ -45,7 +53,7 @@ if parent_track_id > 0: tracks_to_parents[track_id - 1, parent_track_id - 1] = 1 tracks_to_children[parent_track_id - 1, track_id - 1] = 1 - + print(f"Munged {len(points)} points in {time.monotonic() - start} seconds") tracks_to_parents.setdiag(1) @@ -55,6 +63,8 @@ start = time.monotonic() iter = 0 +# More info on sparse matrix: https://en.wikipedia.org/wiki/Sparse_matrix#Compressed_sparse_row_(CSR,_CRS_or_Yale_format) +# Transitive closure: https://en.wikipedia.org/wiki/Transitive_closure while tracks_to_parents.nnz != (nxt := tracks_to_parents ** 2).nnz: tracks_to_parents = nxt iter += 1 @@ -71,6 +81,17 @@ start = time.monotonic() tracks_to_tracks = tracks_to_parents + tracks_to_children +tracks_to_tracks = tracks_to_tracks.tolil() +non_zero = tracks_to_tracks.nonzero() + +for i in range(len(non_zero[0])): + track_id = non_zero[0][i] + 1 + parent_track_id = direct_parent_map[track_id] + + tracks_to_tracks[non_zero[0][i], non_zero[1][i]] = parent_track_id + +# scipy sparse +# numpy where np.nonzero # Convert to CSR format for efficient row slicing tracks_to_points = points_to_tracks.T.tocsr() @@ -157,4 +178,4 @@ # note the relatively small size of the indptr arrays # tracks_to_points/data is a redundant copy of the points array to avoid having -# to fetch point coordinates individually +# to fetch point coordinates individually \ No newline at end of file From f2e21e236f1a32cce79a8c345bd70da06365b2ec Mon Sep 17 00:00:00 2001 From: Erin Hoops Date: Fri, 7 Jun 2024 12:23:24 -0700 Subject: [PATCH 02/20] wip - with feedback changes --- src/lib/TrackManager.ts | 2 +- tools/convert_tracks_csv_to_sparse_zarr.py | 26 +++++++++++----------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/lib/TrackManager.ts b/src/lib/TrackManager.ts index 7ae6add8..7fff70c5 100644 --- a/src/lib/TrackManager.ts +++ b/src/lib/TrackManager.ts @@ -146,7 +146,7 @@ export async function loadTrackManager(url: string) { }); const pointsToTracks = await openSparseZarrArray(url, "points_to_tracks", false); const tracksToPoints = await openSparseZarrArray(url, "tracks_to_points", true); - const tracksToTracks = await openSparseZarrArray(url, "tracks_to_tracks", false); + const tracksToTracks = await openSparseZarrArray(url, "tracks_to_tracks", true); trackManager = new TrackManager(url, points, pointsToTracks, tracksToPoints, tracksToTracks); } catch (err) { console.error("Error opening TrackManager: %s", err); diff --git a/tools/convert_tracks_csv_to_sparse_zarr.py b/tools/convert_tracks_csv_to_sparse_zarr.py index a0dc2b84..93fd695a 100644 --- a/tools/convert_tracks_csv_to_sparse_zarr.py +++ b/tools/convert_tracks_csv_to_sparse_zarr.py @@ -6,7 +6,7 @@ import zarr from scipy.sparse import lil_matrix -root_dir = "/Users/aandersoniii/Data/tracking/" +root_dir = "/Users/ehoops/development/" start = time.monotonic() points = [] @@ -35,17 +35,19 @@ tracks_to_children = lil_matrix((tracks, tracks), dtype=np.int32) tracks_to_parents = lil_matrix((tracks, tracks), dtype=np.int32) -# create a map of trackIds to parent trackIds -direct_parent_map = {} -for point in points: - track_id, t, z, y, x, parent_track_id, n = point # n is the nth point in this timepoint - if direct_parent_map.get(track_id) is None: - direct_parent_map[track_id] = parent_track_id +# create a map of the track_index to the parent_track_index +# track_id and parent_track_id are 1-indexed, track_index and parent_track_index are 0-indexed +direct_parent_index_map = {} for point in points: track_id, t, z, y, x, parent_track_id, n = point # n is the nth point in this timepoint point_id = t * max_points_in_timepoint + n # creates a sequential ID for each point, but there is no guarantee that the points close together in space + track_index = track_id - 1 + if track_index not in direct_parent_index_map: + # maps the track_index to the parent_track_index + direct_parent_index_map[track_index] = parent_track_id - 1 + points_array[t, 3 * n:3 * (n + 1)] = [z, y, x] points_to_tracks[point_id, track_id - 1] = 1 @@ -85,13 +87,11 @@ non_zero = tracks_to_tracks.nonzero() for i in range(len(non_zero[0])): - track_id = non_zero[0][i] + 1 - parent_track_id = direct_parent_map[track_id] - - tracks_to_tracks[non_zero[0][i], non_zero[1][i]] = parent_track_id + # track_index = track_id - 1 since track_id is 1-indexed + track_index = non_zero[1][i] + parent_track_index = direct_parent_index_map[track_index] -# scipy sparse -# numpy where np.nonzero + tracks_to_tracks[non_zero[0][i], non_zero[1][i]] = parent_track_index + 1 # Convert to CSR format for efficient row slicing tracks_to_points = points_to_tracks.T.tocsr() From 5e7ce085acfc2c73687b799e85f4ea7f5f857f41 Mon Sep 17 00:00:00 2001 From: Erin Hoops Date: Mon, 10 Jun 2024 15:16:03 -0700 Subject: [PATCH 03/20] wip - tracks are broken --- src/components/App.tsx | 10 +++++----- src/lib/PointCanvas.ts | 4 ++-- src/lib/TrackManager.ts | 11 ++++++++--- src/lib/ViewerState.ts | 7 ++++--- src/lib/three/Track.ts | 4 +++- tools/convert_tracks_csv_to_sparse_zarr.py | 6 ++++-- 6 files changed, 26 insertions(+), 16 deletions(-) diff --git a/src/components/App.tsx b/src/components/App.tsx index 3bfaf99b..f6b5d971 100644 --- a/src/components/App.tsx +++ b/src/components/App.tsx @@ -158,15 +158,15 @@ export default function App() { for (const trackId of trackIds) { if (canvas.fetchedRootTrackIds.has(trackId)) continue; canvas.fetchedRootTrackIds.add(trackId); - const lineage = await trackManager.fetchLineageForTrack(trackId); - for (const relatedTrackId of lineage) { - if (canvas.tracks.has(relatedTrackId)) continue; + const [lineage, trackData] = await trackManager.fetchLineageForTrack(trackId); + lineage.forEach(async (relatedTrackId: number, index) => { + if (canvas.tracks.has(relatedTrackId)) return; const [pos, ids] = await trackManager.fetchPointsForTrack(relatedTrackId); // adding the track *in* the dispatcher creates issues with duplicate fetching // but we refresh so the selected/loaded count is updated - canvas.addTrack(relatedTrackId, pos, ids); + canvas.addTrack(relatedTrackId, pos, ids, trackData[index]); dispatchCanvas({ type: ActionType.REFRESH }); - } + }); } setNumLoadingTracks((n) => n - 1); } diff --git a/src/lib/PointCanvas.ts b/src/lib/PointCanvas.ts index 11516acd..189fad17 100644 --- a/src/lib/PointCanvas.ts +++ b/src/lib/PointCanvas.ts @@ -222,13 +222,13 @@ export class PointCanvas { this.points.geometry.computeBoundingSphere(); } - addTrack(trackID: number, positions: Float32Array, ids: Int32Array): Track | null { + addTrack(trackID: number, positions: Float32Array, ids: Int32Array, parentTrackID: number): Track | null { if (this.tracks.has(trackID)) { // this is a warning because it should alert us to duplicate fetching console.warn("Track with ID %d already exists", trackID); return null; } - const track = Track.new(positions, ids, this.maxPointsPerTimepoint); + const track = Track.new(positions, ids, this.maxPointsPerTimepoint, parentTrackID); track.updateAppearance(this.showTracks, this.showTrackHighlights, this.minTime, this.maxTime); this.tracks.set(trackID, track); this.scene.add(track); diff --git a/src/lib/TrackManager.ts b/src/lib/TrackManager.ts index 7fff70c5..9acfd73e 100644 --- a/src/lib/TrackManager.ts +++ b/src/lib/TrackManager.ts @@ -129,10 +129,15 @@ export class TrackManager { return [flatPoints, pointIDs]; } - async fetchLineageForTrack(trackID: number): Promise { + async fetchLineageForTrack(trackID: number): Promise<[Int32Array, Int32Array]> { const rowStartEnd = await this.tracksToTracks.getIndPtr(slice(trackID, trackID + 2)); - const lineage = await this.tracksToTracks.indices.get([slice(rowStartEnd[0], rowStartEnd[1])]); - return lineage.data; + const lineage = await this.tracksToTracks.indices + .get([slice(rowStartEnd[0], rowStartEnd[1])]) + .then((lineage: SparseZarrArray) => lineage.data); + const trackData = await this.tracksToTracks.data + .get([slice(rowStartEnd[0], rowStartEnd[1])]) + .then((trackData: SparseZarrArray) => trackData.data); + return Promise.all([lineage, trackData]); } } diff --git a/src/lib/ViewerState.ts b/src/lib/ViewerState.ts index 8fd63b40..dc56a1ad 100644 --- a/src/lib/ViewerState.ts +++ b/src/lib/ViewerState.ts @@ -1,6 +1,7 @@ -export const DEFAULT_ZARR_URL = - "https://sci-imaging-vis-public-demo-data.s3.us-west-2.amazonaws.com" + - "/points-web-viewer/sparse-zarr-v2/ZSNS001_tracks_bundle.zarr"; +// export const DEFAULT_ZARR_URL = +// "https://sci-imaging-vis-public-demo-data.s3.us-west-2.amazonaws.com" + +// "/points-web-viewer/sparse-zarr-v2/ZSNS001_tracks_bundle.zarr"; +export const DEFAULT_ZARR_URL = "http://localhost:8000/ZSNS001_tracks_bundle.zarr"; const HASH_KEY = "viewerState"; diff --git a/src/lib/three/Track.ts b/src/lib/three/Track.ts index aa6298df..90d5022f 100644 --- a/src/lib/three/Track.ts +++ b/src/lib/three/Track.ts @@ -14,10 +14,11 @@ import { TrackMaterial } from "./TrackMaterial.js"; export class Track extends Mesh { isTrack = true; type = "Track"; + parentTrackID: number = -1; declare geometry: TrackGeometry; declare material: TrackMaterial; - static new(positions: Float32Array, pointIDs: Int32Array, maxPointsPerTimepoint: number) { + static new(positions: Float32Array, pointIDs: Int32Array, maxPointsPerTimepoint: number, parentTrackID: number) { const geometry = new TrackGeometry(); const material = new TrackMaterial({ vertexColors: true, @@ -44,6 +45,7 @@ export class Track extends Mesh { track.geometry.setColors(colors); track.geometry.setTime(time); track.geometry.computeBoundingSphere(); + track.parentTrackID = parentTrackID; return track; } diff --git a/tools/convert_tracks_csv_to_sparse_zarr.py b/tools/convert_tracks_csv_to_sparse_zarr.py index 93fd695a..4c59975a 100644 --- a/tools/convert_tracks_csv_to_sparse_zarr.py +++ b/tools/convert_tracks_csv_to_sparse_zarr.py @@ -45,8 +45,10 @@ track_index = track_id - 1 if track_index not in direct_parent_index_map: - # maps the track_index to the parent_track_index - direct_parent_index_map[track_index] = parent_track_id - 1 + # maps the track_index to the parent_track_id + direct_parent_index_map[track_id - 1] = parent_track_id - 1 + if parent_track_id >= 0: + print(f"Track {track_id} has point {point_id} and parent {parent_track_id}") points_array[t, 3 * n:3 * (n + 1)] = [z, y, x] From a372a66f34f42d52b66510cf6bdbbd91a138309b Mon Sep 17 00:00:00 2001 From: Erin Hoops Date: Tue, 11 Jun 2024 09:29:49 -0700 Subject: [PATCH 04/20] wip - tracks loading works. moving parentId to pointcanvas. --- src/components/App.tsx | 21 +++++++++++++++------ src/lib/PointCanvas.ts | 22 ++++++++++++++-------- src/lib/TrackManager.ts | 1 + src/lib/three/Track.ts | 4 +--- src/lib/three/TrackGeometry.ts | 8 ++++++-- tools/convert_tracks_csv_to_sparse_zarr.py | 7 ++++--- 6 files changed, 41 insertions(+), 22 deletions(-) diff --git a/src/components/App.tsx b/src/components/App.tsx index f6b5d971..ffa26ca0 100644 --- a/src/components/App.tsx +++ b/src/components/App.tsx @@ -149,27 +149,36 @@ export default function App() { // this fetches the entire lineage for each track const updateTracks = async () => { console.debug("updateTracks: ", canvas.selectedPointIds); - for (const pointId of canvas.selectedPointIds) { - if (canvas.fetchedPointIds.has(pointId)) continue; + canvas.selectedPointIds.forEach(async (pointId) => { + if (canvas.fetchedPointIds.has(pointId)) return; setNumLoadingTracks((n) => n + 1); canvas.fetchedPointIds.add(pointId); const trackIds = await trackManager.fetchTrackIDsForPoint(pointId); // TODO: points actually only belong to one track, so can get rid of the outer loop - for (const trackId of trackIds) { - if (canvas.fetchedRootTrackIds.has(trackId)) continue; + trackIds.forEach(async (trackId) => { + if (canvas.fetchedRootTrackIds.has(trackId)) return; canvas.fetchedRootTrackIds.add(trackId); const [lineage, trackData] = await trackManager.fetchLineageForTrack(trackId); + console.log("lineage for track %d: %o", trackId, lineage); + console.log("track data for track %d: %o", trackId, trackData); lineage.forEach(async (relatedTrackId: number, index) => { if (canvas.tracks.has(relatedTrackId)) return; const [pos, ids] = await trackManager.fetchPointsForTrack(relatedTrackId); // adding the track *in* the dispatcher creates issues with duplicate fetching // but we refresh so the selected/loaded count is updated + console.log( + "add track %d at pos %o, ids %o, parentTrackId %o", + relatedTrackId, + pos, + ids, + trackData[index], + ); canvas.addTrack(relatedTrackId, pos, ids, trackData[index]); dispatchCanvas({ type: ActionType.REFRESH }); }); - } + }); setNumLoadingTracks((n) => n - 1); - } + }); }; updateTracks(); // TODO: add missing dependencies diff --git a/src/lib/PointCanvas.ts b/src/lib/PointCanvas.ts index 189fad17..9799efbd 100644 --- a/src/lib/PointCanvas.ts +++ b/src/lib/PointCanvas.ts @@ -24,7 +24,12 @@ import { Track } from "@/lib/three/Track"; import { PointSelector, PointSelectionMode } from "@/lib/PointSelector"; import { ViewerState } from "./ViewerState"; -type Tracks = Map; +// TrackType is a place to store the visual information about a track and any track-specific attributes +type TrackType = { + threeTrack: Track; + parentTrackId: number; +}; +type Tracks = Map; export class PointCanvas { readonly scene: Scene; @@ -46,6 +51,7 @@ export class PointCanvas { readonly fetchedRootTrackIds = new Set(); // Needed to skip fetches for point IDs that been selected. readonly fetchedPointIds = new Set(); + // TODO: (Erin): - track relationships - map of trackId -> parentTrackId // All the point IDs that have been selected. // PointCanvas.selector.selection is the transient array of selected @@ -228,24 +234,24 @@ export class PointCanvas { console.warn("Track with ID %d already exists", trackID); return null; } - const track = Track.new(positions, ids, this.maxPointsPerTimepoint, parentTrackID); + const track = Track.new(positions, ids, this.maxPointsPerTimepoint); track.updateAppearance(this.showTracks, this.showTrackHighlights, this.minTime, this.maxTime); - this.tracks.set(trackID, track); + this.tracks.set(trackID, { threeTrack: track, parentTrackId: parentTrackID }); // TODO: (Erin): - update here this.scene.add(track); return track; } updateAllTrackHighlights() { - for (const track of this.tracks.values()) { - track.updateAppearance(this.showTracks, this.showTrackHighlights, this.minTime, this.maxTime); - } + this.tracks.forEach((track) => { + track.threeTrack.updateAppearance(this.showTracks, this.showTrackHighlights, this.minTime, this.maxTime); + }); } removeTrack(trackID: number) { const track = this.tracks.get(trackID); if (track) { - this.scene.remove(track); - track.dispose(); + this.scene.remove(track.threeTrack); + track.threeTrack.dispose(); this.tracks.delete(trackID); } else { console.warn("No track with ID %d to remove", trackID); diff --git a/src/lib/TrackManager.ts b/src/lib/TrackManager.ts index 9acfd73e..561f742f 100644 --- a/src/lib/TrackManager.ts +++ b/src/lib/TrackManager.ts @@ -131,6 +131,7 @@ export class TrackManager { async fetchLineageForTrack(trackID: number): Promise<[Int32Array, Int32Array]> { const rowStartEnd = await this.tracksToTracks.getIndPtr(slice(trackID, trackID + 2)); + console.log("rowStartEnd: %s", rowStartEnd); // THIS IS A PROBLEM - START AND END ARE THE SAME const lineage = await this.tracksToTracks.indices .get([slice(rowStartEnd[0], rowStartEnd[1])]) .then((lineage: SparseZarrArray) => lineage.data); diff --git a/src/lib/three/Track.ts b/src/lib/three/Track.ts index 90d5022f..aa6298df 100644 --- a/src/lib/three/Track.ts +++ b/src/lib/three/Track.ts @@ -14,11 +14,10 @@ import { TrackMaterial } from "./TrackMaterial.js"; export class Track extends Mesh { isTrack = true; type = "Track"; - parentTrackID: number = -1; declare geometry: TrackGeometry; declare material: TrackMaterial; - static new(positions: Float32Array, pointIDs: Int32Array, maxPointsPerTimepoint: number, parentTrackID: number) { + static new(positions: Float32Array, pointIDs: Int32Array, maxPointsPerTimepoint: number) { const geometry = new TrackGeometry(); const material = new TrackMaterial({ vertexColors: true, @@ -45,7 +44,6 @@ export class Track extends Mesh { track.geometry.setColors(colors); track.geometry.setTime(time); track.geometry.computeBoundingSphere(); - track.parentTrackID = parentTrackID; return track; } diff --git a/src/lib/three/TrackGeometry.ts b/src/lib/three/TrackGeometry.ts index b9eff7c6..d5fda153 100644 --- a/src/lib/three/TrackGeometry.ts +++ b/src/lib/three/TrackGeometry.ts @@ -14,22 +14,26 @@ class TrackGeometry extends LineSegmentsGeometry { // converts [ x1, y1, z1, x2, y2, z2, ... ] to pairs format const length = array.length - 3; - const points = new Float32Array(2 * length); + const points = new Float32Array(2 * length); // start and end of each line for (let i = 0; i < length; i += 3) { + // start point points[2 * i] = array[i]; points[2 * i + 1] = array[i + 1]; points[2 * i + 2] = array[i + 2]; - + // end point points[2 * i + 3] = array[i + 3]; points[2 * i + 4] = array[i + 4]; points[2 * i + 5] = array[i + 5]; } + // TODO: (Erin) undo this to get the actual positions. start point for the first and end point for the rest or vice versa + super.setPositions(points); return this; } + // TODO: (Erin) getPositions() here setColors(array: number[] | Float32Array) { // converts [ r1, g1, b1, r2, g2, b2, ... ] to pairs format diff --git a/tools/convert_tracks_csv_to_sparse_zarr.py b/tools/convert_tracks_csv_to_sparse_zarr.py index 4c59975a..22235ac3 100644 --- a/tools/convert_tracks_csv_to_sparse_zarr.py +++ b/tools/convert_tracks_csv_to_sparse_zarr.py @@ -45,10 +45,10 @@ track_index = track_id - 1 if track_index not in direct_parent_index_map: - # maps the track_index to the parent_track_id + # maps the track_index to the parent_track_index direct_parent_index_map[track_id - 1] = parent_track_id - 1 if parent_track_id >= 0: - print(f"Track {track_id} has point {point_id} and parent {parent_track_id}") + print(f"Track {track_id} has point {point_id} and parent {parent_track_id}") # this seems to work points_array[t, 3 * n:3 * (n + 1)] = [z, y, x] @@ -92,7 +92,8 @@ # track_index = track_id - 1 since track_id is 1-indexed track_index = non_zero[1][i] parent_track_index = direct_parent_index_map[track_index] - + if parent_track_index >= -1: + print(f"Track {track_index + 1} has parent {parent_track_index + 1}") tracks_to_tracks[non_zero[0][i], non_zero[1][i]] = parent_track_index + 1 # Convert to CSR format for efficient row slicing From e68f702ae6c41d44f50ef2a9e0380e62a303caa6 Mon Sep 17 00:00:00 2001 From: Erin Hoops Date: Tue, 11 Jun 2024 14:52:15 -0700 Subject: [PATCH 05/20] WIP - download a csv with trackID and parentTrackID --- src/components/App.tsx | 19 ++++++++------- src/components/DownloadButton.tsx | 40 +++++++++++++++++++++++++++++++ src/lib/TrackManager.ts | 1 - 3 files changed, 50 insertions(+), 10 deletions(-) create mode 100644 src/components/DownloadButton.tsx diff --git a/src/components/App.tsx b/src/components/App.tsx index ffa26ca0..ff0680eb 100644 --- a/src/components/App.tsx +++ b/src/components/App.tsx @@ -16,6 +16,7 @@ import { PointSelectionMode } from "@/lib/PointSelector"; import LeftSidebarWrapper from "./leftSidebar/LeftSidebarWrapper"; import { TimestampOverlay } from "./overlays/TimestampOverlay"; import { ColorMap } from "./overlays/ColorMap"; +import { DownloadButton, TrackDownloadData } from "./DownloadButton"; // Ideally we do this here so that we can use initial values as default values for React state. const initialViewerState = ViewerState.fromUrlHash(window.location.hash); @@ -159,20 +160,11 @@ export default function App() { if (canvas.fetchedRootTrackIds.has(trackId)) return; canvas.fetchedRootTrackIds.add(trackId); const [lineage, trackData] = await trackManager.fetchLineageForTrack(trackId); - console.log("lineage for track %d: %o", trackId, lineage); - console.log("track data for track %d: %o", trackId, trackData); lineage.forEach(async (relatedTrackId: number, index) => { if (canvas.tracks.has(relatedTrackId)) return; const [pos, ids] = await trackManager.fetchPointsForTrack(relatedTrackId); // adding the track *in* the dispatcher creates issues with duplicate fetching // but we refresh so the selected/loaded count is updated - console.log( - "add track %d at pos %o, ids %o, parentTrackId %o", - relatedTrackId, - pos, - ids, - trackData[index], - ); canvas.addTrack(relatedTrackId, pos, ids, trackData[index]); dispatchCanvas({ type: ActionType.REFRESH }); }); @@ -203,6 +195,14 @@ export default function App() { } }, [dispatchCanvas, numTimes, playing]); + const getTrackDownloadData = () => { + const trackData: TrackDownloadData[] = []; + canvas.tracks.forEach((track, trackID) => { + trackData.push([trackID, track.parentTrackId]); + }); + return trackData; + }; + return ( {/* TODO: components *could* go deeper still for organization */} @@ -254,6 +254,7 @@ export default function App() { dispatchCanvas({ type: ActionType.SELECTION_MODE, selectionMode: value }); }} /> + diff --git a/src/components/DownloadButton.tsx b/src/components/DownloadButton.tsx new file mode 100644 index 00000000..3c68a8af --- /dev/null +++ b/src/components/DownloadButton.tsx @@ -0,0 +1,40 @@ +import { Button } from "@czi-sds/components"; + +// TrackDownloadData is a tuple of trackID and parentTrackID +export type TrackDownloadData = [number, number]; +interface DownloadButtonProps { + getDownloadData: () => TrackDownloadData[]; +} + +const dataHeaders = ["trackID", "parentTrackID"]; + +const convertToCSV = (nestedArray: (string | number)[][]) => { + let csvString = ""; + + for (let row = 0; row < nestedArray.length; row++) { + let line = ""; + for (const entry in nestedArray[row]) { + if (line !== "") line += ","; + + line += nestedArray[row][entry]; + } + csvString += line + "\r\n"; + } + return csvString; +}; + +export const DownloadButton = (props: DownloadButtonProps) => { + const downloadCSV = () => { + const data = props.getDownloadData(); + const csvData = new Blob([`${convertToCSV([dataHeaders])}${convertToCSV(data)}`], { type: "text/csv" }); + const csvURL = URL.createObjectURL(csvData); + const link = document.createElement("a"); + link.href = csvURL; + link.download = "points.csv"; + document.body.appendChild(link); + link.click(); + document.body.removeChild(link); + }; + + return ; +}; diff --git a/src/lib/TrackManager.ts b/src/lib/TrackManager.ts index 561f742f..9acfd73e 100644 --- a/src/lib/TrackManager.ts +++ b/src/lib/TrackManager.ts @@ -131,7 +131,6 @@ export class TrackManager { async fetchLineageForTrack(trackID: number): Promise<[Int32Array, Int32Array]> { const rowStartEnd = await this.tracksToTracks.getIndPtr(slice(trackID, trackID + 2)); - console.log("rowStartEnd: %s", rowStartEnd); // THIS IS A PROBLEM - START AND END ARE THE SAME const lineage = await this.tracksToTracks.indices .get([slice(rowStartEnd[0], rowStartEnd[1])]) .then((lineage: SparseZarrArray) => lineage.data); From 95784c5787750c6a98f0eb2c411fe20e89716f31 Mon Sep 17 00:00:00 2001 From: Erin Hoops Date: Tue, 11 Jun 2024 15:18:31 -0700 Subject: [PATCH 06/20] Exports an incorrect csv --- src/components/App.tsx | 9 ++++++++- src/components/DownloadButton.tsx | 7 ++++--- src/lib/PointCanvas.ts | 5 ++--- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/components/App.tsx b/src/components/App.tsx index ff0680eb..9b3116de 100644 --- a/src/components/App.tsx +++ b/src/components/App.tsx @@ -198,7 +198,14 @@ export default function App() { const getTrackDownloadData = () => { const trackData: TrackDownloadData[] = []; canvas.tracks.forEach((track, trackID) => { - trackData.push([trackID, track.parentTrackId]); + const startPositions = track.threeTrack.geometry.getAttribute("instanceStart"); + const startTimes = track.threeTrack.geometry.getAttribute("instanceTimeStart"); + for (let i = 0; i < startPositions.count / 3; i += 3) { + const x = startPositions.array[i]; + const y = startPositions.array[i + 1]; + const z = startPositions.array[i + 2]; + trackData.push([trackID, startTimes.array[i], x, y, z, track.parentTrackID]); + } }); return trackData; }; diff --git a/src/components/DownloadButton.tsx b/src/components/DownloadButton.tsx index 3c68a8af..dc0280c1 100644 --- a/src/components/DownloadButton.tsx +++ b/src/components/DownloadButton.tsx @@ -1,12 +1,13 @@ import { Button } from "@czi-sds/components"; -// TrackDownloadData is a tuple of trackID and parentTrackID -export type TrackDownloadData = [number, number]; +// TrackDownloadData is a list for each point +// It contains trackID, time, x, y, z, parentTrackID +export type TrackDownloadData = [number, number, number, number, number, number]; interface DownloadButtonProps { getDownloadData: () => TrackDownloadData[]; } -const dataHeaders = ["trackID", "parentTrackID"]; +const dataHeaders = ["trackID", "t", "x", "y", "z", "parentTrackID"]; const convertToCSV = (nestedArray: (string | number)[][]) => { let csvString = ""; diff --git a/src/lib/PointCanvas.ts b/src/lib/PointCanvas.ts index 9799efbd..1deb874a 100644 --- a/src/lib/PointCanvas.ts +++ b/src/lib/PointCanvas.ts @@ -27,7 +27,7 @@ import { ViewerState } from "./ViewerState"; // TrackType is a place to store the visual information about a track and any track-specific attributes type TrackType = { threeTrack: Track; - parentTrackId: number; + parentTrackID: number; }; type Tracks = Map; @@ -51,7 +51,6 @@ export class PointCanvas { readonly fetchedRootTrackIds = new Set(); // Needed to skip fetches for point IDs that been selected. readonly fetchedPointIds = new Set(); - // TODO: (Erin): - track relationships - map of trackId -> parentTrackId // All the point IDs that have been selected. // PointCanvas.selector.selection is the transient array of selected @@ -236,7 +235,7 @@ export class PointCanvas { } const track = Track.new(positions, ids, this.maxPointsPerTimepoint); track.updateAppearance(this.showTracks, this.showTrackHighlights, this.minTime, this.maxTime); - this.tracks.set(trackID, { threeTrack: track, parentTrackId: parentTrackID }); // TODO: (Erin): - update here + this.tracks.set(trackID, { threeTrack: track, parentTrackID: parentTrackID }); // TODO: (Erin): - update here this.scene.add(track); return track; } From 98978acfc16f0e79bc64563ce3dbd12fb786a9b7 Mon Sep 17 00:00:00 2001 From: Erin Hoops Date: Thu, 13 Jun 2024 13:21:22 -0700 Subject: [PATCH 07/20] Fixes. Seems to be correct now --- src/components/App.tsx | 23 +++++++++++++++++----- tools/convert_tracks_csv_to_sparse_zarr.py | 4 ---- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/src/components/App.tsx b/src/components/App.tsx index 9b3116de..6ebec359 100644 --- a/src/components/App.tsx +++ b/src/components/App.tsx @@ -200,12 +200,25 @@ export default function App() { canvas.tracks.forEach((track, trackID) => { const startPositions = track.threeTrack.geometry.getAttribute("instanceStart"); const startTimes = track.threeTrack.geometry.getAttribute("instanceTimeStart"); - for (let i = 0; i < startPositions.count / 3; i += 3) { - const x = startPositions.array[i]; - const y = startPositions.array[i + 1]; - const z = startPositions.array[i + 2]; - trackData.push([trackID, startTimes.array[i], x, y, z, track.parentTrackID]); + for (let i = 0; i < startPositions.count; i++) { + const t = startTimes.getX(i); + const x = startPositions.getX(i); + const y = startPositions.getY(i); + const z = startPositions.getZ(i); + trackData.push([trackID + 1, t, x, y, z, track.parentTrackID]); } + const endPositions = track.threeTrack.geometry.getAttribute("instanceEnd"); + const endTimes = track.threeTrack.geometry.getAttribute("instanceTimeEnd"); + const lastIndex = endPositions.count - 1; + trackData.push([ + trackID + 1, + endTimes.getX(lastIndex), + endPositions.getX(lastIndex), + endPositions.getY(lastIndex), + endPositions.getZ(lastIndex), + track.parentTrackID, + ]); + debugger; }); return trackData; }; diff --git a/tools/convert_tracks_csv_to_sparse_zarr.py b/tools/convert_tracks_csv_to_sparse_zarr.py index 22235ac3..a32dc98f 100644 --- a/tools/convert_tracks_csv_to_sparse_zarr.py +++ b/tools/convert_tracks_csv_to_sparse_zarr.py @@ -47,8 +47,6 @@ if track_index not in direct_parent_index_map: # maps the track_index to the parent_track_index direct_parent_index_map[track_id - 1] = parent_track_id - 1 - if parent_track_id >= 0: - print(f"Track {track_id} has point {point_id} and parent {parent_track_id}") # this seems to work points_array[t, 3 * n:3 * (n + 1)] = [z, y, x] @@ -92,8 +90,6 @@ # track_index = track_id - 1 since track_id is 1-indexed track_index = non_zero[1][i] parent_track_index = direct_parent_index_map[track_index] - if parent_track_index >= -1: - print(f"Track {track_index + 1} has parent {parent_track_index + 1}") tracks_to_tracks[non_zero[0][i], non_zero[1][i]] = parent_track_index + 1 # Convert to CSR format for efficient row slicing From 527273a9d3d0713f0832e0cb189d334c7af65e64 Mon Sep 17 00:00:00 2001 From: Erin Hoops Date: Thu, 13 Jun 2024 13:38:09 -0700 Subject: [PATCH 08/20] Round positions to nearest thousandth --- src/components/App.tsx | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/src/components/App.tsx b/src/components/App.tsx index 6ebec359..516db553 100644 --- a/src/components/App.tsx +++ b/src/components/App.tsx @@ -201,24 +201,28 @@ export default function App() { const startPositions = track.threeTrack.geometry.getAttribute("instanceStart"); const startTimes = track.threeTrack.geometry.getAttribute("instanceTimeStart"); for (let i = 0; i < startPositions.count; i++) { - const t = startTimes.getX(i); - const x = startPositions.getX(i); - const y = startPositions.getY(i); - const z = startPositions.getZ(i); - trackData.push([trackID + 1, t, x, y, z, track.parentTrackID]); + trackData.push([ + // trackID is 1-indexed in input and output CSVs + trackID + 1, + startTimes.getX(i), + Math.round(startPositions.getX(i) * 1000) / 1000, // round to 3 decimal places + Math.round(startPositions.getY(i) * 1000) / 1000, + Math.round(startPositions.getZ(i) * 1000) / 1000, + track.parentTrackID, + ]); } const endPositions = track.threeTrack.geometry.getAttribute("instanceEnd"); const endTimes = track.threeTrack.geometry.getAttribute("instanceTimeEnd"); const lastIndex = endPositions.count - 1; trackData.push([ + // trackID is 1-indexed in input and output CSVs trackID + 1, - endTimes.getX(lastIndex), - endPositions.getX(lastIndex), - endPositions.getY(lastIndex), - endPositions.getZ(lastIndex), + Math.round(endTimes.getX(lastIndex)), + Math.round(endPositions.getX(lastIndex) * 1000) / 1000, // round to 3 decimal places + Math.round(endPositions.getY(lastIndex) * 1000) / 1000, + Math.round(endPositions.getZ(lastIndex) * 1000) / 1000, track.parentTrackID, ]); - debugger; }); return trackData; }; From 098da05f63c2f95812833e924b569f5b203a5cf4 Mon Sep 17 00:00:00 2001 From: Erin Hoops Date: Thu, 13 Jun 2024 13:47:33 -0700 Subject: [PATCH 09/20] Basic button styling/placement --- src/components/App.tsx | 4 ++-- src/components/CellControls.tsx | 3 +++ src/components/DownloadButton.tsx | 6 +++++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/components/App.tsx b/src/components/App.tsx index 516db553..389bdb75 100644 --- a/src/components/App.tsx +++ b/src/components/App.tsx @@ -16,7 +16,7 @@ import { PointSelectionMode } from "@/lib/PointSelector"; import LeftSidebarWrapper from "./leftSidebar/LeftSidebarWrapper"; import { TimestampOverlay } from "./overlays/TimestampOverlay"; import { ColorMap } from "./overlays/ColorMap"; -import { DownloadButton, TrackDownloadData } from "./DownloadButton"; +import { TrackDownloadData } from "./DownloadButton"; // Ideally we do this here so that we can use initial values as default values for React state. const initialViewerState = ViewerState.fromUrlHash(window.location.hash); @@ -267,6 +267,7 @@ export default function App() { clearTracks={() => { dispatchCanvas({ type: ActionType.REMOVE_ALL_TRACKS }); }} + getTrackDownloadData={getTrackDownloadData} numSelectedCells={numTracksLoaded} trackManager={trackManager} pointBrightness={canvas.pointBrightness} @@ -278,7 +279,6 @@ export default function App() { dispatchCanvas({ type: ActionType.SELECTION_MODE, selectionMode: value }); }} /> - diff --git a/src/components/CellControls.tsx b/src/components/CellControls.tsx index 25907d20..e9238713 100644 --- a/src/components/CellControls.tsx +++ b/src/components/CellControls.tsx @@ -4,9 +4,11 @@ import { FontS, SmallCapsButton, ControlLabel } from "@/components/Styled"; import { PointSelectionMode } from "@/lib/PointSelector"; import { TrackManager } from "@/lib/TrackManager"; +import { DownloadButton, TrackDownloadData } from "./DownloadButton"; interface CellControlsProps { clearTracks: () => void; + getTrackDownloadData: () => TrackDownloadData[]; numSelectedCells?: number; trackManager: TrackManager | null; pointBrightness: number; @@ -36,6 +38,7 @@ export default function CellControls(props: CellControlsProps) { + { document.body.removeChild(link); }; - return ; + return ( + + ); }; From 1e4ec40fa85b4ff6c707861193036a475116aa18 Mon Sep 17 00:00:00 2001 From: Erin Hoops Date: Thu, 13 Jun 2024 14:00:54 -0700 Subject: [PATCH 10/20] hide when 0 cells selected --- src/components/CellControls.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/CellControls.tsx b/src/components/CellControls.tsx index e9238713..9fe1964d 100644 --- a/src/components/CellControls.tsx +++ b/src/components/CellControls.tsx @@ -35,10 +35,10 @@ export default function CellControls(props: CellControlsProps) { {props.numSelectedCells ?? 0} cells selected + {!!props.numSelectedCells && } - Date: Fri, 14 Jun 2024 12:00:53 -0700 Subject: [PATCH 11/20] Account for tracks with length 1 --- src/components/App.tsx | 31 +++++++++++++------ src/lib/PointCanvas.ts | 2 +- src/lib/three/TrackGeometry.ts | 56 +++++++++++++++++++++++----------- 3 files changed, 61 insertions(+), 28 deletions(-) diff --git a/src/components/App.tsx b/src/components/App.tsx index 389bdb75..ac47385e 100644 --- a/src/components/App.tsx +++ b/src/components/App.tsx @@ -198,9 +198,16 @@ export default function App() { const getTrackDownloadData = () => { const trackData: TrackDownloadData[] = []; canvas.tracks.forEach((track, trackID) => { + // Keep track of the timepoints we've seen in this track to avoid duplication + // This is necessary because if a track contains a single point, we set + // the start and end positions to be the same + const timepointsInTrack = new Set(); + const startPositions = track.threeTrack.geometry.getAttribute("instanceStart"); const startTimes = track.threeTrack.geometry.getAttribute("instanceTimeStart"); - for (let i = 0; i < startPositions.count; i++) { + + for (let i = 0; i < startTimes.count; i++) { + timepointsInTrack.add(startTimes.getX(i)); trackData.push([ // trackID is 1-indexed in input and output CSVs trackID + 1, @@ -214,15 +221,19 @@ export default function App() { const endPositions = track.threeTrack.geometry.getAttribute("instanceEnd"); const endTimes = track.threeTrack.geometry.getAttribute("instanceTimeEnd"); const lastIndex = endPositions.count - 1; - trackData.push([ - // trackID is 1-indexed in input and output CSVs - trackID + 1, - Math.round(endTimes.getX(lastIndex)), - Math.round(endPositions.getX(lastIndex) * 1000) / 1000, // round to 3 decimal places - Math.round(endPositions.getY(lastIndex) * 1000) / 1000, - Math.round(endPositions.getZ(lastIndex) * 1000) / 1000, - track.parentTrackID, - ]); + + // Only add the end position if it's not the same as the start position + if (!timepointsInTrack.has(endTimes.getX(lastIndex))) { + trackData.push([ + // trackID is 1-indexed in input and output CSVs + trackID + 1, + endTimes.getX(lastIndex), + Math.round(endPositions.getX(lastIndex) * 1000) / 1000, // round to 3 decimal places + Math.round(endPositions.getY(lastIndex) * 1000) / 1000, + Math.round(endPositions.getZ(lastIndex) * 1000) / 1000, + track.parentTrackID, + ]); + } }); return trackData; }; diff --git a/src/lib/PointCanvas.ts b/src/lib/PointCanvas.ts index 1deb874a..7d080a60 100644 --- a/src/lib/PointCanvas.ts +++ b/src/lib/PointCanvas.ts @@ -235,7 +235,7 @@ export class PointCanvas { } const track = Track.new(positions, ids, this.maxPointsPerTimepoint); track.updateAppearance(this.showTracks, this.showTrackHighlights, this.minTime, this.maxTime); - this.tracks.set(trackID, { threeTrack: track, parentTrackID: parentTrackID }); // TODO: (Erin): - update here + this.tracks.set(trackID, { threeTrack: track, parentTrackID: parentTrackID }); this.scene.add(track); return track; } diff --git a/src/lib/three/TrackGeometry.ts b/src/lib/three/TrackGeometry.ts index d5fda153..8c757231 100644 --- a/src/lib/three/TrackGeometry.ts +++ b/src/lib/three/TrackGeometry.ts @@ -14,26 +14,38 @@ class TrackGeometry extends LineSegmentsGeometry { // converts [ x1, y1, z1, x2, y2, z2, ... ] to pairs format const length = array.length - 3; - const points = new Float32Array(2 * length); // start and end of each line - for (let i = 0; i < length; i += 3) { - // start point - points[2 * i] = array[i]; - points[2 * i + 1] = array[i + 1]; - points[2 * i + 2] = array[i + 2]; - // end point - points[2 * i + 3] = array[i + 3]; - points[2 * i + 4] = array[i + 4]; - points[2 * i + 5] = array[i + 5]; + // if array.length === 3, then this track has a single point + // in order to keep this point in our data, we are using the point as both the start and end + // Initialize the points array with a minimum of 6 to store a single point and larger if there + // are more points in the input array + const points = new Float32Array(Math.max(3, 2 * length)); // start and end of each line + + if (array.length === 3) { + console.log({ array }); + points[0] = array[0]; + points[1] = array[1]; + points[2] = array[2]; + points[3] = array[0]; + points[4] = array[1]; + points[5] = array[2]; + } else { + for (let i = 0; i < length; i += 3) { + // start point + points[2 * i] = array[i]; + points[2 * i + 1] = array[i + 1]; + points[2 * i + 2] = array[i + 2]; + // end point + points[2 * i + 3] = array[i + 3]; + points[2 * i + 4] = array[i + 4]; + points[2 * i + 5] = array[i + 5]; + } } - // TODO: (Erin) undo this to get the actual positions. start point for the first and end point for the rest or vice versa - super.setPositions(points); return this; } - // TODO: (Erin) getPositions() here setColors(array: number[] | Float32Array) { // converts [ r1, g1, b1, r2, g2, b2, ... ] to pairs format @@ -65,11 +77,21 @@ class TrackGeometry extends LineSegmentsGeometry { // float32 should be sufficient given we're expecting ~1000 timepoints const length = array.length - 1; - const times = new Float32Array(2 * length); - for (let i = 0; i < length; i++) { - times[2 * i] = array[i]; - times[2 * i + 1] = array[i + 1]; + // if array.length === 1, then this track has a single point + // in order to keep this point in our data, we are using the point as both the start and end + // Initialize the times array with a minimum of 2 to store a single point and larger if there + // are more times in the input array + const times = new Float32Array(Math.max(2, 2 * length)); + + if (array.length === 1) { + times[0] = array[0]; + times[1] = array[0]; + } else { + for (let i = 0; i < length; i++) { + times[2 * i] = array[i]; + times[2 * i + 1] = array[i + 1]; + } } const time = new InstancedInterleavedBuffer(times, 2, 1); From 77c65a383716c7c4f51aa2952a25c7f089d75614 Mon Sep 17 00:00:00 2001 From: Erin Hoops Date: Fri, 14 Jun 2024 12:02:20 -0700 Subject: [PATCH 12/20] Update headers --- src/components/DownloadButton.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/DownloadButton.tsx b/src/components/DownloadButton.tsx index f98cc272..4f1e14a7 100644 --- a/src/components/DownloadButton.tsx +++ b/src/components/DownloadButton.tsx @@ -7,7 +7,7 @@ interface DownloadButtonProps { getDownloadData: () => TrackDownloadData[]; } -const dataHeaders = ["trackID", "t", "x", "y", "z", "parentTrackID"]; +const dataHeaders = ["track_id", "t", "x", "y", "z", "parent_track_id"]; const convertToCSV = (nestedArray: (string | number)[][]) => { let csvString = ""; From 56a3f466f8cc1cf04efe0c20deac7e333a992a1c Mon Sep 17 00:00:00 2001 From: Ashley Anderson Date: Mon, 17 Jun 2024 11:24:52 -0400 Subject: [PATCH 13/20] Add argument parsing to data conversion script --- tools/convert_tracks_csv_to_sparse_zarr.py | 25 ++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/tools/convert_tracks_csv_to_sparse_zarr.py b/tools/convert_tracks_csv_to_sparse_zarr.py index a32dc98f..1d63d8d2 100644 --- a/tools/convert_tracks_csv_to_sparse_zarr.py +++ b/tools/convert_tracks_csv_to_sparse_zarr.py @@ -1,17 +1,34 @@ +import argparse import csv import time from collections import Counter +from pathlib import Path import numpy as np import zarr from scipy.sparse import lil_matrix -root_dir = "/Users/ehoops/development/" +parser = argparse.ArgumentParser(description="Convert a CSV of tracks to a sparse Zarr store") +parser.add_argument("csv_file", type=str, help="Path to the CSV file") +parser.add_argument( + "out_dir", + type=str, + help="Path to the output directory (optional, defaults to the parent dir of the CSV file)", + nargs="?", +) +args = parser.parse_args() + +csv_file = Path(args.csv_file) +if args.out_dir is None: + out_dir = csv_file.parent +else: + out_dir = Path(args.out_dir) +zarr_path = out_dir / f"{csv_file.stem}_bundle.zarr" start = time.monotonic() points = [] points_in_timepoint = Counter() -with open(root_dir + "ZSNS001_tracks.csv", "r") as f: +with open(csv_file, "r") as f: reader = csv.reader(f) next(reader) # Skip the header # TrackID,t,z,y,x,parent_track_id @@ -102,7 +119,7 @@ # save the points array (same format as ZSHS001_nodes.zarr) top_level_group = zarr.hierarchy.group( - zarr.storage.DirectoryStore(root_dir + "ZSNS001_tracks_bundle.zarr"), + zarr.storage.DirectoryStore(zarr_path.as_posix()), overwrite=True, ) @@ -177,4 +194,4 @@ # note the relatively small size of the indptr arrays # tracks_to_points/data is a redundant copy of the points array to avoid having -# to fetch point coordinates individually \ No newline at end of file +# to fetch point coordinates individually From cfc05567462d7aa2856b91e9ca5f959d9eeba835 Mon Sep 17 00:00:00 2001 From: Erin Hoops Date: Tue, 18 Jun 2024 14:28:50 -0700 Subject: [PATCH 14/20] convert to string before csv conversion --- src/components/App.tsx | 14 +++++++------- src/components/CellControls.tsx | 4 ++-- src/components/DownloadButton.tsx | 4 ++-- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/components/App.tsx b/src/components/App.tsx index ac47385e..2886e988 100644 --- a/src/components/App.tsx +++ b/src/components/App.tsx @@ -212,9 +212,9 @@ export default function App() { // trackID is 1-indexed in input and output CSVs trackID + 1, startTimes.getX(i), - Math.round(startPositions.getX(i) * 1000) / 1000, // round to 3 decimal places - Math.round(startPositions.getY(i) * 1000) / 1000, - Math.round(startPositions.getZ(i) * 1000) / 1000, + startPositions.getX(i), + startPositions.getY(i), + startPositions.getZ(i), track.parentTrackID, ]); } @@ -228,14 +228,14 @@ export default function App() { // trackID is 1-indexed in input and output CSVs trackID + 1, endTimes.getX(lastIndex), - Math.round(endPositions.getX(lastIndex) * 1000) / 1000, // round to 3 decimal places - Math.round(endPositions.getY(lastIndex) * 1000) / 1000, - Math.round(endPositions.getZ(lastIndex) * 1000) / 1000, + endPositions.getX(lastIndex), + endPositions.getY(lastIndex), + endPositions.getZ(lastIndex), track.parentTrackID, ]); } }); - return trackData; + return trackData.map((row) => row.map((entry) => entry.toFixed(3))); }; return ( diff --git a/src/components/CellControls.tsx b/src/components/CellControls.tsx index 9fe1964d..75ca95f4 100644 --- a/src/components/CellControls.tsx +++ b/src/components/CellControls.tsx @@ -4,11 +4,11 @@ import { FontS, SmallCapsButton, ControlLabel } from "@/components/Styled"; import { PointSelectionMode } from "@/lib/PointSelector"; import { TrackManager } from "@/lib/TrackManager"; -import { DownloadButton, TrackDownloadData } from "./DownloadButton"; +import { DownloadButton } from "./DownloadButton"; interface CellControlsProps { clearTracks: () => void; - getTrackDownloadData: () => TrackDownloadData[]; + getTrackDownloadData: () => string[][]; numSelectedCells?: number; trackManager: TrackManager | null; pointBrightness: number; diff --git a/src/components/DownloadButton.tsx b/src/components/DownloadButton.tsx index 4f1e14a7..4ff7fdaf 100644 --- a/src/components/DownloadButton.tsx +++ b/src/components/DownloadButton.tsx @@ -4,12 +4,12 @@ import { Button } from "@czi-sds/components"; // It contains trackID, time, x, y, z, parentTrackID export type TrackDownloadData = [number, number, number, number, number, number]; interface DownloadButtonProps { - getDownloadData: () => TrackDownloadData[]; + getDownloadData: () => string[][]; } const dataHeaders = ["track_id", "t", "x", "y", "z", "parent_track_id"]; -const convertToCSV = (nestedArray: (string | number)[][]) => { +const convertToCSV = (nestedArray: string[][]) => { let csvString = ""; for (let row = 0; row < nestedArray.length; row++) { From b906f1f702339f8829eb79ed2d2cc663896c573e Mon Sep 17 00:00:00 2001 From: Erin Hoops <109251328+ehoops-cz@users.noreply.github.com> Date: Tue, 18 Jun 2024 14:36:09 -0700 Subject: [PATCH 15/20] Update src/lib/three/TrackGeometry.ts min length of points should be 6 Co-authored-by: Andy Sweet --- src/lib/three/TrackGeometry.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/three/TrackGeometry.ts b/src/lib/three/TrackGeometry.ts index 8c757231..0adcf8d8 100644 --- a/src/lib/three/TrackGeometry.ts +++ b/src/lib/three/TrackGeometry.ts @@ -19,7 +19,7 @@ class TrackGeometry extends LineSegmentsGeometry { // in order to keep this point in our data, we are using the point as both the start and end // Initialize the points array with a minimum of 6 to store a single point and larger if there // are more points in the input array - const points = new Float32Array(Math.max(3, 2 * length)); // start and end of each line + const points = new Float32Array(Math.max(6, 2 * length)); // start and end of each line if (array.length === 3) { console.log({ array }); From d1f7a4271333d50f7fad4739760921444088b678 Mon Sep 17 00:00:00 2001 From: Erin Hoops Date: Tue, 18 Jun 2024 15:06:47 -0700 Subject: [PATCH 16/20] Address feedback --- src/components/App.tsx | 3 +- src/components/CellControls.tsx | 4 +-- src/components/DownloadButton.tsx | 14 ++++----- src/lib/ViewerState.ts | 7 ++--- src/lib/three/Track.ts | 9 ++++++ src/lib/three/TrackGeometry.ts | 52 ++++++++----------------------- 6 files changed, 36 insertions(+), 53 deletions(-) diff --git a/src/components/App.tsx b/src/components/App.tsx index 2886e988..c52ae546 100644 --- a/src/components/App.tsx +++ b/src/components/App.tsx @@ -235,7 +235,8 @@ export default function App() { ]); } }); - return trackData.map((row) => row.map((entry) => entry.toFixed(3))); + // Round to 3 decimal places, but turn back into a number to strip trailing zeros + return trackData.map((row) => row.map((entry) => parseFloat(entry.toFixed(3)))) as TrackDownloadData[]; }; return ( diff --git a/src/components/CellControls.tsx b/src/components/CellControls.tsx index 75ca95f4..9fe1964d 100644 --- a/src/components/CellControls.tsx +++ b/src/components/CellControls.tsx @@ -4,11 +4,11 @@ import { FontS, SmallCapsButton, ControlLabel } from "@/components/Styled"; import { PointSelectionMode } from "@/lib/PointSelector"; import { TrackManager } from "@/lib/TrackManager"; -import { DownloadButton } from "./DownloadButton"; +import { DownloadButton, TrackDownloadData } from "./DownloadButton"; interface CellControlsProps { clearTracks: () => void; - getTrackDownloadData: () => string[][]; + getTrackDownloadData: () => TrackDownloadData[]; numSelectedCells?: number; trackManager: TrackManager | null; pointBrightness: number; diff --git a/src/components/DownloadButton.tsx b/src/components/DownloadButton.tsx index 4ff7fdaf..2c454c13 100644 --- a/src/components/DownloadButton.tsx +++ b/src/components/DownloadButton.tsx @@ -4,23 +4,23 @@ import { Button } from "@czi-sds/components"; // It contains trackID, time, x, y, z, parentTrackID export type TrackDownloadData = [number, number, number, number, number, number]; interface DownloadButtonProps { - getDownloadData: () => string[][]; + getDownloadData: () => TrackDownloadData[]; } const dataHeaders = ["track_id", "t", "x", "y", "z", "parent_track_id"]; -const convertToCSV = (nestedArray: string[][]) => { +const convertToCSV = (nestedArray: TrackDownloadData[] | string[][]) => { let csvString = ""; - for (let row = 0; row < nestedArray.length; row++) { + nestedArray.forEach((row) => { let line = ""; - for (const entry in nestedArray[row]) { + row.forEach((entry) => { if (line !== "") line += ","; - line += nestedArray[row][entry]; - } + line += entry; + }); csvString += line + "\r\n"; - } + }); return csvString; }; diff --git a/src/lib/ViewerState.ts b/src/lib/ViewerState.ts index dc56a1ad..8fd63b40 100644 --- a/src/lib/ViewerState.ts +++ b/src/lib/ViewerState.ts @@ -1,7 +1,6 @@ -// export const DEFAULT_ZARR_URL = -// "https://sci-imaging-vis-public-demo-data.s3.us-west-2.amazonaws.com" + -// "/points-web-viewer/sparse-zarr-v2/ZSNS001_tracks_bundle.zarr"; -export const DEFAULT_ZARR_URL = "http://localhost:8000/ZSNS001_tracks_bundle.zarr"; +export const DEFAULT_ZARR_URL = + "https://sci-imaging-vis-public-demo-data.s3.us-west-2.amazonaws.com" + + "/points-web-viewer/sparse-zarr-v2/ZSNS001_tracks_bundle.zarr"; const HASH_KEY = "viewerState"; diff --git a/src/lib/three/Track.ts b/src/lib/three/Track.ts index aa6298df..acb17ba6 100644 --- a/src/lib/three/Track.ts +++ b/src/lib/three/Track.ts @@ -40,6 +40,15 @@ export class Track extends Mesh { // TODO: use a LUT for the main track, too colors.push(((0.9 * (n - i)) / n) ** 3, ((0.9 * (n - i)) / n) ** 3, (0.9 * (n - i)) / n); } + + // if this track has a single point, in order to keep this point in our data, + // we are using the point as both the start and end of the line segment + if (pos.length === 3) { + pos.push(pos[0], pos[1], pos[2]); + colors.push(colors[0], colors[1], colors[2]); + time.push(time[0]); + } + track.geometry.setPositions(pos); track.geometry.setColors(colors); track.geometry.setTime(time); diff --git a/src/lib/three/TrackGeometry.ts b/src/lib/three/TrackGeometry.ts index 0adcf8d8..b9eff7c6 100644 --- a/src/lib/three/TrackGeometry.ts +++ b/src/lib/three/TrackGeometry.ts @@ -14,32 +14,16 @@ class TrackGeometry extends LineSegmentsGeometry { // converts [ x1, y1, z1, x2, y2, z2, ... ] to pairs format const length = array.length - 3; + const points = new Float32Array(2 * length); - // if array.length === 3, then this track has a single point - // in order to keep this point in our data, we are using the point as both the start and end - // Initialize the points array with a minimum of 6 to store a single point and larger if there - // are more points in the input array - const points = new Float32Array(Math.max(6, 2 * length)); // start and end of each line - - if (array.length === 3) { - console.log({ array }); - points[0] = array[0]; - points[1] = array[1]; - points[2] = array[2]; - points[3] = array[0]; - points[4] = array[1]; - points[5] = array[2]; - } else { - for (let i = 0; i < length; i += 3) { - // start point - points[2 * i] = array[i]; - points[2 * i + 1] = array[i + 1]; - points[2 * i + 2] = array[i + 2]; - // end point - points[2 * i + 3] = array[i + 3]; - points[2 * i + 4] = array[i + 4]; - points[2 * i + 5] = array[i + 5]; - } + for (let i = 0; i < length; i += 3) { + points[2 * i] = array[i]; + points[2 * i + 1] = array[i + 1]; + points[2 * i + 2] = array[i + 2]; + + points[2 * i + 3] = array[i + 3]; + points[2 * i + 4] = array[i + 4]; + points[2 * i + 5] = array[i + 5]; } super.setPositions(points); @@ -77,21 +61,11 @@ class TrackGeometry extends LineSegmentsGeometry { // float32 should be sufficient given we're expecting ~1000 timepoints const length = array.length - 1; + const times = new Float32Array(2 * length); - // if array.length === 1, then this track has a single point - // in order to keep this point in our data, we are using the point as both the start and end - // Initialize the times array with a minimum of 2 to store a single point and larger if there - // are more times in the input array - const times = new Float32Array(Math.max(2, 2 * length)); - - if (array.length === 1) { - times[0] = array[0]; - times[1] = array[0]; - } else { - for (let i = 0; i < length; i++) { - times[2 * i] = array[i]; - times[2 * i + 1] = array[i + 1]; - } + for (let i = 0; i < length; i++) { + times[2 * i] = array[i]; + times[2 * i + 1] = array[i + 1]; } const time = new InstancedInterleavedBuffer(times, 2, 1); From fe3ef40234d078eac0153034c3e2e7be0a9f0ce8 Mon Sep 17 00:00:00 2001 From: Erin Hoops Date: Tue, 18 Jun 2024 15:13:53 -0700 Subject: [PATCH 17/20] Rename variable for clarity --- src/lib/PointCanvas.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/lib/PointCanvas.ts b/src/lib/PointCanvas.ts index 7d080a60..1bc35eb7 100644 --- a/src/lib/PointCanvas.ts +++ b/src/lib/PointCanvas.ts @@ -233,11 +233,11 @@ export class PointCanvas { console.warn("Track with ID %d already exists", trackID); return null; } - const track = Track.new(positions, ids, this.maxPointsPerTimepoint); - track.updateAppearance(this.showTracks, this.showTrackHighlights, this.minTime, this.maxTime); - this.tracks.set(trackID, { threeTrack: track, parentTrackID: parentTrackID }); - this.scene.add(track); - return track; + const threeTrack = Track.new(positions, ids, this.maxPointsPerTimepoint); + threeTrack.updateAppearance(this.showTracks, this.showTrackHighlights, this.minTime, this.maxTime); + this.tracks.set(trackID, { threeTrack, parentTrackID: parentTrackID }); + this.scene.add(threeTrack); + return threeTrack; } updateAllTrackHighlights() { From e099aa7d0a0e7df358bc9ad51b45ebdd242b0985 Mon Sep 17 00:00:00 2001 From: Erin Hoops <109251328+ehoops-cz@users.noreply.github.com> Date: Thu, 20 Jun 2024 08:54:18 -0700 Subject: [PATCH 18/20] Update src/components/DownloadButton.tsx simplify csv conversion function Co-authored-by: Andy Sweet --- src/components/DownloadButton.tsx | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/src/components/DownloadButton.tsx b/src/components/DownloadButton.tsx index 2c454c13..5d8f9849 100644 --- a/src/components/DownloadButton.tsx +++ b/src/components/DownloadButton.tsx @@ -10,18 +10,7 @@ interface DownloadButtonProps { const dataHeaders = ["track_id", "t", "x", "y", "z", "parent_track_id"]; const convertToCSV = (nestedArray: TrackDownloadData[] | string[][]) => { - let csvString = ""; - - nestedArray.forEach((row) => { - let line = ""; - row.forEach((entry) => { - if (line !== "") line += ","; - - line += entry; - }); - csvString += line + "\r\n"; - }); - return csvString; + return nestedArray.map((row) => row.join(",")).join("\r\n"); }; export const DownloadButton = (props: DownloadButtonProps) => { From c8a5d5f7b3d97cd684fd53977eb90b8d27be1fb9 Mon Sep 17 00:00:00 2001 From: Erin Hoops Date: Thu, 20 Jun 2024 12:15:11 -0700 Subject: [PATCH 19/20] fixes --- src/components/App.tsx | 7 +++++-- src/components/CellControls.tsx | 4 ++-- src/components/DownloadButton.tsx | 6 +++--- src/lib/PointCanvas.ts | 2 +- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/components/App.tsx b/src/components/App.tsx index c52ae546..eb083ad9 100644 --- a/src/components/App.tsx +++ b/src/components/App.tsx @@ -17,6 +17,7 @@ import LeftSidebarWrapper from "./leftSidebar/LeftSidebarWrapper"; import { TimestampOverlay } from "./overlays/TimestampOverlay"; import { ColorMap } from "./overlays/ColorMap"; import { TrackDownloadData } from "./DownloadButton"; +import { temp } from "three/examples/jsm/nodes/Nodes.js"; // Ideally we do this here so that we can use initial values as default values for React state. const initialViewerState = ViewerState.fromUrlHash(window.location.hash); @@ -235,8 +236,10 @@ export default function App() { ]); } }); - // Round to 3 decimal places, but turn back into a number to strip trailing zeros - return trackData.map((row) => row.map((entry) => parseFloat(entry.toFixed(3)))) as TrackDownloadData[]; + + // Round to 3 decimal places + const formatter = Intl.NumberFormat("en-US", { useGrouping: false }); + return trackData.map((row) => row.map(formatter.format)); }; return ( diff --git a/src/components/CellControls.tsx b/src/components/CellControls.tsx index 9fe1964d..75ca95f4 100644 --- a/src/components/CellControls.tsx +++ b/src/components/CellControls.tsx @@ -4,11 +4,11 @@ import { FontS, SmallCapsButton, ControlLabel } from "@/components/Styled"; import { PointSelectionMode } from "@/lib/PointSelector"; import { TrackManager } from "@/lib/TrackManager"; -import { DownloadButton, TrackDownloadData } from "./DownloadButton"; +import { DownloadButton } from "./DownloadButton"; interface CellControlsProps { clearTracks: () => void; - getTrackDownloadData: () => TrackDownloadData[]; + getTrackDownloadData: () => string[][]; numSelectedCells?: number; trackManager: TrackManager | null; pointBrightness: number; diff --git a/src/components/DownloadButton.tsx b/src/components/DownloadButton.tsx index 5d8f9849..07bdce14 100644 --- a/src/components/DownloadButton.tsx +++ b/src/components/DownloadButton.tsx @@ -4,19 +4,19 @@ import { Button } from "@czi-sds/components"; // It contains trackID, time, x, y, z, parentTrackID export type TrackDownloadData = [number, number, number, number, number, number]; interface DownloadButtonProps { - getDownloadData: () => TrackDownloadData[]; + getDownloadData: () => string[][]; } const dataHeaders = ["track_id", "t", "x", "y", "z", "parent_track_id"]; -const convertToCSV = (nestedArray: TrackDownloadData[] | string[][]) => { +const convertToCSV = (nestedArray: string[][]) => { return nestedArray.map((row) => row.join(",")).join("\r\n"); }; export const DownloadButton = (props: DownloadButtonProps) => { const downloadCSV = () => { const data = props.getDownloadData(); - const csvData = new Blob([`${convertToCSV([dataHeaders])}${convertToCSV(data)}`], { type: "text/csv" }); + const csvData = new Blob([`${convertToCSV([dataHeaders])}\r\n${convertToCSV(data)}`], { type: "text/csv" }); const csvURL = URL.createObjectURL(csvData); const link = document.createElement("a"); link.href = csvURL; diff --git a/src/lib/PointCanvas.ts b/src/lib/PointCanvas.ts index 1bc35eb7..0669e9c5 100644 --- a/src/lib/PointCanvas.ts +++ b/src/lib/PointCanvas.ts @@ -235,7 +235,7 @@ export class PointCanvas { } const threeTrack = Track.new(positions, ids, this.maxPointsPerTimepoint); threeTrack.updateAppearance(this.showTracks, this.showTrackHighlights, this.minTime, this.maxTime); - this.tracks.set(trackID, { threeTrack, parentTrackID: parentTrackID }); + this.tracks.set(trackID, { threeTrack, parentTrackID }); this.scene.add(threeTrack); return threeTrack; } From 34d7d37a1cd9a1f64e4057b4e6b0331f5c0f4c45 Mon Sep 17 00:00:00 2001 From: Ashley Anderson Date: Thu, 20 Jun 2024 15:37:38 -0400 Subject: [PATCH 20/20] Update src/components/App.tsx --- src/components/App.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/src/components/App.tsx b/src/components/App.tsx index eb083ad9..e0d04aaa 100644 --- a/src/components/App.tsx +++ b/src/components/App.tsx @@ -17,7 +17,6 @@ import LeftSidebarWrapper from "./leftSidebar/LeftSidebarWrapper"; import { TimestampOverlay } from "./overlays/TimestampOverlay"; import { ColorMap } from "./overlays/ColorMap"; import { TrackDownloadData } from "./DownloadButton"; -import { temp } from "three/examples/jsm/nodes/Nodes.js"; // Ideally we do this here so that we can use initial values as default values for React state. const initialViewerState = ViewerState.fromUrlHash(window.location.hash);