Skip to content

Commit

Permalink
Mobile device accessibility (#121)
Browse files Browse the repository at this point in the history 
* on mobile device, only third selection mode is allowed

* typo

* added alert to check if mobile is detected

* iPad wasnt detected

* iPad stil not detected

* initialize sphere selector in middle of point cloud

* added select-cells button when on Mobile device, to place shift+click"

* changed sections in leftSideBar

* renamed tooltips + cleanup

* detect desktop/tablet/phone

* removed device alert

* scale canvas to screen (tablet/phone/desktop) to prevent the playBackControls to be hidden

* added margins for phone/tablet

* corrected typos

* clean up

* don't reduce point brightness when no cells are selected

* remove spherical selector when no mode is selected

* disable any selector on phone

* on iPad: only show the select cells button if the selection mode is actually selected. The button disappears when the user deselects any mode

* selection info box disappears when no mode is selected

* center the select cells button on

* disable timeStamp on Canvas, since it is already on top of the slider

* lint fixes

* added slider to iPad configuration, to allow changing selector size with UI

* fixed bug in selector scale slider

* updated ControlInstructions syntax

* added conditional iPad statement to the ConrolInstructions

* minor textual change

* fixed lint issue

* popup on phone

* changed phone popup from alert to confirm

* change to popup

* changed data path

* keep preview when setting brightness or changing the selector size on iPad

* fixed that DataControls stay on pays and dont scroll
  • Loading branch information
@TeunHuijben
TeunHuijben authored Oct 8, 2024
1 parent 91ca28f commit ec602b1
Show file tree
Hide file tree
Showing 11 changed files with 374 additions and 138 deletions.
277 changes: 177 additions & 100 deletions src/components/App.tsx
View file

Large diffs are not rendered by default.

61 changes: 54 additions & 7 deletions src/components/CellControls.tsx
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { Box, Stack } from "@mui/material";
import { InputSlider, SegmentedControl, SingleButtonDefinition } from "@czi-sds/components";
import { InputSlider, SegmentedControl, SingleButtonDefinition, Button } from "@czi-sds/components";
import { FontS, SmallCapsButton, ControlLabel } from "@/components/Styled";

import { PointSelectionMode } from "@/lib/PointSelector";
Expand All @@ -16,17 +16,35 @@ interface CellControlsProps {
setPointBrightness: (value: number) => void;
pointSize: number;
setPointSize: (value: number) => void;
selectionMode: PointSelectionMode;
selectionMode: PointSelectionMode | null;
setSelectionMode: (value: PointSelectionMode) => void;
isTablet: boolean;
MobileSelectCells: () => void;
setSelectorScale: (value: number) => void;
selectorScale: number;
}

export default function CellControls(props: CellControlsProps) {
const buttonDefinition: SingleButtonDefinition[] = [
{ icon: "Cube", tooltipText: "Box", value: PointSelectionMode.BOX },
{ icon: "Starburst", tooltipText: "Spherical cursor", value: PointSelectionMode.SPHERICAL_CURSOR },
{ icon: "Globe", tooltipText: "Sphere", value: PointSelectionMode.SPHERE },
{ icon: "Starburst", tooltipText: "Sphere", value: PointSelectionMode.SPHERICAL_CURSOR },
{ icon: "Globe", tooltipText: "Adjustable sphere", value: PointSelectionMode.SPHERE },
];

// Intercept onChange of selection buttons to prevent the first two buttons from being selected on mobile devices
const handleSegmentedControlChange = (_e: React.MouseEvent<HTMLElement>, newValue: PointSelectionMode | null) => {
// If isTablet is true and the selected value corresponds to the first or second button, do nothing
if (
props.isTablet &&
(newValue === PointSelectionMode.BOX || newValue === PointSelectionMode.SPHERICAL_CURSOR)
) {
window.alert("This selection mode is not available on mobile devices.");
console.log("Mobile device detected, preventing selection of box or spherical cursor");
return; // Prevent selection
}
props.setSelectionMode(newValue!); // Otherwise, update the selection mode
};

return (
<Stack spacing="1em">
<Box display="flex" flexDirection="row" alignItems="center" justifyContent="space-between">
Expand All @@ -49,12 +67,41 @@ export default function CellControls(props: CellControlsProps) {
<SegmentedControl
id="selection-mode-control"
buttonDefinition={buttonDefinition}
onChange={(_e, v) => {
props.setSelectionMode(v);
}}
onChange={handleSegmentedControlChange}
value={props.selectionMode}
/>
</Box>
<Box display="flex" justifyContent="center" alignItems="center">
{props.isTablet && props.selectionMode === PointSelectionMode.SPHERE && (
<Button sdsStyle="square" sdsType="primary" onClick={props.MobileSelectCells}>
Select cells
</Button>
)}
</Box>
{(props.selectionMode === PointSelectionMode.SPHERICAL_CURSOR ||
props.selectionMode === PointSelectionMode.SPHERE) &&
props.isTablet && (
<>
<div style={{ display: "flex", justifyContent: "center", alignItems: "center" }}>
<label htmlFor="selector-radius-slider">
<FontS id="input-selector-radius-slider">selector radius:</FontS>
</label>
</div>
<InputSlider
id="selector-radius-slider"
aria-labelledby="input-selector-radius-slider"
min={0.5}
max={5}
step={0.1}
// valueLabelDisplay="on"
valueLabelFormat={(value) => `${value}`}
onChange={(_, value) => {
props.setSelectorScale(value as number);
}}
value={props.selectorScale}
/>
</>
)}
{numberOfValuesPerPoint !== 4 && (
<>
<label htmlFor="points-sizes-slider">
Expand Down
57 changes: 44 additions & 13 deletions src/components/leftSidebar/ControlInstructions.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@ import { Callout } from "@czi-sds/components";
import { PointSelectionMode } from "@/lib/PointSelector";

interface ControlInstructionsProps {
selectionMode: PointSelectionMode;
selectionMode: PointSelectionMode | null;
isTablet: boolean;
}

export default function ControlInstructions(props: ControlInstructionsProps) {
Expand All @@ -15,24 +16,54 @@ export default function ControlInstructions(props: ControlInstructionsProps) {
case PointSelectionMode.SPHERICAL_CURSOR:
instructionText = (
<>
<p>Hold Shift and a sphere will follow your cursor, centering on nearby points.</p>
<p>Shift-Click to select cells within the sphere.</p>
<p>Additional controls:</p>
<p>Ctrl+scroll: scale sphere</p>
<p>s: show/hide sphere</p>
<p>
Hold <strong>Shift</strong> and the selector will follow your cursor.
</p>
<p>
<strong>Shift-Click</strong> to select cells within the sphere.
</p>
<em>Additional controls:</em>
<ul style={{ paddingLeft: "15px", marginLeft: "0" }}>
<li>
<strong>ctrl+scroll</strong>: scale sphere
</li>
<li>
<strong>s</strong>: show/hide sphere
</li>
</ul>
</>
);
break;
case PointSelectionMode.SPHERE:
instructionText = (
<>
<p>Shift-click to select cells within the sphere.</p>
<p>Additional controls:</p>
<p>w: position mode</p>
<p>e: rotation mode</p>
<p>r: scale mode</p>
<p>Ctrl+scroll: scale</p>
<p>s: show/hide sphere</p>
{props.isTablet && (
<p>
If using a tablet without keyboard, use the UI controls above to select cells and scale the
selector (use keyboard for full functionality)
</p>
)}
<p>
<strong>Shift-click</strong> to select cells within the selector.
</p>
<em>Additional controls:</em>
<ul style={{ paddingLeft: "15px", marginLeft: "0" }}>
<li>
<strong>w</strong>: position
</li>
<li>
<strong>e</strong>: rotation
</li>
<li>
<strong>r</strong>: scale
</li>
<li>
<strong>Ctrl+scroll</strong>: scale
</li>
<li>
<strong>s</strong>: show/hide selector
</li>
</ul>
</>
);
break;
Expand Down
6 changes: 4 additions & 2 deletions src/components/leftSidebar/LeftSidebarWrapper.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,8 @@ interface LeftSidebarWrapperProps {
showTrackHighlights: boolean;
setShowTrackHighlights: (showTrackHighlights: boolean) => void;
setTrackHighlightLength: (trackHighlightLength: number) => void;
selectionMode: PointSelectionMode;
selectionMode: PointSelectionMode | null;
isTablet: boolean;
}

export default function LeftSidebarWrapper({
Expand All @@ -25,6 +26,7 @@ export default function LeftSidebarWrapper({
setShowTrackHighlights,
setTrackHighlightLength,
selectionMode,
isTablet,
}: LeftSidebarWrapperProps) {
return (
<>
Expand All @@ -39,7 +41,7 @@ export default function LeftSidebarWrapper({
setTrackHighlightLength={setTrackHighlightLength}
/>
)}
<ControlInstructions selectionMode={selectionMode} />
{selectionMode !== null && <ControlInstructions selectionMode={selectionMode} isTablet={isTablet} />}
</>
);
}
9 changes: 5 additions & 4 deletions src/components/leftSidebar/TrackControls.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import { TrackManager } from "@/lib/TrackManager";
import { InputSlider, InputToggle } from "@czi-sds/components";
import { Box, Stack } from "@mui/material";

import { ControlLabel } from "@/components/Styled";
import { ControlLabel, FontS } from "@/components/Styled";

interface TrackControlsProps {
trackManager: TrackManager | null;
Expand All @@ -19,9 +19,10 @@ export default function TrackControls(props: TrackControlsProps) {

return (
<Stack spacing={"2em"}>
<ControlLabel>Visualization options</ControlLabel>
<Box display="flex" flexDirection="row" alignItems="center" justifyContent="space-between">
<label htmlFor="show-tracks">
<ControlLabel>Tracks</ControlLabel>
<FontS>Tracks</FontS>
</label>
<InputToggle
id="show-tracks"
Expand All @@ -33,7 +34,7 @@ export default function TrackControls(props: TrackControlsProps) {
</Box>
<Box display="flex" flexDirection="row" alignItems="center" justifyContent="space-between">
<label htmlFor="show-track-highlights">
<ControlLabel>Track Highlights</ControlLabel>
<FontS>Track Highlights</FontS>
</label>
<Box>
<InputToggle
Expand All @@ -46,7 +47,7 @@ export default function TrackControls(props: TrackControlsProps) {
</Box>
</Box>
<label htmlFor="track-highlight-length-slider">
<ControlLabel>Track Highlight Length</ControlLabel>
<FontS>Track Highlight Length</FontS>
</label>
<InputSlider
id="track-highlight-length-slider"
Expand Down
2 changes: 1 addition & 1 deletion src/components/overlays/ColorMap.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ export const ColorMap = () => {
<Box
sx={{
position: "absolute",
bottom: "5.0rem",
bottom: "0.5rem",
right: "0.5rem",
width: "2.5rem",
height: "6.5rem",
Expand Down
2 changes: 1 addition & 1 deletion src/components/overlays/TimestampOverlay.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ export const TimestampOverlay = (props: TimestampOverlayProps) => {
size="small"
sx={{
position: "absolute",
bottom: "4.75rem",
bottom: "0.5rem",
left: "16.5rem",
backgroundColor: "rgba(255, 255, 255, 0.6)",
zIndex: 100,
Expand Down
34 changes: 29 additions & 5 deletions src/hooks/usePointCanvas.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ enum ActionType {
ADD_SELECTED_POINT_IDS = "ADD_SELECTED_POINT_IDS",
SHOW_PREVIEW_POINTS = "SHOW_PREVIEW_POINTS",
UPDATE_WITH_STATE = "UPDATE_WITH_STATE",
MOBILE_SELECT_CELLS = "MOBILE_SELECT_CELLS",
SELECTOR_SCALE = "SELECTOR_SCALE",
}

interface AutoRotate {
Expand Down Expand Up @@ -115,6 +117,15 @@ interface UpdateWithState {
state: ViewerState;
}

interface MobileSelectCells {
type: ActionType.MOBILE_SELECT_CELLS;
}

interface SelectorScale {
type: ActionType.SELECTOR_SCALE;
scale: number;
}

// setting up a tagged union for the actions
type PointCanvasAction =
| AutoRotate
Expand All @@ -134,7 +145,9 @@ type PointCanvasAction =
| MinMaxTime
| AddSelectedPointIds
| ShowPreviewPoints
| UpdateWithState;
| UpdateWithState
| MobileSelectCells
| SelectorScale;

function reducer(canvas: PointCanvas, action: PointCanvasAction): PointCanvas {
console.debug("usePointCanvas.reducer: ", action);
Expand Down Expand Up @@ -164,6 +177,7 @@ function reducer(canvas: PointCanvas, action: PointCanvasAction): PointCanvas {
newCanvas.pointBrightness = action.brightness;
newCanvas.resetPointColors();
newCanvas.updateSelectedPointIndices();
newCanvas.updatePreviewPoints();
break;
case ActionType.POINT_SIZES:
newCanvas.pointSize = action.pointSize;
Expand All @@ -188,7 +202,7 @@ function reducer(canvas: PointCanvas, action: PointCanvasAction): PointCanvas {
newCanvas.resetPointColors();
break;
case ActionType.SELECTION_MODE: {
const modeOld: PointSelectionMode = canvas.selector.selectionMode;
const modeOld: PointSelectionMode | null = canvas.selector.selectionMode;
const modeNew: PointSelectionMode = action.selectionMode;
newCanvas.setSelectionMode(action.selectionMode);

Expand All @@ -197,7 +211,7 @@ function reducer(canvas: PointCanvas, action: PointCanvasAction): PointCanvas {
newCanvas.resetPointColors();
newCanvas.highlightPoints(newCanvas.selectedPointIndices);
}
if (modeOld !== PointSelectionMode.BOX && modeNew == PointSelectionMode.BOX) {
if (modeOld !== PointSelectionMode.BOX && (modeNew == PointSelectionMode.BOX || modeNew == null)) {
newCanvas.resetPointColors();
newCanvas.highlightPoints(newCanvas.selectedPointIndices);
}
Expand All @@ -220,12 +234,15 @@ function reducer(canvas: PointCanvas, action: PointCanvasAction): PointCanvas {
newCanvas.updateAllTrackHighlights();
break;
case ActionType.ADD_SELECTED_POINT_IDS: {
newCanvas.pointBrightness = 0.8;
newCanvas.resetPointColors();
const newSelectedPointIds = new Set(canvas.selectedPointIds);
for (const trackId of action.selectedPointIds) {
newSelectedPointIds.add(trackId);
}
if (action.selectedPointIds.size !== 0) {
// only reduce pointBrightness if there are selected points
newCanvas.pointBrightness = 0.8;
}
newCanvas.resetPointColors();
newCanvas.selectedPointIds = newSelectedPointIds;
newCanvas.updateSelectedPointIndices();
newCanvas.highlightPoints(action.selectedPointIndices);
Expand All @@ -242,6 +259,13 @@ function reducer(canvas: PointCanvas, action: PointCanvasAction): PointCanvas {
case ActionType.UPDATE_WITH_STATE:
newCanvas.updateWithState(action.state);
break;
case ActionType.MOBILE_SELECT_CELLS:
newCanvas.MobileSelectCells();
break;
case ActionType.SELECTOR_SCALE:
newCanvas.setSelectorScale(action.scale);
newCanvas.updatePreviewPoints();
break;
default:
console.warn("usePointCanvas reducer - unknown action type: %s", action);
return canvas;
Expand Down
21 changes: 19 additions & 2 deletions src/lib/PointCanvas.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import { PointSelector, PointSelectionMode } from "@/lib/PointSelector";
import { ViewerState } from "./ViewerState";
import { numberOfValuesPerPoint } from "./TrackManager";

import { detectedDevice } from "@/components/App.tsx";
import config from "../../CONFIG.ts";
const initialPointSize = config.settings.point_size;
const pointColor = config.settings.point_color;
Expand Down Expand Up @@ -153,7 +154,13 @@ export class PointCanvas {

// Set up selection
this.selector = new PointSelector(this.scene, this.renderer, this.camera, this.controls, this.points);
this.setSelectionMode(PointSelectionMode.BOX);
if (detectedDevice.isTablet) {
this.setSelectionMode(PointSelectionMode.SPHERE);
} else if (detectedDevice.isPhone) {
this.setSelectionMode(null); // no selection functionality on phone
} else {
this.setSelectionMode(PointSelectionMode.BOX);
}
}

shallowCopy(): PointCanvas {
Expand Down Expand Up @@ -191,7 +198,7 @@ export class PointCanvas {
this.controls.target.fromArray(state.cameraTarget);
}

setSelectionMode(mode: PointSelectionMode) {
setSelectionMode(mode: PointSelectionMode | null) {
this.selector.setSelectionMode(mode);
}

Expand Down Expand Up @@ -424,4 +431,14 @@ export class PointCanvas {
this.points.material.dispose();
}
}

MobileSelectCells() {
// if used on tablet, this will select the cells upon button click
this.selector.sphereSelector.MobileFindAndSelect();
}

setSelectorScale(scale: number) {
// on tablet: this will set the size of the sphere selector upon the user using the slider
this.selector.sphereSelector.cursor.scale.set(scale, scale, scale);
}
}
Loading

0 comments on commit ec602b1

Please sign in to comment.