diff --git a/apps/web/.claude/VISION_DOCK_INTEGRATION_PLAN.md b/apps/web/.claude/VISION_DOCK_INTEGRATION_PLAN.md new file mode 100644 index 00000000..d8b94cfb --- /dev/null +++ b/apps/web/.claude/VISION_DOCK_INTEGRATION_PLAN.md @@ -0,0 +1,335 @@ +# Plan: Abacus Vision as Docked Abacus Video Source + +**Status:** In Progress +**Created:** 2026-01-01 +**Last Updated:** 2026-01-01 + +## Overview + +Transform abacus vision from a standalone modal into an alternate "source" for the docked abacus. When vision is enabled, the processed camera feed replaces the SVG abacus representation in the dock. + +**Current Architecture:** + +``` +AbacusDock → MyAbacus (SVG) → value displayed +AbacusVisionBridge → Modal → onValueDetected callback +``` + +**Target Architecture:** + +``` +AbacusDock + ├─ [vision disabled] → MyAbacus (SVG) + └─ [vision enabled] → VisionFeed (processed video) + value detection + ↓ + Broadcasts to observers +``` + +--- + +## Key Requirements + +1. **Vision hint on docks** - Camera icon visible on/near AbacusDock +2. **Persistent across docking** - Vision icon stays visible when abacus is docked +3. **Setup gating** - Clicking opens setup if no source/calibration configured +4. **Video replaces SVG** - When enabled, camera feed shows instead of SVG abacus +5. **Observer visibility** - Teachers/parents see student's video feed during observation + +--- + +## Progress Tracker + +- [ ] **Phase 1:** Vision State in MyAbacusContext +- [ ] **Phase 2:** Vision Indicator on AbacusDock +- [ ] **Phase 3:** Video Feed Replaces Docked Abacus +- [ ] **Phase 4:** Vision Setup Modal Refactor +- [ ] **Phase 5:** Broadcast Video Feed to Observers +- [ ] **Phase 6:** Polish & Edge Cases + +--- + +## Phase 1: Vision State in MyAbacusContext + +**Goal:** Add vision-related state to the abacus context so it's globally accessible. + +**Files to modify:** + +- `apps/web/src/contexts/MyAbacusContext.tsx` + +**State to add:** + +```typescript +interface VisionConfig { + enabled: boolean + cameraDeviceId: string | null + calibration: CalibrationGrid | null + remoteCameraSessionId: string | null // For phone camera +} + +// In context: +visionConfig: VisionConfig +setVisionEnabled: (enabled: boolean) => void +setVisionCalibration: (calibration: CalibrationGrid | null) => void +setVisionCamera: (deviceId: string | null) => void +isVisionSetupComplete: boolean // Derived: has camera AND calibration +``` + +**Persistence:** Save to localStorage alongside existing abacus display config. + +**Testable outcome:** + +- Open browser console, check that vision config is in context +- Toggle vision state programmatically, see it persist across refresh + +--- + +## Phase 2: Vision Indicator on AbacusDock + +**Goal:** Show a camera icon near the dock that indicates vision status and opens setup. + +**Files to modify:** + +- `apps/web/src/components/AbacusDock.tsx` - Add vision indicator +- `apps/web/src/components/MyAbacus.tsx` - Show indicator when docked + +**UI Design:** + +``` +┌─────────────────────────────────┐ +│ [Docked Abacus] [↗] │ ← Undock button (existing) +│ │ +│ [📷] │ ← Vision toggle (NEW) +│ │ +└─────────────────────────────────┘ +``` + +**Behavior:** + +- Icon shows camera with status indicator: + - 🔴 Red dot = not configured + - 🟢 Green dot = configured and enabled + - ⚪ No dot = configured but disabled +- Click opens VisionSetupModal (Phase 4) +- Visible in BOTH floating button AND docked states + +**Testable outcome:** + +- See camera icon on docked abacus +- Click icon, see setup modal open +- Icon shows different states based on config + +--- + +## Phase 3: Video Feed Replaces Docked Abacus + +**Goal:** When vision is enabled, render processed video instead of SVG abacus. + +**Files to modify:** + +- `apps/web/src/components/MyAbacus.tsx` - Conditional rendering +- Create: `apps/web/src/components/vision/DockedVisionFeed.tsx` + +**DockedVisionFeed component:** + +```typescript +interface DockedVisionFeedProps { + width: number; + height: number; + onValueDetected: (value: number) => void; +} + +// Renders: +// - Processed/cropped camera feed +// - Overlays detected column values +// - Small "disable vision" button +``` + +**MyAbacus docked mode change:** + +```tsx +// In docked rendering section: +{isDocked && ( + visionConfig.enabled && isVisionSetupComplete ? ( + + ) : ( + + ) +)} +``` + +**Testable outcome:** + +- Enable vision (manually set in console if needed) +- See video feed in dock instead of SVG abacus +- Detected values update the context + +--- + +## Phase 4: Vision Setup Modal Refactor + +**Goal:** Streamline the setup flow - AbacusVisionBridge becomes a setup wizard. + +**Files to modify:** + +- `apps/web/src/components/vision/AbacusVisionBridge.tsx` - Simplify to setup-only +- Create: `apps/web/src/components/vision/VisionSetupModal.tsx` + +**Setup flow:** + +``` +[Open Modal] + ↓ +Is camera selected? ─No──→ [Select Camera Screen] + │Yes ↓ + ↓ Select device +Is calibrated? ─No───→ [Calibration Screen] + │Yes ↓ + ↓ Manual or ArUco +[Ready Screen] + ├─ Preview of what vision sees + ├─ [Enable Vision] button + └─ [Reconfigure] button +``` + +**Quick-toggle behavior:** + +- If fully configured: clicking vision icon toggles on/off immediately +- If not configured: opens setup modal +- Long-press or secondary click: always opens settings + +**Testable outcome:** + +- Complete setup flow from scratch +- Settings persist across refresh +- Quick toggle works when configured + +--- + +## Phase 5: Broadcast Video Feed to Observers + +**Goal:** Teachers/parents observing a session see the student's vision video feed. + +**Files to modify:** + +- `apps/web/src/hooks/useSessionBroadcast.ts` - Add vision frame broadcasting +- `apps/web/src/hooks/useSessionObserver.ts` - Receive vision frames +- `apps/web/src/components/classroom/SessionObserverView.tsx` - Display vision feed + +**Broadcasting strategy:** + +```typescript +// In useSessionBroadcast, when vision is enabled: +// Emit compressed frames at reduced rate (5 fps for bandwidth) + +socket.emit("vision-frame", { + sessionId, + imageData: compressedJpegBase64, + timestamp: Date.now(), + detectedValue: currentValue, +}); + +// Also broadcast vision state: +socket.emit("practice-state", { + ...existingState, + visionEnabled: true, + visionConfidence: confidence, +}); +``` + +**Observer display:** + +```typescript +// In SessionObserverView, when student has vision enabled: +// Show video feed instead of SVG abacus in the observation panel + +{studentState.visionEnabled ? ( + +) : ( + +)} +``` + +**Testable outcome:** + +- Student enables vision, starts practice +- Teacher opens observer modal +- Teacher sees student's camera feed (not SVG abacus) + +--- + +## Phase 6: Polish & Edge Cases + +**Goal:** Handle edge cases and improve UX. + +**Items:** + +1. **Connection loss handling** - Fall back to SVG if video stops +2. **Bandwidth management** - Adaptive quality based on connection +3. **Mobile optimization** - Vision setup works on phone screens +4. **Reconnection** - Re-establish vision feed after disconnect +5. **Multiple observers** - Efficient multicast of video frames + +**Testable outcome:** + +- Disconnect/reconnect scenarios work smoothly +- Mobile users can configure vision +- Multiple teachers can observe same student + +--- + +## Implementation Order & Dependencies + +``` +Phase 1 (Foundation) + ↓ +Phase 2 (UI Integration) + ↓ +Phase 3 (Core Feature) ←── Requires Phase 1, 2 + ↓ +Phase 4 (UX Refinement) ←── Can start in parallel with Phase 3 + ↓ +Phase 5 (Observation) ←── Requires Phase 3 + ↓ +Phase 6 (Polish) ←── After all features work +``` + +--- + +## Files Summary + +### Modify + +| File | Changes | +| ---------------------------------------------- | --------------------------------------------- | +| `contexts/MyAbacusContext.tsx` | Add vision state, persistence | +| `components/MyAbacus.tsx` | Vision indicator, conditional video rendering | +| `components/AbacusDock.tsx` | Pass through vision-related props | +| `hooks/useSessionBroadcast.ts` | Emit vision frames | +| `hooks/useSessionObserver.ts` | Receive vision frames | +| `components/classroom/SessionObserverView.tsx` | Display vision feed | + +### Create + +| File | Purpose | +| ------------------------------------------ | ----------------------------- | +| `components/vision/VisionSetupModal.tsx` | Streamlined setup wizard | +| `components/vision/DockedVisionFeed.tsx` | Video display for docked mode | +| `components/vision/VisionIndicator.tsx` | Camera icon with status | +| `components/vision/ObserverVisionFeed.tsx` | Observer-side video display | + +--- + +## Testing Checkpoints + +After each phase, manually verify: + +- [ ] **Phase 1:** Console shows vision config in context, persists on refresh +- [ ] **Phase 2:** Camera icon visible on dock, opens modal on click +- [ ] **Phase 3:** Enable vision → video shows in dock instead of SVG +- [ ] **Phase 4:** Full setup flow works, quick toggle works when configured +- [ ] **Phase 5:** Observer sees student's video feed during session +- [ ] **Phase 6:** Edge cases handled gracefully diff --git a/apps/web/.gitignore b/apps/web/.gitignore index bf5cba95..a47682a3 100644 --- a/apps/web/.gitignore +++ b/apps/web/.gitignore @@ -54,3 +54,12 @@ src/generated/build-info.json # biome .biome + +# Python virtual environments +.venv*/ + +# User uploads +data/uploads/ + +# ML training data +training-data/ diff --git a/apps/web/src/components/vision/VisionCameraControls.stories.tsx b/apps/web/src/components/vision/VisionCameraControls.stories.tsx new file mode 100644 index 00000000..da9ca8b0 --- /dev/null +++ b/apps/web/src/components/vision/VisionCameraControls.stories.tsx @@ -0,0 +1,675 @@ +import type { Meta, StoryObj } from '@storybook/react' +import { useState } from 'react' +import { css } from '../../../styled-system/css' + +// ============================================================================= +// Mock Types (matching the real implementation) +// ============================================================================= + +interface MockDevice { + deviceId: string + label: string +} + +type CameraSource = 'local' | 'phone' + +// ============================================================================= +// Mock Camera Controls Component +// ============================================================================= + +interface VisionCameraControlsProps { + /** Camera source: local or phone */ + cameraSource: CameraSource + /** Available camera devices */ + availableDevices: MockDevice[] + /** Currently selected device ID */ + selectedDeviceId: string | null + /** Whether torch is available */ + isTorchAvailable: boolean + /** Whether torch is on */ + isTorchOn: boolean + /** Current facing mode */ + facingMode: 'user' | 'environment' + /** Whether phone is connected (for remote camera) */ + isPhoneConnected?: boolean + /** Remote torch available */ + remoteTorchAvailable?: boolean + /** Remote torch on */ + remoteTorchOn?: boolean + /** Callback when camera is selected */ + onCameraSelect?: (deviceId: string) => void + /** Callback when camera is flipped */ + onFlipCamera?: () => void + /** Callback when torch is toggled */ + onToggleTorch?: () => void + /** Callback when camera source changes */ + onCameraSourceChange?: (source: CameraSource) => void +} + +/** + * VisionCameraControls - UI for camera selection, torch control, and camera source toggle + * + * This component demonstrates the unified camera controls that appear in AbacusVisionBridge: + * - Camera selector dropdown (always visible for local camera, even with 1 device) + * - Flip camera button (only when multiple cameras available) + * - Unified torch button (works for both local and remote cameras) + * - Camera source toggle (local vs phone) + */ +function VisionCameraControls({ + cameraSource, + availableDevices, + selectedDeviceId, + isTorchAvailable, + isTorchOn, + facingMode, + isPhoneConnected = false, + remoteTorchAvailable = false, + remoteTorchOn = false, + onCameraSelect, + onFlipCamera, + onToggleTorch, + onCameraSourceChange, +}: VisionCameraControlsProps) { + // Determine if torch button should show + const showTorchButton = + (cameraSource === 'local' && isTorchAvailable) || + (cameraSource === 'phone' && isPhoneConnected && remoteTorchAvailable) + + // Get current torch state based on source + const currentTorchOn = cameraSource === 'local' ? isTorchOn : remoteTorchOn + + return ( +
+ {/* Header */} +
+ 📷 + Camera Controls +
+ + {/* Camera source selector */} +
+ Source: + + +
+ + {/* Camera controls - unified for both local and phone */} +
+ {/* Camera selector - always show for local camera (even with 1 device) */} + {cameraSource === 'local' && availableDevices.length > 0 && ( + + )} + + {/* Flip camera button - only show if multiple cameras available */} + {cameraSource === 'local' && availableDevices.length > 1 && ( + + )} + + {/* Torch toggle button - unified for both local and remote */} + {showTorchButton && ( + + )} + + {/* Phone status when using phone camera */} + {cameraSource === 'phone' && ( +
+ + {isPhoneConnected ? 'Phone Connected' : 'Waiting for phone...'} +
+ )} +
+ + {/* Info about wide-angle preference */} +
+ 📐 + Cameras default to widest angle lens (zoom: 1) +
+
+ ) +} + +// ============================================================================= +// Storybook Meta +// ============================================================================= + +const meta: Meta = { + title: 'Vision/VisionCameraControls', + component: VisionCameraControls, + decorators: [ + (Story) => ( +
+ +
+ ), + ], + parameters: { + layout: 'centered', + backgrounds: { default: 'dark' }, + }, +} + +export default meta +type Story = StoryObj + +// ============================================================================= +// Mock Data +// ============================================================================= + +const singleCamera: MockDevice[] = [{ deviceId: 'camera-1', label: 'FaceTime HD Camera' }] + +const multipleCameras: MockDevice[] = [ + { deviceId: 'camera-1', label: 'FaceTime HD Camera' }, + { deviceId: 'camera-2', label: 'iPhone Continuity Camera (Wide)' }, + { deviceId: 'camera-3', label: 'iPhone Continuity Camera (Ultra Wide)' }, + { deviceId: 'camera-4', label: 'Desk View Camera' }, +] + +const iphoneCameras: MockDevice[] = [ + { deviceId: 'camera-wide', label: 'Back Camera (Wide)' }, + { deviceId: 'camera-ultrawide', label: 'Back Camera (Ultra Wide)' }, + { deviceId: 'camera-front', label: 'Front Camera' }, +] + +// ============================================================================= +// Stories: Single Camera +// ============================================================================= + +export const SingleCameraNoTorch: Story = { + name: 'Single Camera - No Torch', + args: { + cameraSource: 'local', + availableDevices: singleCamera, + selectedDeviceId: 'camera-1', + isTorchAvailable: false, + isTorchOn: false, + facingMode: 'environment', + }, +} + +export const SingleCameraWithTorch: Story = { + name: 'Single Camera - With Torch', + args: { + cameraSource: 'local', + availableDevices: singleCamera, + selectedDeviceId: 'camera-1', + isTorchAvailable: true, + isTorchOn: false, + facingMode: 'environment', + }, +} + +export const SingleCameraTorchOn: Story = { + name: 'Single Camera - Torch On', + args: { + cameraSource: 'local', + availableDevices: singleCamera, + selectedDeviceId: 'camera-1', + isTorchAvailable: true, + isTorchOn: true, + facingMode: 'environment', + }, +} + +// ============================================================================= +// Stories: Multiple Cameras +// ============================================================================= + +export const MultipleCameras: Story = { + name: 'Multiple Cameras - Desktop', + args: { + cameraSource: 'local', + availableDevices: multipleCameras, + selectedDeviceId: 'camera-2', + isTorchAvailable: true, + isTorchOn: false, + facingMode: 'environment', + }, +} + +export const MultipleCamerasUltraWideSelected: Story = { + name: 'Multiple Cameras - Ultra Wide Selected', + args: { + cameraSource: 'local', + availableDevices: multipleCameras, + selectedDeviceId: 'camera-3', + isTorchAvailable: true, + isTorchOn: false, + facingMode: 'environment', + }, +} + +// ============================================================================= +// Stories: Phone Camera +// ============================================================================= + +export const PhoneCameraWaiting: Story = { + name: 'Phone Camera - Waiting for Connection', + args: { + cameraSource: 'phone', + availableDevices: [], + selectedDeviceId: null, + isTorchAvailable: false, + isTorchOn: false, + facingMode: 'environment', + isPhoneConnected: false, + remoteTorchAvailable: false, + remoteTorchOn: false, + }, +} + +export const PhoneCameraConnected: Story = { + name: 'Phone Camera - Connected', + args: { + cameraSource: 'phone', + availableDevices: [], + selectedDeviceId: null, + isTorchAvailable: false, + isTorchOn: false, + facingMode: 'environment', + isPhoneConnected: true, + remoteTorchAvailable: true, + remoteTorchOn: false, + }, +} + +export const PhoneCameraTorchOn: Story = { + name: 'Phone Camera - Torch On', + args: { + cameraSource: 'phone', + availableDevices: [], + selectedDeviceId: null, + isTorchAvailable: false, + isTorchOn: false, + facingMode: 'environment', + isPhoneConnected: true, + remoteTorchAvailable: true, + remoteTorchOn: true, + }, +} + +// ============================================================================= +// Stories: Interactive +// ============================================================================= + +function InteractiveCameraControls() { + const [cameraSource, setCameraSource] = useState('local') + const [selectedDeviceId, setSelectedDeviceId] = useState('camera-2') + const [isTorchOn, setIsTorchOn] = useState(false) + const [remoteTorchOn, setRemoteTorchOn] = useState(false) + const [facingMode, setFacingMode] = useState<'user' | 'environment'>('environment') + + return ( + setFacingMode((m) => (m === 'user' ? 'environment' : 'user'))} + onToggleTorch={() => { + if (cameraSource === 'local') { + setIsTorchOn((t) => !t) + } else { + setRemoteTorchOn((t) => !t) + } + }} + /> + ) +} + +export const Interactive: Story = { + name: 'Interactive Demo', + render: () => , +} + +// ============================================================================= +// Stories: Feature Showcase +// ============================================================================= + +export const FeatureShowcase: Story = { + name: 'Feature Showcase - All Features', + render: () => ( +
+
+

+ Camera Control Features +

+ +
+
+

+ 1. Camera Selector Always Visible +

+

+ The camera dropdown now shows even with just 1 camera, so you can always see which + device is selected. +

+ +
+ +
+

+ 2. Unified Torch Control +

+

+ The torch button works for both local and remote cameras. It automatically controls + the active camera's flash. +

+
+ + +
+
+ +
+

+ 3. Wide-Angle Lens Preference +

+

+ Cameras default to their widest field of view using{' '} + + zoom: { ideal: 1 } + {' '} + constraint, ensuring you capture the full abacus. +

+ +
+
+
+
+ ), +} + +// ============================================================================= +// Stories: Comparison +// ============================================================================= + +export const LocalVsPhoneComparison: Story = { + name: 'Local vs Phone Camera Comparison', + render: () => ( +
+
+ + Local Camera (Multiple) + + +
+ +
+ + Phone Camera (Connected) + + +
+
+ ), +}