diff --git a/apps/web/src/app/practice/[studentId]/PracticeClient.tsx b/apps/web/src/app/practice/[studentId]/PracticeClient.tsx index 1677839d..77353d7f 100644 --- a/apps/web/src/app/practice/[studentId]/PracticeClient.tsx +++ b/apps/web/src/app/practice/[studentId]/PracticeClient.tsx @@ -1,8 +1,9 @@ 'use client' import { useRouter } from 'next/navigation' -import { useCallback, useMemo, useState } from 'react' +import { useCallback, useEffect, useMemo, useState } from 'react' import { useToast } from '@/components/common/ToastContext' +import { useMyAbacus } from '@/contexts/MyAbacusContext' import { PageWithNav } from '@/components/PageWithNav' import { ActiveSession, @@ -43,6 +44,7 @@ interface PracticeClientProps { export function PracticeClient({ studentId, player, initialSession }: PracticeClientProps) { const router = useRouter() const { showError } = useToast() + const { setVisionFrameCallback } = useMyAbacus() // Track pause state for HUD display (ActiveSession owns the modal and actual pause logic) const [isPaused, setIsPaused] = useState(false) @@ -168,7 +170,7 @@ export function PracticeClient({ studentId, player, initialSession }: PracticeCl // broadcastState is updated by ActiveSession via the onBroadcastStateChange callback // onAbacusControl receives control events from observing teacher // onTeacherPause/onTeacherResume receive pause/resume commands from teacher - const { sendPartTransition, sendPartTransitionComplete } = useSessionBroadcast( + const { sendPartTransition, sendPartTransitionComplete, sendVisionFrame } = useSessionBroadcast( currentPlan.id, studentId, broadcastState, @@ -179,6 +181,17 @@ export function PracticeClient({ studentId, player, initialSession }: PracticeCl } ) + // Wire vision frame callback to broadcast vision frames to observers + useEffect(() => { + setVisionFrameCallback((frame) => { + sendVisionFrame(frame.imageData, frame.detectedValue, frame.confidence) + }) + + return () => { + setVisionFrameCallback(null) + } + }, [setVisionFrameCallback, sendVisionFrame]) + // Build session HUD data for PracticeSubNav const sessionHud: SessionHudData | undefined = currentPart ? { diff --git a/apps/web/src/components/classroom/SessionObserverModal.tsx b/apps/web/src/components/classroom/SessionObserverModal.tsx index ddd80b0c..ec7b3d40 100644 --- a/apps/web/src/components/classroom/SessionObserverModal.tsx +++ b/apps/web/src/components/classroom/SessionObserverModal.tsx @@ -21,6 +21,7 @@ import { PracticeFeedback } from '../practice/PracticeFeedback' import { PurposeBadge } from '../practice/PurposeBadge' import { SessionProgressIndicator } from '../practice/SessionProgressIndicator' import { VerticalProblem } from '../practice/VerticalProblem' +import { ObserverVisionFeed } from '../vision/ObserverVisionFeed' interface SessionObserverModalProps { /** Whether the modal is open */ @@ -162,6 +163,7 @@ export function SessionObserverView({ state, results, transitionState, + visionFrame, isConnected, isObserving, error, @@ -756,15 +758,9 @@ export function SessionObserverView({ /> - {/* AbacusDock - positioned exactly like ActiveSession */} + {/* Vision feed or AbacusDock - positioned exactly like ActiveSession */} {state.phase === 'problem' && (problemHeight ?? 0) > 0 && ( - + > + {/* Show vision feed if available, otherwise show teacher's abacus dock */} + {visionFrame ? ( + + ) : ( + + )} + )} diff --git a/apps/web/src/components/vision/DockedVisionFeed.tsx b/apps/web/src/components/vision/DockedVisionFeed.tsx index 1b52ccec..9703096e 100644 --- a/apps/web/src/components/vision/DockedVisionFeed.tsx +++ b/apps/web/src/components/vision/DockedVisionFeed.tsx @@ -33,13 +33,15 @@ interface DockedVisionFeedProps { * - Shows the video feed with detection overlay */ export function DockedVisionFeed({ onValueDetected, columnCount = 5 }: DockedVisionFeedProps) { - const { visionConfig, setDockedValue, setVisionEnabled, setVisionCalibration } = useMyAbacus() + const { visionConfig, setDockedValue, setVisionEnabled, setVisionCalibration, emitVisionFrame } = useMyAbacus() const videoRef = useRef(null) const remoteImageRef = useRef(null) + const rectifiedCanvasRef = useRef(null) const animationFrameRef = useRef(null) const markerDetectionFrameRef = useRef(null) const lastInferenceTimeRef = useRef(0) + const lastBroadcastTimeRef = useRef(0) const [videoStream, setVideoStream] = useState(null) const [error, setError] = useState(null) @@ -336,6 +338,61 @@ export function DockedVisionFeed({ onValueDetected, columnCount = 5 }: DockedVis } }, [stability.stableValue, stability.currentConfidence, detectedValue, setDockedValue, onValueDetected]) + // Broadcast vision frames to observers (5fps to save bandwidth) + const BROADCAST_INTERVAL_MS = 200 + useEffect(() => { + if (!visionConfig.enabled) return + + let running = true + + const broadcastLoop = () => { + if (!running) return + + const now = performance.now() + if (now - lastBroadcastTimeRef.current >= BROADCAST_INTERVAL_MS) { + lastBroadcastTimeRef.current = now + + // Capture from rectified canvas (local camera) or remote image + let imageData: string | null = null + + if (isLocalCamera && rectifiedCanvasRef.current) { + const canvas = rectifiedCanvasRef.current + if (canvas.width > 0 && canvas.height > 0) { + // Convert canvas to JPEG (quality 0.7 for bandwidth) + imageData = canvas.toDataURL('image/jpeg', 0.7).replace('data:image/jpeg;base64,', '') + } + } else if (isRemoteCamera && remoteLatestFrame) { + // Remote camera already sends base64 JPEG + imageData = remoteLatestFrame.imageData + } + + if (imageData) { + emitVisionFrame({ + imageData, + detectedValue, + confidence, + }) + } + } + + requestAnimationFrame(broadcastLoop) + } + + broadcastLoop() + + return () => { + running = false + } + }, [ + visionConfig.enabled, + isLocalCamera, + isRemoteCamera, + remoteLatestFrame, + detectedValue, + confidence, + emitVisionFrame, + ]) + const handleDisableVision = (e: React.MouseEvent) => { e.stopPropagation() setVisionEnabled(false) @@ -436,6 +493,9 @@ export function DockedVisionFeed({ onValueDetected, columnCount = 5 }: DockedVis videoRef={(el) => { videoRef.current = el }} + rectifiedCanvasRef={(el) => { + rectifiedCanvasRef.current = el + }} /> )} diff --git a/apps/web/src/components/vision/ObserverVisionFeed.tsx b/apps/web/src/components/vision/ObserverVisionFeed.tsx new file mode 100644 index 00000000..f4df2750 --- /dev/null +++ b/apps/web/src/components/vision/ObserverVisionFeed.tsx @@ -0,0 +1,123 @@ +'use client' + +import type { ObservedVisionFrame } from '@/hooks/useSessionObserver' +import { css } from '../../../styled-system/css' + +interface ObserverVisionFeedProps { + /** The latest vision frame from the observed student */ + frame: ObservedVisionFrame +} + +/** + * Displays the vision feed received from an observed student's session. + * + * Used in the SessionObserver modal when the student has abacus vision enabled. + * Shows the processed camera feed with detection status overlay. + */ +export function ObserverVisionFeed({ frame }: ObserverVisionFeedProps) { + // Calculate age of frame for staleness indicator + const frameAge = Date.now() - frame.receivedAt + const isStale = frameAge > 1000 // More than 1 second old + + return ( +
+ {/* Video frame */} + Student's abacus vision feed + + {/* Detection overlay */} +
+ {/* Detected value */} +
+ + {frame.detectedValue !== null ? frame.detectedValue : '---'} + + {frame.detectedValue !== null && ( + + {Math.round(frame.confidence * 100)}% + + )} +
+ + {/* Live indicator */} +
+
+ + {isStale ? 'Stale' : 'Live'} + +
+
+ + {/* Vision mode badge */} +
+ 📷 + Vision +
+
+ ) +} diff --git a/apps/web/src/components/vision/VisionCameraFeed.tsx b/apps/web/src/components/vision/VisionCameraFeed.tsx index 58fa7bdc..896f9819 100644 --- a/apps/web/src/components/vision/VisionCameraFeed.tsx +++ b/apps/web/src/components/vision/VisionCameraFeed.tsx @@ -19,6 +19,8 @@ export interface VisionCameraFeedProps { showRectifiedView?: boolean /** Video element ref callback for external access */ videoRef?: (el: HTMLVideoElement | null) => void + /** Rectified canvas ref callback for external access (only when showRectifiedView=true) */ + rectifiedCanvasRef?: (el: HTMLCanvasElement | null) => void /** Called when video metadata is loaded (provides dimensions) */ onVideoReady?: (width: number, height: number) => void /** Children rendered over the video (e.g., CalibrationOverlay) */ @@ -55,6 +57,7 @@ export function VisionCameraFeed({ showCalibrationGrid = false, showRectifiedView = false, videoRef: externalVideoRef, + rectifiedCanvasRef: externalCanvasRef, onVideoReady, children, }: VisionCameraFeedProps): ReactNode { @@ -82,6 +85,13 @@ export function VisionCameraFeed({ } }, [externalVideoRef]) + // Set canvas ref for external access (when rectified view is active) + useEffect(() => { + if (externalCanvasRef && showRectifiedView) { + externalCanvasRef(rectifiedCanvasRef.current) + } + }, [externalCanvasRef, showRectifiedView]) + // Attach stream to video element useEffect(() => { const video = internalVideoRef.current diff --git a/apps/web/src/contexts/MyAbacusContext.tsx b/apps/web/src/contexts/MyAbacusContext.tsx index 447f435b..a8bf2649 100644 --- a/apps/web/src/contexts/MyAbacusContext.tsx +++ b/apps/web/src/contexts/MyAbacusContext.tsx @@ -113,6 +113,23 @@ export interface DockAnimationState { toScale: number } +/** + * Vision frame data for broadcasting + */ +export interface VisionFrameData { + /** Base64-encoded JPEG image data */ + imageData: string + /** Detected abacus value (null if not yet detected) */ + detectedValue: number | null + /** Detection confidence (0-1) */ + confidence: number +} + +/** + * Callback type for vision frame broadcasting + */ +export type VisionFrameCallback = (frame: VisionFrameData) => void + interface MyAbacusContextValue { isOpen: boolean open: () => void @@ -185,6 +202,10 @@ interface MyAbacusContextValue { openVisionSetup: () => void /** Close the vision setup modal */ closeVisionSetup: () => void + /** Set a callback for receiving vision frames (for broadcasting to observers) */ + setVisionFrameCallback: (callback: VisionFrameCallback | null) => void + /** Emit a vision frame (called by DockedVisionFeed) */ + emitVisionFrame: (frame: VisionFrameData) => void } const MyAbacusContext = createContext(undefined) @@ -333,6 +354,17 @@ export function MyAbacusProvider({ children }: { children: React.ReactNode }) { setIsVisionSetupOpen(false) }, []) + // Vision frame broadcasting + const visionFrameCallbackRef = useRef(null) + + const setVisionFrameCallback = useCallback((callback: VisionFrameCallback | null) => { + visionFrameCallbackRef.current = callback + }, []) + + const emitVisionFrame = useCallback((frame: VisionFrameData) => { + visionFrameCallbackRef.current?.(frame) + }, []) + return ( {children} diff --git a/apps/web/src/hooks/useSessionBroadcast.ts b/apps/web/src/hooks/useSessionBroadcast.ts index 19adb1c6..b5eccc4b 100644 --- a/apps/web/src/hooks/useSessionBroadcast.ts +++ b/apps/web/src/hooks/useSessionBroadcast.ts @@ -11,6 +11,7 @@ import type { PracticeStateEvent, SessionPausedEvent, SessionResumedEvent, + VisionFrameEvent, } from '@/lib/classroom/socket-events' /** @@ -64,6 +65,12 @@ export interface UseSessionBroadcastResult { ) => void /** Send part transition complete event to observers */ sendPartTransitionComplete: () => void + /** Send vision frame to observers (when student has vision mode enabled) */ + sendVisionFrame: ( + imageData: string, + detectedValue: number | null, + confidence: number + ) => void } export function useSessionBroadcast( @@ -271,10 +278,31 @@ export function useSessionBroadcast( console.log('[SessionBroadcast] Emitted part-transition-complete') }, [sessionId]) + // Broadcast vision frame to observers + const sendVisionFrame = useCallback( + (imageData: string, detectedValue: number | null, confidence: number) => { + if (!socketRef.current || !isConnectedRef.current || !sessionId) { + return + } + + const event: VisionFrameEvent = { + sessionId, + imageData, + detectedValue, + confidence, + timestamp: Date.now(), + } + + socketRef.current.emit('vision-frame', event) + }, + [sessionId] + ) + return { isConnected: isConnectedRef.current, isBroadcasting: isConnectedRef.current && !!state, sendPartTransition, sendPartTransitionComplete, + sendVisionFrame, } } diff --git a/apps/web/src/hooks/useSessionObserver.ts b/apps/web/src/hooks/useSessionObserver.ts index a03d0e6d..0c1d756f 100644 --- a/apps/web/src/hooks/useSessionObserver.ts +++ b/apps/web/src/hooks/useSessionObserver.ts @@ -10,6 +10,7 @@ import type { PracticeStateEvent, SessionPausedEvent, SessionResumedEvent, + VisionFrameEvent, } from '@/lib/classroom/socket-events' /** @@ -110,6 +111,20 @@ export interface ObservedResult { recordedAt: number } +/** + * Vision frame received from student's abacus camera + */ +export interface ObservedVisionFrame { + /** Base64-encoded JPEG image data */ + imageData: string + /** Detected abacus value (null if not yet detected) */ + detectedValue: number | null + /** Detection confidence (0-1) */ + confidence: number + /** When this frame was received by observer */ + receivedAt: number +} + interface UseSessionObserverResult { /** Current observed state (null if not yet received) */ state: ObservedSessionState | null @@ -117,6 +132,8 @@ interface UseSessionObserverResult { results: ObservedResult[] /** Current part transition state (null if not in transition) */ transitionState: ObservedTransitionState | null + /** Latest vision frame from student's camera (null if vision not enabled) */ + visionFrame: ObservedVisionFrame | null /** Whether connected to the session channel */ isConnected: boolean /** Whether actively observing (connected and joined session) */ @@ -155,6 +172,7 @@ export function useSessionObserver( const [state, setState] = useState(null) const [results, setResults] = useState([]) const [transitionState, setTransitionState] = useState(null) + const [visionFrame, setVisionFrame] = useState(null) const [isConnected, setIsConnected] = useState(false) const [isObserving, setIsObserving] = useState(false) const [error, setError] = useState(null) @@ -354,6 +372,16 @@ export function useSessionObserver( setTransitionState(null) }) + // Listen for vision frames from student's camera + socket.on('vision-frame', (data: VisionFrameEvent) => { + setVisionFrame({ + imageData: data.imageData, + detectedValue: data.detectedValue, + confidence: data.confidence, + receivedAt: Date.now(), + }) + }) + // Listen for session ended event socket.on('session-ended', () => { console.log('[SessionObserver] Session ended') @@ -445,6 +473,7 @@ export function useSessionObserver( state, results, transitionState, + visionFrame, isConnected, isObserving, error, diff --git a/apps/web/src/lib/classroom/socket-events.ts b/apps/web/src/lib/classroom/socket-events.ts index 0e98e1d8..ccb47720 100644 --- a/apps/web/src/lib/classroom/socket-events.ts +++ b/apps/web/src/lib/classroom/socket-events.ts @@ -268,6 +268,22 @@ export interface PartTransitionCompleteEvent { sessionId: string } +/** + * Vision frame from student's abacus camera. + * Sent when student has vision mode enabled during practice. + */ +export interface VisionFrameEvent { + sessionId: string + /** Base64-encoded JPEG image data */ + imageData: string + /** Detected abacus value (null if not yet detected) */ + detectedValue: number | null + /** Detection confidence (0-1) */ + confidence: number + /** Timestamp when frame was captured */ + timestamp: number +} + /** * Sent when a student starts a practice session while present in a classroom. * Allows teacher to see session status update in real-time. @@ -401,6 +417,7 @@ export interface ClassroomServerToClientEvents { 'session-resumed': (data: SessionResumedEvent) => void 'part-transition': (data: PartTransitionEvent) => void 'part-transition-complete': (data: PartTransitionCompleteEvent) => void + 'vision-frame': (data: VisionFrameEvent) => void // Session status events (classroom channel - for teacher's active sessions view) 'session-started': (data: SessionStartedEvent) => void @@ -427,6 +444,7 @@ export interface ClassroomClientToServerEvents { // Session state broadcasts (from student client) 'practice-state': (data: PracticeStateEvent) => void 'tutorial-state': (data: TutorialStateEvent) => void + 'vision-frame': (data: VisionFrameEvent) => void // Observer controls 'tutorial-control': (data: TutorialControlEvent) => void diff --git a/apps/web/src/socket-server.ts b/apps/web/src/socket-server.ts index 43ac3c40..fc98247f 100644 --- a/apps/web/src/socket-server.ts +++ b/apps/web/src/socket-server.ts @@ -978,6 +978,21 @@ export function initializeSocketServer(httpServer: HTTPServer) { io!.to(`session:${data.sessionId}`).emit('session-resumed', data) }) + // Session Observation: Broadcast vision frame from student's abacus camera + socket.on( + 'vision-frame', + (data: { + sessionId: string + imageData: string + detectedValue: number | null + confidence: number + timestamp: number + }) => { + // Broadcast to all observers in the session channel + socket.to(`session:${data.sessionId}`).emit('vision-frame', data) + } + ) + // Skill Tutorial: Broadcast state from student to classroom (for teacher observation) // The student joins the classroom channel and emits their tutorial state socket.on(