antialias
/
soroban-abacus-flashcards
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(vision): broadcast vision frames to observers (Phase 5) 

Wire up the vision broadcast pipeline:

1. DockedVisionFeed captures rectified frames from canvas and emits
   them at 5fps via the context's emitVisionFrame callback

2. PracticeClient wires setVisionFrameCallback to call sendVisionFrame
   from useSessionBroadcast, connecting the context to the socket

3. useSessionBroadcast sends VisionFrameEvent to the session channel
   with imageData, detectedValue, and confidence

4. socket-server relays vision-frame events to observers

5. useSessionObserver receives and stores visionFrame for display

6. SessionObserverModal shows ObserverVisionFeed when visionFrame
   is available, replacing the interactive AbacusDock with the
   student's live camera feed

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Thomas Hallock 2026-01-01 15:28:59 -06:00
parent ff59612e7b
commit b3b769c0e2
10 changed files with 353 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
apps/web/src/app/practice/[studentId]/PracticeClient.tsx
View File

@ -1,8 +1,9 @@
'use client'
import { useRouter } from 'next/navigation'
import { useCallback, useMemo, useState } from 'react'
import { useCallback, useEffect, useMemo, useState } from 'react'
import { useToast } from '@/components/common/ToastContext'
import { useMyAbacus } from '@/contexts/MyAbacusContext'
import { PageWithNav } from '@/components/PageWithNav'
import {
ActiveSession,
@ -43,6 +44,7 @@ interface PracticeClientProps {
export function PracticeClient({ studentId, player, initialSession }: PracticeClientProps) {
const router = useRouter()
const { showError } = useToast()
const { setVisionFrameCallback } = useMyAbacus()
// Track pause state for HUD display (ActiveSession owns the modal and actual pause logic)
const [isPaused, setIsPaused] = useState(false)
@ -168,7 +170,7 @@ export function PracticeClient({ studentId, player, initialSession }: PracticeCl
// broadcastState is updated by ActiveSession via the onBroadcastStateChange callback
// onAbacusControl receives control events from observing teacher
// onTeacherPause/onTeacherResume receive pause/resume commands from teacher
const { sendPartTransition, sendPartTransitionComplete } = useSessionBroadcast(
const { sendPartTransition, sendPartTransitionComplete, sendVisionFrame } = useSessionBroadcast(
currentPlan.id,
studentId,
broadcastState,
@ -179,6 +181,17 @@ export function PracticeClient({ studentId, player, initialSession }: PracticeCl
}
)
// Wire vision frame callback to broadcast vision frames to observers
useEffect(() => {
setVisionFrameCallback((frame) => {
sendVisionFrame(frame.imageData, frame.detectedValue, frame.confidence)
})
return () => {
setVisionFrameCallback(null)
}
}, [setVisionFrameCallback, sendVisionFrame])
// Build session HUD data for PracticeSubNav
const sessionHud: SessionHudData | undefined = currentPart
? {

29
apps/web/src/components/classroom/SessionObserverModal.tsx
View File

@ -21,6 +21,7 @@ import { PracticeFeedback } from '../practice/PracticeFeedback'
import { PurposeBadge } from '../practice/PurposeBadge'
import { SessionProgressIndicator } from '../practice/SessionProgressIndicator'
import { VerticalProblem } from '../practice/VerticalProblem'
import { ObserverVisionFeed } from '../vision/ObserverVisionFeed'
interface SessionObserverModalProps {
/** Whether the modal is open */
@ -162,6 +163,7 @@ export function SessionObserverView({
state,
results,
transitionState,
visionFrame,
isConnected,
isObserving,
error,
@ -756,15 +758,9 @@ export function SessionObserverView({
/>
</div>
{/* AbacusDock - positioned exactly like ActiveSession */}
{/* Vision feed or AbacusDock - positioned exactly like ActiveSession */}
{state.phase === 'problem' && (problemHeight ?? 0) > 0 && (
<AbacusDock
id="teacher-observer-dock"
columns={abacusColumns}
interactive={true}
showNumbers={false}
animated={true}
onValueChange={handleTeacherAbacusChange}
<div
className={css({
position: 'absolute',
left: '100%',
@ -773,7 +769,22 @@ export function SessionObserverView({
marginLeft: '1.5rem',
})}
style={{ height: problemHeight ?? undefined }}
/>
>
{/* Show vision feed if available, otherwise show teacher's abacus dock */}
{visionFrame ? (
<ObserverVisionFeed frame={visionFrame} />
) : (
<AbacusDock
id="teacher-observer-dock"
columns={abacusColumns}
interactive={true}
showNumbers={false}
animated={true}
onValueChange={handleTeacherAbacusChange}
style={{ height: '100%' }}
/>
)}
</div>
)}
</div>

62
apps/web/src/components/vision/DockedVisionFeed.tsx
View File

@ -33,13 +33,15 @@ interface DockedVisionFeedProps {
* - Shows the video feed with detection overlay
*/
export function DockedVisionFeed({ onValueDetected, columnCount = 5 }: DockedVisionFeedProps) {
const { visionConfig, setDockedValue, setVisionEnabled, setVisionCalibration } = useMyAbacus()
const { visionConfig, setDockedValue, setVisionEnabled, setVisionCalibration, emitVisionFrame } = useMyAbacus()
const videoRef = useRef<HTMLVideoElement>(null)
const remoteImageRef = useRef<HTMLImageElement>(null)
const rectifiedCanvasRef = useRef<HTMLCanvasElement | null>(null)
const animationFrameRef = useRef<number | null>(null)
const markerDetectionFrameRef = useRef<number | null>(null)
const lastInferenceTimeRef = useRef<number>(0)
const lastBroadcastTimeRef = useRef<number>(0)
const [videoStream, setVideoStream] = useState<MediaStream | null>(null)
const [error, setError] = useState<string | null>(null)
@ -336,6 +338,61 @@ export function DockedVisionFeed({ onValueDetected, columnCount = 5 }: DockedVis
}
}, [stability.stableValue, stability.currentConfidence, detectedValue, setDockedValue, onValueDetected])
// Broadcast vision frames to observers (5fps to save bandwidth)
const BROADCAST_INTERVAL_MS = 200
useEffect(() => {
if (!visionConfig.enabled) return
let running = true
const broadcastLoop = () => {
if (!running) return
const now = performance.now()
if (now - lastBroadcastTimeRef.current >= BROADCAST_INTERVAL_MS) {
lastBroadcastTimeRef.current = now
// Capture from rectified canvas (local camera) or remote image
let imageData: string | null = null
if (isLocalCamera && rectifiedCanvasRef.current) {
const canvas = rectifiedCanvasRef.current
if (canvas.width > 0 && canvas.height > 0) {
// Convert canvas to JPEG (quality 0.7 for bandwidth)
imageData = canvas.toDataURL('image/jpeg', 0.7).replace('data:image/jpeg;base64,', '')
}
} else if (isRemoteCamera && remoteLatestFrame) {
// Remote camera already sends base64 JPEG
imageData = remoteLatestFrame.imageData
}
if (imageData) {
emitVisionFrame({
imageData,
detectedValue,
confidence,
})
}
}
requestAnimationFrame(broadcastLoop)
}
broadcastLoop()
return () => {
running = false
}
}, [
visionConfig.enabled,
isLocalCamera,
isRemoteCamera,
remoteLatestFrame,
detectedValue,
confidence,
emitVisionFrame,
])
const handleDisableVision = (e: React.MouseEvent) => {
e.stopPropagation()
setVisionEnabled(false)
@ -436,6 +493,9 @@ export function DockedVisionFeed({ onValueDetected, columnCount = 5 }: DockedVis
videoRef={(el) => {
videoRef.current = el
}}
rectifiedCanvasRef={(el) => {
rectifiedCanvasRef.current = el
}}
/>
)}

123
apps/web/src/components/vision/ObserverVisionFeed.tsx Normal file
View File

@ -0,0 +1,123 @@
'use client'
import type { ObservedVisionFrame } from '@/hooks/useSessionObserver'
import { css } from '../../../styled-system/css'
interface ObserverVisionFeedProps {
/** The latest vision frame from the observed student */
frame: ObservedVisionFrame
}
/**
* Displays the vision feed received from an observed student's session.
*
* Used in the SessionObserver modal when the student has abacus vision enabled.
* Shows the processed camera feed with detection status overlay.
*/
export function ObserverVisionFeed({ frame }: ObserverVisionFeedProps) {
// Calculate age of frame for staleness indicator
const frameAge = Date.now() - frame.receivedAt
const isStale = frameAge > 1000 // More than 1 second old
return (
<div
data-component="observer-vision-feed"
data-stale={isStale}
className={css({
position: 'relative',
display: 'flex',
flexDirection: 'column',
borderRadius: 'lg',
overflow: 'hidden',
bg: 'black',
})}
>
{/* Video frame */}
<img
src={`data:image/jpeg;base64,${frame.imageData}`}
alt="Student's abacus vision feed"
className={css({
width: '100%',
height: 'auto',
display: 'block',
opacity: isStale ? 0.5 : 1,
transition: 'opacity 0.3s',
})}
/>
{/* Detection overlay */}
<div
data-element="detection-overlay"
className={css({
position: 'absolute',
bottom: 0,
left: 0,
right: 0,
display: 'flex',
justifyContent: 'space-between',
alignItems: 'center',
p: 2,
bg: 'rgba(0, 0, 0, 0.7)',
backdropFilter: 'blur(4px)',
})}
>
{/* Detected value */}
<div className={css({ display: 'flex', alignItems: 'center', gap: 2 })}>
<span
className={css({ fontSize: 'lg', fontWeight: 'bold', color: 'white', fontFamily: 'mono' })}
>
{frame.detectedValue !== null ? frame.detectedValue : '---'}
</span>
{frame.detectedValue !== null && (
<span className={css({ fontSize: 'xs', color: 'gray.400' })}>
{Math.round(frame.confidence * 100)}%
</span>
)}
</div>
{/* Live indicator */}
<div className={css({ display: 'flex', alignItems: 'center', gap: 1 })}>
<div
className={css({
w: '8px',
h: '8px',
borderRadius: 'full',
bg: isStale ? 'gray.500' : 'green.500',
animation: isStale ? 'none' : 'pulse 2s infinite',
})}
/>
<span
className={css({
fontSize: 'xs',
color: isStale ? 'gray.500' : 'green.400',
})}
>
{isStale ? 'Stale' : 'Live'}
</span>
</div>
</div>
{/* Vision mode badge */}
<div
data-element="vision-badge"
className={css({
position: 'absolute',
top: '4px',
left: '4px',
display: 'flex',
alignItems: 'center',
gap: 1,
px: 2,
py: 1,
bg: 'rgba(0, 0, 0, 0.6)',
borderRadius: 'md',
fontSize: 'xs',
color: 'cyan.400',
})}
>
<span>📷</span>
<span>Vision</span>
</div>
</div>
)
}

10
apps/web/src/components/vision/VisionCameraFeed.tsx
View File

@ -19,6 +19,8 @@ export interface VisionCameraFeedProps {
showRectifiedView?: boolean
/** Video element ref callback for external access */
videoRef?: (el: HTMLVideoElement | null) => void
/** Rectified canvas ref callback for external access (only when showRectifiedView=true) */
rectifiedCanvasRef?: (el: HTMLCanvasElement | null) => void
/** Called when video metadata is loaded (provides dimensions) */
onVideoReady?: (width: number, height: number) => void
/** Children rendered over the video (e.g., CalibrationOverlay) */
@ -55,6 +57,7 @@ export function VisionCameraFeed({
showCalibrationGrid = false,
showRectifiedView = false,
videoRef: externalVideoRef,
rectifiedCanvasRef: externalCanvasRef,
onVideoReady,
children,
}: VisionCameraFeedProps): ReactNode {
@ -82,6 +85,13 @@ export function VisionCameraFeed({
}
}, [externalVideoRef])
// Set canvas ref for external access (when rectified view is active)
useEffect(() => {
if (externalCanvasRef && showRectifiedView) {
externalCanvasRef(rectifiedCanvasRef.current)
}
}, [externalCanvasRef, showRectifiedView])
// Attach stream to video element
useEffect(() => {
const video = internalVideoRef.current

34
apps/web/src/contexts/MyAbacusContext.tsx
View File

@ -113,6 +113,23 @@ export interface DockAnimationState {
toScale: number
}
/**
* Vision frame data for broadcasting
*/
export interface VisionFrameData {
/** Base64-encoded JPEG image data */
imageData: string
/** Detected abacus value (null if not yet detected) */
detectedValue: number | null
/** Detection confidence (0-1) */
confidence: number
}
/**
* Callback type for vision frame broadcasting
*/
export type VisionFrameCallback = (frame: VisionFrameData) => void
interface MyAbacusContextValue {
isOpen: boolean
open: () => void
@ -185,6 +202,10 @@ interface MyAbacusContextValue {
openVisionSetup: () => void
/** Close the vision setup modal */
closeVisionSetup: () => void
/** Set a callback for receiving vision frames (for broadcasting to observers) */
setVisionFrameCallback: (callback: VisionFrameCallback | null) => void
/** Emit a vision frame (called by DockedVisionFeed) */
emitVisionFrame: (frame: VisionFrameData) => void
}
const MyAbacusContext = createContext<MyAbacusContextValue | undefined>(undefined)
@ -333,6 +354,17 @@ export function MyAbacusProvider({ children }: { children: React.ReactNode }) {
setIsVisionSetupOpen(false)
}, [])
// Vision frame broadcasting
const visionFrameCallbackRef = useRef<VisionFrameCallback | null>(null)
const setVisionFrameCallback = useCallback((callback: VisionFrameCallback | null) => {
visionFrameCallbackRef.current = callback
}, [])
const emitVisionFrame = useCallback((frame: VisionFrameData) => {
visionFrameCallbackRef.current?.(frame)
}, [])
return (
<MyAbacusContext.Provider
value={{
@ -376,6 +408,8 @@ export function MyAbacusProvider({ children }: { children: React.ReactNode }) {
isVisionSetupOpen,
openVisionSetup,
closeVisionSetup,
setVisionFrameCallback,
emitVisionFrame,
}}
>
{children}

28
apps/web/src/hooks/useSessionBroadcast.ts
View File

@ -11,6 +11,7 @@ import type {
PracticeStateEvent,
SessionPausedEvent,
SessionResumedEvent,
VisionFrameEvent,
} from '@/lib/classroom/socket-events'
/**
@ -64,6 +65,12 @@ export interface UseSessionBroadcastResult {
) => void
/** Send part transition complete event to observers */
sendPartTransitionComplete: () => void
/** Send vision frame to observers (when student has vision mode enabled) */
sendVisionFrame: (
imageData: string,
detectedValue: number | null,
confidence: number
) => void
}
export function useSessionBroadcast(
@ -271,10 +278,31 @@ export function useSessionBroadcast(
console.log('[SessionBroadcast] Emitted part-transition-complete')
}, [sessionId])
// Broadcast vision frame to observers
const sendVisionFrame = useCallback(
(imageData: string, detectedValue: number | null, confidence: number) => {
if (!socketRef.current || !isConnectedRef.current || !sessionId) {
return
}
const event: VisionFrameEvent = {
sessionId,
imageData,
detectedValue,
confidence,
timestamp: Date.now(),
}
socketRef.current.emit('vision-frame', event)
},
[sessionId]
)
return {
isConnected: isConnectedRef.current,
isBroadcasting: isConnectedRef.current && !!state,
sendPartTransition,
sendPartTransitionComplete,
sendVisionFrame,
}
}

29
apps/web/src/hooks/useSessionObserver.ts
View File

@ -10,6 +10,7 @@ import type {
PracticeStateEvent,
SessionPausedEvent,
SessionResumedEvent,
VisionFrameEvent,
} from '@/lib/classroom/socket-events'
/**
@ -110,6 +111,20 @@ export interface ObservedResult {
recordedAt: number
}
/**
* Vision frame received from student's abacus camera
*/
export interface ObservedVisionFrame {
/** Base64-encoded JPEG image data */
imageData: string
/** Detected abacus value (null if not yet detected) */
detectedValue: number | null
/** Detection confidence (0-1) */
confidence: number
/** When this frame was received by observer */
receivedAt: number
}
interface UseSessionObserverResult {
/** Current observed state (null if not yet received) */
state: ObservedSessionState | null
@ -117,6 +132,8 @@ interface UseSessionObserverResult {
results: ObservedResult[]
/** Current part transition state (null if not in transition) */
transitionState: ObservedTransitionState | null
/** Latest vision frame from student's camera (null if vision not enabled) */
visionFrame: ObservedVisionFrame | null
/** Whether connected to the session channel */
isConnected: boolean
/** Whether actively observing (connected and joined session) */
@ -155,6 +172,7 @@ export function useSessionObserver(
const [state, setState] = useState<ObservedSessionState | null>(null)
const [results, setResults] = useState<ObservedResult[]>([])
const [transitionState, setTransitionState] = useState<ObservedTransitionState | null>(null)
const [visionFrame, setVisionFrame] = useState<ObservedVisionFrame | null>(null)
const [isConnected, setIsConnected] = useState(false)
const [isObserving, setIsObserving] = useState(false)
const [error, setError] = useState<string | null>(null)
@ -354,6 +372,16 @@ export function useSessionObserver(
setTransitionState(null)
})
// Listen for vision frames from student's camera
socket.on('vision-frame', (data: VisionFrameEvent) => {
setVisionFrame({
imageData: data.imageData,
detectedValue: data.detectedValue,
confidence: data.confidence,
receivedAt: Date.now(),
})
})
// Listen for session ended event
socket.on('session-ended', () => {
console.log('[SessionObserver] Session ended')
@ -445,6 +473,7 @@ export function useSessionObserver(
state,
results,
transitionState,
visionFrame,
isConnected,
isObserving,
error,

18
apps/web/src/lib/classroom/socket-events.ts
View File

@ -268,6 +268,22 @@ export interface PartTransitionCompleteEvent {
sessionId: string
}
/**
* Vision frame from student's abacus camera.
* Sent when student has vision mode enabled during practice.
*/
export interface VisionFrameEvent {
sessionId: string
/** Base64-encoded JPEG image data */
imageData: string
/** Detected abacus value (null if not yet detected) */
detectedValue: number | null
/** Detection confidence (0-1) */
confidence: number
/** Timestamp when frame was captured */
timestamp: number
}
/**
* Sent when a student starts a practice session while present in a classroom.
* Allows teacher to see session status update in real-time.
@ -401,6 +417,7 @@ export interface ClassroomServerToClientEvents {
'session-resumed': (data: SessionResumedEvent) => void
'part-transition': (data: PartTransitionEvent) => void
'part-transition-complete': (data: PartTransitionCompleteEvent) => void
'vision-frame': (data: VisionFrameEvent) => void
// Session status events (classroom channel - for teacher's active sessions view)
'session-started': (data: SessionStartedEvent) => void
@ -427,6 +444,7 @@ export interface ClassroomClientToServerEvents {
// Session state broadcasts (from student client)
'practice-state': (data: PracticeStateEvent) => void
'tutorial-state': (data: TutorialStateEvent) => void
'vision-frame': (data: VisionFrameEvent) => void
// Observer controls
'tutorial-control': (data: TutorialControlEvent) => void

15
apps/web/src/socket-server.ts
View File

@ -978,6 +978,21 @@ export function initializeSocketServer(httpServer: HTTPServer) {
io!.to(`session:${data.sessionId}`).emit('session-resumed', data)
})
// Session Observation: Broadcast vision frame from student's abacus camera
socket.on(
'vision-frame',
(data: {
sessionId: string
imageData: string
detectedValue: number | null
confidence: number
timestamp: number
}) => {
// Broadcast to all observers in the session channel
socket.to(`session:${data.sessionId}`).emit('vision-frame', data)
}
)
// Skill Tutorial: Broadcast state from student to classroom (for teacher observation)
// The student joins the classroom channel and emits their tutorial state
socket.on(