diff --git a/DEPLOYMENT.md b/DEPLOYMENT.md index 195fa23c..4eb890fc 100644 --- a/DEPLOYMENT.md +++ b/DEPLOYMENT.md @@ -52,6 +52,7 @@ curl https://abaci.one/api/health ``` Response: + ```json { "status": "healthy", @@ -119,8 +120,8 @@ services: image: ghcr.io/antialias/soroban-abacus-flashcards:latest container_name: abaci-blue volumes: - - ./data:/app/apps/web/data # Shared database - - ./uploads:/app/uploads # Shared uploads + - ./data:/app/apps/web/data # Shared database + - ./uploads:/app/uploads # Shared uploads healthcheck: test: ["CMD", "curl", "-f", "http://localhost:3000/api/health"] labels: @@ -173,11 +174,13 @@ ssh nas.home.network "cd /volume1/homes/antialias/projects/abaci.one && docker-c ### Health Check Failing 1. Check container logs: + ```bash ssh nas.home.network "docker logs abaci-blue" ``` 2. Test health endpoint manually: + ```bash ssh nas.home.network "docker exec abaci-blue curl -sf http://localhost:3000/api/health" ``` @@ -209,6 +212,7 @@ If upgrading from the old single-container setup: ``` This script will: + 1. Stop the old `soroban-abacus-flashcards` container 2. Stop compose-updater temporarily 3. Deploy the new docker-compose.yaml diff --git a/apps/web/.claude/DEPLOYMENT.md b/apps/web/.claude/DEPLOYMENT.md index 1f37f44d..6986224e 100644 --- a/apps/web/.claude/DEPLOYMENT.md +++ b/apps/web/.claude/DEPLOYMENT.md @@ -37,6 +37,7 @@ curl https://abaci.one/api/health ``` Response: + ```json { "status": "healthy", @@ -174,11 +175,13 @@ git rev-parse HEAD ### Health Check Failing 1. Check container logs: + ```bash ssh nas.home.network "docker logs abaci-blue" ``` 2. Test health endpoint manually: + ```bash ssh nas.home.network "docker exec abaci-blue curl -sf http://localhost:3000/api/health" ``` diff --git a/apps/web/src/app/api/curriculum/[playerId]/sessions/[sessionId]/problems/[problemNumber]/video/route.ts b/apps/web/src/app/api/curriculum/[playerId]/sessions/[sessionId]/problems/[problemNumber]/video/route.ts index b5989842..a4c89dba 100644 --- a/apps/web/src/app/api/curriculum/[playerId]/sessions/[sessionId]/problems/[problemNumber]/video/route.ts +++ b/apps/web/src/app/api/curriculum/[playerId]/sessions/[sessionId]/problems/[problemNumber]/video/route.ts @@ -152,7 +152,9 @@ export async function GET(request: Request, { params }: RouteParams) { if (video.status === 'no_video') { return NextResponse.json( - { error: 'No video was recorded for this problem (camera may have been off)' }, + { + error: 'No video was recorded for this problem (camera may have been off)', + }, { status: 404 } ) } diff --git a/apps/web/src/app/practice/[studentId]/PracticeClient.tsx b/apps/web/src/app/practice/[studentId]/PracticeClient.tsx index f9515d54..180e10c1 100644 --- a/apps/web/src/app/practice/[studentId]/PracticeClient.tsx +++ b/apps/web/src/app/practice/[studentId]/PracticeClient.tsx @@ -582,7 +582,10 @@ export function PracticeClient({ studentId, player, initialSession }: PracticeCl // Handle part transition complete - called when transition screen finishes // This is where we trigger game break (after "put away abacus" message is shown) const handlePartTransitionComplete = useCallback(() => { - console.log('[PracticeClient] handlePartTransitionComplete called, pendingGameBreak:', pendingGameBreak) + console.log( + '[PracticeClient] handlePartTransitionComplete called, pendingGameBreak:', + pendingGameBreak + ) // First, broadcast to observers sendPartTransitionComplete() diff --git a/apps/web/src/components/debug/BroadcastDebugPanel.tsx b/apps/web/src/components/debug/BroadcastDebugPanel.tsx index 52a93d97..33b9843d 100644 --- a/apps/web/src/components/debug/BroadcastDebugPanel.tsx +++ b/apps/web/src/components/debug/BroadcastDebugPanel.tsx @@ -247,7 +247,13 @@ export function BroadcastDebugPanel({ border: '1px solid rgba(96, 165, 250, 0.3)', })} > -
+
Current State
@@ -275,7 +281,13 @@ export function BroadcastDebugPanel({ )} {lastBroadcastTime && ( -
+
Last broadcast: {new Date(lastBroadcastTime).toLocaleTimeString()}
)} diff --git a/apps/web/src/components/debug/ObserverDebugPanel.tsx b/apps/web/src/components/debug/ObserverDebugPanel.tsx index 5cc3a1ef..50f1af91 100644 --- a/apps/web/src/components/debug/ObserverDebugPanel.tsx +++ b/apps/web/src/components/debug/ObserverDebugPanel.tsx @@ -283,7 +283,13 @@ export function ObserverDebugPanel({ border: '1px solid rgba(244, 114, 182, 0.3)', })} > -
+
Observed State
@@ -325,7 +331,13 @@ export function ObserverDebugPanel({ border: '1px solid rgba(96, 165, 250, 0.3)', })} > -
+
Vision Frame
@@ -349,7 +361,13 @@ export function ObserverDebugPanel({ border: '1px solid rgba(168, 85, 247, 0.3)', })} > -
+
DVR Buffer
diff --git a/apps/web/src/components/practice/hooks/useInteractionPhase.ts b/apps/web/src/components/practice/hooks/useInteractionPhase.ts index 737c911a..129e1120 100644 --- a/apps/web/src/components/practice/hooks/useInteractionPhase.ts +++ b/apps/web/src/components/practice/hooks/useInteractionPhase.ts @@ -680,7 +680,10 @@ export function useInteractionPhase( // If no active problem, nothing to do if (!activeProblem) { - console.log('[useInteractionPhase] No active problem - cannot load. Current phase:', phase.phase) + console.log( + '[useInteractionPhase] No active problem - cannot load. Current phase:', + phase.phase + ) prevActiveProblemKeyRef.current = null return } @@ -718,7 +721,10 @@ export function useInteractionPhase( // Case 2: Key changed - handle redo mode or session advancement if (keyChanged) { - console.log('[useInteractionPhase] Key changed:', { prevKey, currentKey }) + console.log('[useInteractionPhase] Key changed:', { + prevKey, + currentKey, + }) // CRITICAL: Don't interrupt normal progression flow // If we're in showingFeedback, submitting, or transitioning, the normal flow diff --git a/apps/web/src/components/vision/ProblemVideoPlayer.tsx b/apps/web/src/components/vision/ProblemVideoPlayer.tsx index 59d86f56..ce373e4a 100644 --- a/apps/web/src/components/vision/ProblemVideoPlayer.tsx +++ b/apps/web/src/components/vision/ProblemVideoPlayer.tsx @@ -469,6 +469,7 @@ export function ProblemVideoPlayer({ src={videoUrl} controls playsInline + muted onCanPlay={handleCanPlay} onTimeUpdate={handleTimeUpdate} className={css({ @@ -839,8 +840,13 @@ export function ProblemVideoPlayer({ Step {noVideoPlaybackIndex + 1} of {answerProgression.length} {currentNoVideoState?.originalTimestamp !== undefined && ( - - (actual: {(currentNoVideoState.originalTimestamp / 1000).toFixed(1)}s) + + (actual: {(currentNoVideoState.originalTimestamp / 1000).toFixed(1)} + s) )}
diff --git a/apps/web/src/hooks/__tests__/useUserPlayers.test.tsx b/apps/web/src/hooks/__tests__/useUserPlayers.test.tsx index ffcbf880..6c4305a5 100644 --- a/apps/web/src/hooks/__tests__/useUserPlayers.test.tsx +++ b/apps/web/src/hooks/__tests__/useUserPlayers.test.tsx @@ -325,7 +325,9 @@ describe('useUserPlayers hooks', () => { }) // Should have invalidated with playerKeys.all - expect(invalidateSpy).toHaveBeenCalledWith({ queryKey: playerKeys.all }) + expect(invalidateSpy).toHaveBeenCalledWith({ + queryKey: playerKeys.all, + }) }) test('invalidates all player queries even on error', async () => { @@ -349,7 +351,9 @@ describe('useUserPlayers hooks', () => { }) // onSettled runs on both success and error - expect(invalidateSpy).toHaveBeenCalledWith({ queryKey: playerKeys.all }) + expect(invalidateSpy).toHaveBeenCalledWith({ + queryKey: playerKeys.all, + }) }) }) @@ -422,7 +426,10 @@ describe('useUserPlayers hooks', () => { const { result } = renderHook(() => useUpdatePlayer(), { wrapper }) act(() => { - result.current.mutate({ id: 'player-1', updates: { name: 'Updated Name' } }) + result.current.mutate({ + id: 'player-1', + updates: { name: 'Updated Name' }, + }) }) // Wait for optimistic update @@ -461,7 +468,10 @@ describe('useUserPlayers hooks', () => { const { result } = renderHook(() => useUpdatePlayer(), { wrapper }) act(() => { - result.current.mutate({ id: 'player-1', updates: { name: 'Updated Name' } }) + result.current.mutate({ + id: 'player-1', + updates: { name: 'Updated Name' }, + }) }) await waitFor(() => { diff --git a/apps/web/src/hooks/useSessionBroadcast.ts b/apps/web/src/hooks/useSessionBroadcast.ts index fd62d932..dc18e1ce 100644 --- a/apps/web/src/hooks/useSessionBroadcast.ts +++ b/apps/web/src/hooks/useSessionBroadcast.ts @@ -406,13 +406,12 @@ export function useSessionBroadcast( return } - // Only send markers if recording is active - if (!isRecordingRef.current) { - return - } + // Always send markers - server will capture metadata even without video frames + // This enables playback of student answers even when camera wasn't enabled socketRef.current.emit('vision-problem-marker', { sessionId, + playerId, // Include playerId for auto-starting metadata-only sessions problemNumber, partIndex, eventType, @@ -430,7 +429,7 @@ export function useSessionBroadcast( retryContext, }) }, - [sessionId] + [sessionId, playerId] ) return { diff --git a/apps/web/src/lib/vision/recording/README.md b/apps/web/src/lib/vision/recording/README.md index 71c35802..5a8c11e0 100644 --- a/apps/web/src/lib/vision/recording/README.md +++ b/apps/web/src/lib/vision/recording/README.md @@ -58,37 +58,39 @@ When a student practices with their camera enabled, the system records a separat Problem markers are socket events that coordinate video recording with problem lifecycle: -| Marker Event | When Sent | VisionRecorder Action | -|--------------|-----------|----------------------| -| `problem-shown` | New problem appears on screen | Start buffering frames for this problem | -| `answer-submitted` | Student submits their answer | Stop buffering, encode video, save to disk | -| `feedback-shown` | Feedback displayed | (Reserved for future use) | +| Marker Event | When Sent | VisionRecorder Action | +| ------------------ | ----------------------------- | ------------------------------------------ | +| `problem-shown` | New problem appears on screen | Start buffering frames for this problem | +| `answer-submitted` | Student submits their answer | Stop buffering, encode video, save to disk | +| `feedback-shown` | Feedback displayed | (Reserved for future use) | ### Marker Payload ```typescript interface ProblemMarker { - sessionId: string - problemNumber: number // 1-indexed problem number - partIndex: number // Which part of the session - eventType: 'problem-shown' | 'answer-submitted' | 'feedback-shown' - isCorrect?: boolean // Only for answer-submitted + sessionId: string; + problemNumber: number; // 1-indexed problem number + partIndex: number; // Which part of the session + eventType: "problem-shown" | "answer-submitted" | "feedback-shown"; + isCorrect?: boolean; // Only for answer-submitted // Retry context (for multiple attempts at same problem) - epochNumber: number // 0 = initial pass, 1+ = retry epochs - attemptNumber: number // Which attempt (1, 2, 3...) - isRetry: boolean // True if in a retry epoch - isManualRedo: boolean // True if student clicked dot to redo + epochNumber: number; // 0 = initial pass, 1+ = retry epochs + attemptNumber: number; // Which attempt (1, 2, 3...) + isRetry: boolean; // True if in a retry epoch + isManualRedo: boolean; // True if student clicked dot to redo } ``` ### Why Retry Context Matters Students can attempt the same problem multiple times: + - **Epoch retries**: End-of-part retry rounds for missed problems - **Manual redos**: Student clicks a completed problem dot to practice again Each attempt gets its own recording. The retry context determines: + - **Filename**: `problem_001_e0_a1.mp4` (epoch 0, attempt 1) vs `problem_001_e0_a2.mp4` (attempt 2) - **Database record**: Separate rows in `vision_problem_videos` with epoch/attempt fields @@ -112,41 +114,41 @@ Each video has a companion `.meta.json` file with time-coded state for synchroni ```typescript interface ProblemMetadata { problem: { - terms: number[] // e.g., [45, -23, 12] - answer: number // e.g., 34 - } + terms: number[]; // e.g., [45, -23, 12] + answer: number; // e.g., 34 + }; entries: Array<{ - t: number // ms from video start - detectedValue: number | null // ML-detected abacus value - confidence: number // 0-1 detection confidence - studentAnswer: string // Current typed answer - phase: 'problem' | 'feedback' - isCorrect?: boolean - }> - durationMs: number - frameCount: number - isCorrect: boolean | null + t: number; // ms from video start + detectedValue: number | null; // ML-detected abacus value + confidence: number; // 0-1 detection confidence + studentAnswer: string; // Current typed answer + phase: "problem" | "feedback"; + isCorrect?: boolean; + }>; + durationMs: number; + frameCount: number; + isCorrect: boolean | null; } ``` ## Key Files -| File | Purpose | -|------|---------| -| `src/lib/vision/recording/VisionRecorder.ts` | Server-side recording manager | -| `src/socket-server.ts` | Socket event routing (lines handling vision-*) | -| `src/hooks/useSessionBroadcast.ts` | Client socket helpers (`sendProblemMarker`) | -| `src/app/practice/[studentId]/PracticeClient.tsx` | Sends markers on problem state changes | -| `src/components/vision/DockedVisionFeed.tsx` | Captures and sends camera frames | -| `src/components/vision/ProblemVideoPlayer.tsx` | Playback UI with metadata sync | +| File | Purpose | +| ------------------------------------------------- | ----------------------------------------------- | +| `src/lib/vision/recording/VisionRecorder.ts` | Server-side recording manager | +| `src/socket-server.ts` | Socket event routing (lines handling vision-\*) | +| `src/hooks/useSessionBroadcast.ts` | Client socket helpers (`sendProblemMarker`) | +| `src/app/practice/[studentId]/PracticeClient.tsx` | Sends markers on problem state changes | +| `src/components/vision/DockedVisionFeed.tsx` | Captures and sends camera frames | +| `src/components/vision/ProblemVideoPlayer.tsx` | Playback UI with metadata sync | ## API Routes -| Route | Purpose | -|-------|---------| -| `GET /api/curriculum/[playerId]/sessions/[sessionId]/videos` | List available recordings | -| `GET /api/.../problems/[problemNumber]/video?epoch=X&attempt=Y` | Stream video file | -| `GET /api/.../problems/[problemNumber]/metadata?epoch=X&attempt=Y` | Get metadata JSON | +| Route | Purpose | +| ------------------------------------------------------------------ | ------------------------- | +| `GET /api/curriculum/[playerId]/sessions/[sessionId]/videos` | List available recordings | +| `GET /api/.../problems/[problemNumber]/video?epoch=X&attempt=Y` | Stream video file | +| `GET /api/.../problems/[problemNumber]/metadata?epoch=X&attempt=Y` | Get metadata JSON | ## Database Schema diff --git a/apps/web/src/lib/vision/recording/VisionRecorder.ts b/apps/web/src/lib/vision/recording/VisionRecorder.ts index f328bc50..26446de3 100644 --- a/apps/web/src/lib/vision/recording/VisionRecorder.ts +++ b/apps/web/src/lib/vision/recording/VisionRecorder.ts @@ -1,4 +1,4 @@ -import { eq, and } from 'drizzle-orm' +import { eq } from 'drizzle-orm' import { createId } from '@paralleldrive/cuid2' import { mkdir, writeFile, rm } from 'fs/promises' import path from 'path' diff --git a/apps/web/src/socket-server.ts b/apps/web/src/socket-server.ts index e8f68c29..b30e7def 100644 --- a/apps/web/src/socket-server.ts +++ b/apps/web/src/socket-server.ts @@ -1,7 +1,6 @@ -import type { Server as HTTPServer, IncomingMessage } from 'http' +import type { Server as HTTPServer } from 'http' import { Server as SocketIOServer } from 'socket.io' import type { Server as SocketIOServerType } from 'socket.io' -import { WebSocketServer, type WebSocket } from 'ws' import { applyGameMove, createArcadeSession, @@ -1174,6 +1173,8 @@ export function initializeSocketServer(httpServer: HTTPServer) { }) // Vision Recording: Handle problem marker (triggers encoding on problem transitions) + // NOTE: Markers are always processed - if no recording session exists, one is auto-started + // for metadata-only capture (student answers without video) socket.on( 'vision-problem-marker', async ({ @@ -1186,6 +1187,7 @@ export function initializeSocketServer(httpServer: HTTPServer) { attemptNumber, isRetry, isManualRedo, + playerId, }: { sessionId: string problemNumber: number @@ -1196,8 +1198,19 @@ export function initializeSocketServer(httpServer: HTTPServer) { attemptNumber?: number isRetry?: boolean isManualRedo?: boolean + playerId?: string }) => { const recorder = VisionRecorder.getInstance() + + // Auto-start a metadata-only session if one doesn't exist + // This allows capturing student answers even when camera isn't enabled + if (!recorder.isRecording(sessionId) && playerId) { + console.log( + `📝 Auto-starting metadata-only recording session for ${sessionId} (no camera)` + ) + recorder.startSession(sessionId, playerId) + } + if (recorder.isRecording(sessionId)) { // This triggers encoding when 'problem-shown' arrives for the next problem await recorder.onProblemMarker(sessionId, { diff --git a/nas-deployment/docker-compose.blue.yaml b/nas-deployment/docker-compose.blue.yaml index c32e7c34..1816809d 100644 --- a/nas-deployment/docker-compose.blue.yaml +++ b/nas-deployment/docker-compose.blue.yaml @@ -12,7 +12,13 @@ services: networks: - webgateway healthcheck: - test: ["CMD", "node", "-e", "require('http').get('http://localhost:3000/', r => process.exit(r.statusCode < 400 ? 0 : 1)).on('error', () => process.exit(1))"] + test: + [ + "CMD", + "node", + "-e", + "require('http').get('http://localhost:3000/', r => process.exit(r.statusCode < 400 ? 0 : 1)).on('error', () => process.exit(1))", + ] interval: 10s timeout: 5s retries: 3 diff --git a/nas-deployment/docker-compose.green.yaml b/nas-deployment/docker-compose.green.yaml index 1854e6d2..a96c1dc0 100644 --- a/nas-deployment/docker-compose.green.yaml +++ b/nas-deployment/docker-compose.green.yaml @@ -12,7 +12,13 @@ services: networks: - webgateway healthcheck: - test: ["CMD", "node", "-e", "require('http').get('http://localhost:3000/', r => process.exit(r.statusCode < 400 ? 0 : 1)).on('error', () => process.exit(1))"] + test: + [ + "CMD", + "node", + "-e", + "require('http').get('http://localhost:3000/', r => process.exit(r.statusCode < 400 ? 0 : 1)).on('error', () => process.exit(1))", + ] interval: 10s timeout: 5s retries: 3 diff --git a/nas-deployment/docker-compose.yaml b/nas-deployment/docker-compose.yaml index 5db972b6..d798c00c 100644 --- a/nas-deployment/docker-compose.yaml +++ b/nas-deployment/docker-compose.yaml @@ -20,7 +20,13 @@ x-app: &app networks: - webgateway healthcheck: - test: ["CMD", "node", "-e", "require('http').get('http://localhost:3000/', r => process.exit(r.statusCode < 400 ? 0 : 1)).on('error', () => process.exit(1))"] + test: + [ + "CMD", + "node", + "-e", + "require('http').get('http://localhost:3000/', r => process.exit(r.statusCode < 400 ? 0 : 1)).on('error', () => process.exit(1))", + ] interval: 10s timeout: 5s retries: 3