diff --git a/apps/web/src/components/practice/ActiveSession.tsx b/apps/web/src/components/practice/ActiveSession.tsx index 123c8ba6..e65180bd 100644 --- a/apps/web/src/components/practice/ActiveSession.tsx +++ b/apps/web/src/components/practice/ActiveSession.tsx @@ -1882,6 +1882,12 @@ export function ActiveSession({ : undefined } rejectedDigit={attempt.rejectedDigit} + detectedPrefixIndex={ + // Show vision feedback for prefix sums (not final answer) + matchedPrefixIndex >= 0 && matchedPrefixIndex < prefixSums.length - 1 + ? matchedPrefixIndex + : undefined + } helpOverlay={ // Always render overlay when in help mode (for exit transition) showHelpOverlay && helpContext ? ( diff --git a/apps/web/src/components/practice/VerticalProblem.tsx b/apps/web/src/components/practice/VerticalProblem.tsx index 0ae4abc0..2bb6819e 100644 --- a/apps/web/src/components/practice/VerticalProblem.tsx +++ b/apps/web/src/components/practice/VerticalProblem.tsx @@ -40,6 +40,8 @@ interface VerticalProblemProps { generationTrace?: GenerationTrace /** Complexity budget constraint (for debug overlay) */ complexityBudget?: number + /** Index of detected prefix sum from vision (shows visual indicator on completed terms) */ + detectedPrefixIndex?: number } /** @@ -68,6 +70,7 @@ export function VerticalProblem({ answerFadingOut = false, generationTrace, complexityBudget, + detectedPrefixIndex, }: VerticalProblemProps) { const { resolvedTheme } = useTheme() const isDark = resolvedTheme === 'dark' @@ -163,12 +166,25 @@ export function VerticalProblem({ const showNeedHelp = index === needHelpTermIndex && !isCurrentHelp // Check if this term is already included in the prefix sum (when in help mode) const isInPrefixSum = currentHelpTermIndex !== undefined && index < currentHelpTermIndex + // Check if this term is completed based on vision detection (before help mode) + const isVisionCompleted = + detectedPrefixIndex !== undefined && + currentHelpTermIndex === undefined && // Only show when NOT in help mode + index <= detectedPrefixIndex return (
)} + {/* Checkmark indicator for vision-detected completed terms */} + {isVisionCompleted && ( +
+ ✓ +
+ )} + {/* Operator column (only show minus for negative) */}
(null) const lastInferenceTimeRef = useRef(0) const lastBroadcastTimeRef = useRef(0) + const isInferringRef = useRef(false) // Prevent concurrent inference const [videoStream, setVideoStream] = useState(null) const [error, setError] = useState(null) @@ -86,6 +67,16 @@ export function DockedVisionFeed({ onValueDetected, columnCount = 5 }: DockedVis // Track video element in state for marker detection hook const [videoElement, setVideoElement] = useState(null) + // ML column classifier hook + const classifier = useColumnClassifier() + + // Preload the ML model when component mounts + useEffect(() => { + if (ENABLE_AUTO_DETECTION) { + classifier.preload() + } + }, [classifier]) + // Stability tracking for detected values (hook must be called unconditionally) const stability = useFrameStability() @@ -225,9 +216,11 @@ export function DockedVisionFeed({ onValueDetected, columnCount = 5 }: DockedVis }, [isRemoteCamera, remoteIsPhoneConnected]) // Process local camera frames for detection (only when enabled) - const processLocalFrame = useCallback(() => { - // Skip detection when feature is disabled + const processLocalFrame = useCallback(async () => { + // Skip detection when feature is disabled or model not ready if (!ENABLE_AUTO_DETECTION) return + if (!classifier.isModelLoaded) return + if (isInferringRef.current) return // Skip if already inferring const now = performance.now() if (now - lastInferenceTimeRef.current < INFERENCE_INTERVAL_MS) { @@ -239,25 +232,36 @@ export function DockedVisionFeed({ onValueDetected, columnCount = 5 }: DockedVis if (!video || video.readyState < 2) return if (!visionConfig.calibration) return - // Process video frame into column strips - const columnImages = processVideoFrame(video, visionConfig.calibration) - if (columnImages.length === 0) return + isInferringRef.current = true - // Use CV-based bead detection - const analyses = analyzeColumns(columnImages) - const { digits, minConfidence } = analysesToDigits(analyses) + try { + // Process video frame into column strips + const columnImages = processVideoFrame(video, visionConfig.calibration) + if (columnImages.length === 0) return - // Convert to number - const value = digitsToNumber(digits) + // Use ML-based digit classification + const results = await classifier.classifyColumns(columnImages) + if (!results || results.digits.length === 0) return - // Push to stability buffer - stability.pushFrame(value, minConfidence) - }, [visionConfig.calibration, stability]) + // Extract digits and minimum confidence + const { digits, confidences } = results + const minConfidence = Math.min(...confidences) + + // Convert to number + const value = digitsToNumber(digits) + + // Push to stability buffer + stability.pushFrame(value, minConfidence) + } finally { + isInferringRef.current = false + } + }, [visionConfig.calibration, stability, classifier]) // Process remote camera frames for detection (only when enabled) useEffect(() => { - // Skip detection when feature is disabled + // Skip detection when feature is disabled or model not ready if (!ENABLE_AUTO_DETECTION) return + if (!classifier.isModelLoaded) return if (!isRemoteCamera || !remoteIsPhoneConnected || !remoteLatestFrame) { return @@ -267,32 +271,46 @@ export function DockedVisionFeed({ onValueDetected, columnCount = 5 }: DockedVis if (now - lastInferenceTimeRef.current < INFERENCE_INTERVAL_MS) { return } - lastInferenceTimeRef.current = now const image = remoteImageRef.current if (!image || !image.complete || image.naturalWidth === 0) { return } + // Prevent concurrent inference + if (isInferringRef.current) return + isInferringRef.current = true + lastInferenceTimeRef.current = now + // Phone sends pre-cropped frames in auto mode, so no calibration needed const columnImages = processImageFrame(image, null, columnCount) - if (columnImages.length === 0) return + if (columnImages.length === 0) { + isInferringRef.current = false + return + } - // Use CV-based bead detection - const analyses = analyzeColumns(columnImages) - const { digits, minConfidence } = analysesToDigits(analyses) + // Use ML-based digit classification (async) + classifier.classifyColumns(columnImages).then((results) => { + isInferringRef.current = false + if (!results || results.digits.length === 0) return - // Convert to number - const value = digitsToNumber(digits) + // Extract digits and minimum confidence + const { digits, confidences } = results + const minConfidence = Math.min(...confidences) - // Push to stability buffer - stability.pushFrame(value, minConfidence) - }, [isRemoteCamera, remoteIsPhoneConnected, remoteLatestFrame, columnCount, stability]) + // Convert to number + const value = digitsToNumber(digits) + + // Push to stability buffer + stability.pushFrame(value, minConfidence) + }) + }, [isRemoteCamera, remoteIsPhoneConnected, remoteLatestFrame, columnCount, stability, classifier]) // Local camera detection loop (only when enabled) useEffect(() => { - // Skip detection loop when feature is disabled + // Skip detection loop when feature is disabled or model not loaded if (!ENABLE_AUTO_DETECTION) return + if (!classifier.isModelLoaded) return if (!visionConfig.enabled || !isLocalCamera || !videoStream || !visionConfig.calibration) { return @@ -303,6 +321,7 @@ export function DockedVisionFeed({ onValueDetected, columnCount = 5 }: DockedVis const loop = () => { if (!running) return + // processLocalFrame is async but we don't await - it handles concurrency internally processLocalFrame() animationFrameRef.current = requestAnimationFrame(loop) } @@ -322,6 +341,7 @@ export function DockedVisionFeed({ onValueDetected, columnCount = 5 }: DockedVis videoStream, visionConfig.calibration, processLocalFrame, + classifier.isModelLoaded, ]) // Handle stable value changes (only when auto-detection is enabled) @@ -553,28 +573,40 @@ export function DockedVisionFeed({ onValueDetected, columnCount = 5 }: DockedVis backdropFilter: 'blur(4px)', })} > - {/* Detected value */} + {/* Model loading or detected value */}
- - {detectedValue !== null ? detectedValue : '---'} - - {detectedValue !== null && ( - - {Math.round(confidence * 100)}% + {classifier.isLoading ? ( + + Loading model... + ) : !classifier.isModelLoaded ? ( + + Model unavailable + + ) : ( + <> + + {detectedValue !== null ? detectedValue : '---'} + + {detectedValue !== null && ( + + {Math.round(confidence * 100)}% + + )} + )}
{/* Stability indicator */}
- {stability.consecutiveFrames > 0 && ( + {classifier.isModelLoaded && stability.consecutiveFrames > 0 && (
{Array.from({ length: 3 }).map((_, i) => (