diff --git a/apps/web/src/components/practice/ActiveSession.tsx b/apps/web/src/components/practice/ActiveSession.tsx
index 123c8ba6..e65180bd 100644
--- a/apps/web/src/components/practice/ActiveSession.tsx
+++ b/apps/web/src/components/practice/ActiveSession.tsx
@@ -1882,6 +1882,12 @@ export function ActiveSession({
: undefined
}
rejectedDigit={attempt.rejectedDigit}
+ detectedPrefixIndex={
+ // Show vision feedback for prefix sums (not final answer)
+ matchedPrefixIndex >= 0 && matchedPrefixIndex < prefixSums.length - 1
+ ? matchedPrefixIndex
+ : undefined
+ }
helpOverlay={
// Always render overlay when in help mode (for exit transition)
showHelpOverlay && helpContext ? (
diff --git a/apps/web/src/components/practice/VerticalProblem.tsx b/apps/web/src/components/practice/VerticalProblem.tsx
index 0ae4abc0..2bb6819e 100644
--- a/apps/web/src/components/practice/VerticalProblem.tsx
+++ b/apps/web/src/components/practice/VerticalProblem.tsx
@@ -40,6 +40,8 @@ interface VerticalProblemProps {
generationTrace?: GenerationTrace
/** Complexity budget constraint (for debug overlay) */
complexityBudget?: number
+ /** Index of detected prefix sum from vision (shows visual indicator on completed terms) */
+ detectedPrefixIndex?: number
}
/**
@@ -68,6 +70,7 @@ export function VerticalProblem({
answerFadingOut = false,
generationTrace,
complexityBudget,
+ detectedPrefixIndex,
}: VerticalProblemProps) {
const { resolvedTheme } = useTheme()
const isDark = resolvedTheme === 'dark'
@@ -163,12 +166,25 @@ export function VerticalProblem({
const showNeedHelp = index === needHelpTermIndex && !isCurrentHelp
// Check if this term is already included in the prefix sum (when in help mode)
const isInPrefixSum = currentHelpTermIndex !== undefined && index < currentHelpTermIndex
+ // Check if this term is completed based on vision detection (before help mode)
+ const isVisionCompleted =
+ detectedPrefixIndex !== undefined &&
+ currentHelpTermIndex === undefined && // Only show when NOT in help mode
+ index <= detectedPrefixIndex
return (
)}
+ {/* Checkmark indicator for vision-detected completed terms */}
+ {isVisionCompleted && (
+
+ ✓
+
+ )}
+
{/* Operator column (only show minus for negative) */}
(null)
const lastInferenceTimeRef = useRef
(0)
const lastBroadcastTimeRef = useRef(0)
+ const isInferringRef = useRef(false) // Prevent concurrent inference
const [videoStream, setVideoStream] = useState(null)
const [error, setError] = useState(null)
@@ -86,6 +67,16 @@ export function DockedVisionFeed({ onValueDetected, columnCount = 5 }: DockedVis
// Track video element in state for marker detection hook
const [videoElement, setVideoElement] = useState(null)
+ // ML column classifier hook
+ const classifier = useColumnClassifier()
+
+ // Preload the ML model when component mounts
+ useEffect(() => {
+ if (ENABLE_AUTO_DETECTION) {
+ classifier.preload()
+ }
+ }, [classifier])
+
// Stability tracking for detected values (hook must be called unconditionally)
const stability = useFrameStability()
@@ -225,9 +216,11 @@ export function DockedVisionFeed({ onValueDetected, columnCount = 5 }: DockedVis
}, [isRemoteCamera, remoteIsPhoneConnected])
// Process local camera frames for detection (only when enabled)
- const processLocalFrame = useCallback(() => {
- // Skip detection when feature is disabled
+ const processLocalFrame = useCallback(async () => {
+ // Skip detection when feature is disabled or model not ready
if (!ENABLE_AUTO_DETECTION) return
+ if (!classifier.isModelLoaded) return
+ if (isInferringRef.current) return // Skip if already inferring
const now = performance.now()
if (now - lastInferenceTimeRef.current < INFERENCE_INTERVAL_MS) {
@@ -239,25 +232,36 @@ export function DockedVisionFeed({ onValueDetected, columnCount = 5 }: DockedVis
if (!video || video.readyState < 2) return
if (!visionConfig.calibration) return
- // Process video frame into column strips
- const columnImages = processVideoFrame(video, visionConfig.calibration)
- if (columnImages.length === 0) return
+ isInferringRef.current = true
- // Use CV-based bead detection
- const analyses = analyzeColumns(columnImages)
- const { digits, minConfidence } = analysesToDigits(analyses)
+ try {
+ // Process video frame into column strips
+ const columnImages = processVideoFrame(video, visionConfig.calibration)
+ if (columnImages.length === 0) return
- // Convert to number
- const value = digitsToNumber(digits)
+ // Use ML-based digit classification
+ const results = await classifier.classifyColumns(columnImages)
+ if (!results || results.digits.length === 0) return
- // Push to stability buffer
- stability.pushFrame(value, minConfidence)
- }, [visionConfig.calibration, stability])
+ // Extract digits and minimum confidence
+ const { digits, confidences } = results
+ const minConfidence = Math.min(...confidences)
+
+ // Convert to number
+ const value = digitsToNumber(digits)
+
+ // Push to stability buffer
+ stability.pushFrame(value, minConfidence)
+ } finally {
+ isInferringRef.current = false
+ }
+ }, [visionConfig.calibration, stability, classifier])
// Process remote camera frames for detection (only when enabled)
useEffect(() => {
- // Skip detection when feature is disabled
+ // Skip detection when feature is disabled or model not ready
if (!ENABLE_AUTO_DETECTION) return
+ if (!classifier.isModelLoaded) return
if (!isRemoteCamera || !remoteIsPhoneConnected || !remoteLatestFrame) {
return
@@ -267,32 +271,46 @@ export function DockedVisionFeed({ onValueDetected, columnCount = 5 }: DockedVis
if (now - lastInferenceTimeRef.current < INFERENCE_INTERVAL_MS) {
return
}
- lastInferenceTimeRef.current = now
const image = remoteImageRef.current
if (!image || !image.complete || image.naturalWidth === 0) {
return
}
+ // Prevent concurrent inference
+ if (isInferringRef.current) return
+ isInferringRef.current = true
+ lastInferenceTimeRef.current = now
+
// Phone sends pre-cropped frames in auto mode, so no calibration needed
const columnImages = processImageFrame(image, null, columnCount)
- if (columnImages.length === 0) return
+ if (columnImages.length === 0) {
+ isInferringRef.current = false
+ return
+ }
- // Use CV-based bead detection
- const analyses = analyzeColumns(columnImages)
- const { digits, minConfidence } = analysesToDigits(analyses)
+ // Use ML-based digit classification (async)
+ classifier.classifyColumns(columnImages).then((results) => {
+ isInferringRef.current = false
+ if (!results || results.digits.length === 0) return
- // Convert to number
- const value = digitsToNumber(digits)
+ // Extract digits and minimum confidence
+ const { digits, confidences } = results
+ const minConfidence = Math.min(...confidences)
- // Push to stability buffer
- stability.pushFrame(value, minConfidence)
- }, [isRemoteCamera, remoteIsPhoneConnected, remoteLatestFrame, columnCount, stability])
+ // Convert to number
+ const value = digitsToNumber(digits)
+
+ // Push to stability buffer
+ stability.pushFrame(value, minConfidence)
+ })
+ }, [isRemoteCamera, remoteIsPhoneConnected, remoteLatestFrame, columnCount, stability, classifier])
// Local camera detection loop (only when enabled)
useEffect(() => {
- // Skip detection loop when feature is disabled
+ // Skip detection loop when feature is disabled or model not loaded
if (!ENABLE_AUTO_DETECTION) return
+ if (!classifier.isModelLoaded) return
if (!visionConfig.enabled || !isLocalCamera || !videoStream || !visionConfig.calibration) {
return
@@ -303,6 +321,7 @@ export function DockedVisionFeed({ onValueDetected, columnCount = 5 }: DockedVis
const loop = () => {
if (!running) return
+ // processLocalFrame is async but we don't await - it handles concurrency internally
processLocalFrame()
animationFrameRef.current = requestAnimationFrame(loop)
}
@@ -322,6 +341,7 @@ export function DockedVisionFeed({ onValueDetected, columnCount = 5 }: DockedVis
videoStream,
visionConfig.calibration,
processLocalFrame,
+ classifier.isModelLoaded,
])
// Handle stable value changes (only when auto-detection is enabled)
@@ -553,28 +573,40 @@ export function DockedVisionFeed({ onValueDetected, columnCount = 5 }: DockedVis
backdropFilter: 'blur(4px)',
})}
>
- {/* Detected value */}
+ {/* Model loading or detected value */}
-
- {detectedValue !== null ? detectedValue : '---'}
-
- {detectedValue !== null && (
-
- {Math.round(confidence * 100)}%
+ {classifier.isLoading ? (
+
+ Loading model...
+ ) : !classifier.isModelLoaded ? (
+
+ Model unavailable
+
+ ) : (
+ <>
+
+ {detectedValue !== null ? detectedValue : '---'}
+
+ {detectedValue !== null && (
+
+ {Math.round(confidence * 100)}%
+
+ )}
+ >
)}
{/* Stability indicator */}
- {stability.consecutiveFrames > 0 && (
+ {classifier.isModelLoaded && stability.consecutiveFrames > 0 && (
{Array.from({ length: 3 }).map((_, i) => (