feat(vision): integrate vision feed into docked abacus
- Add vision state management to MyAbacusContext (camera, calibration, remote session, enabled state) - Add VisionIndicator component showing vision status on dock header - Add VisionSetupModal for configuring camera and calibration - Add DockedVisionFeed component that replaces SVG abacus when vision is enabled, with: - Continuous ArUco marker detection for auto-calibration - OpenCV perspective correction via VisionCameraFeed - Real-time bead detection and value display - Support for both local camera and remote phone camera - Wire AbacusVisionBridge to save config to context via onConfigurationChange callback - Update MyAbacus to conditionally render DockedVisionFeed vs AbacusReact based on vision state 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Thomas Hallock
@@ -9,6 +9,9 @@ import { createRoot } from 'react-dom/client'
|
||||
import { HomeHeroContext } from '@/contexts/HomeHeroContext'
|
||||
import { type DockAnimationState, useMyAbacus } from '@/contexts/MyAbacusContext'
|
||||
import { useTheme } from '@/contexts/ThemeContext'
|
||||
import { DockedVisionFeed } from '@/components/vision/DockedVisionFeed'
|
||||
import { VisionIndicator } from '@/components/vision/VisionIndicator'
|
||||
import { VisionSetupModal } from '@/components/vision/VisionSetupModal'
|
||||
import { css } from '../../styled-system/css'
|
||||
|
||||
/**
|
||||
@@ -85,6 +88,8 @@ export function MyAbacus() {
|
||||
clearDockRequest,
|
||||
abacusValue: contextAbacusValue,
|
||||
setDockedValue,
|
||||
visionConfig,
|
||||
isVisionSetupComplete,
|
||||
} = useMyAbacus()
|
||||
const appConfig = useAbacusConfig()
|
||||
const pathname = usePathname()
|
||||
@@ -493,6 +498,9 @@ export function MyAbacus() {
|
||||
position: 'relative',
|
||||
})}
|
||||
>
|
||||
{/* Vision indicator - positioned at top-right, before undock button */}
|
||||
<VisionIndicator size="small" position="top-left" />
|
||||
|
||||
{/* Undock button - positioned at top-right of dock container */}
|
||||
<button
|
||||
data-action="undock-abacus"
|
||||
@@ -536,44 +544,67 @@ export function MyAbacus() {
|
||||
data-element="abacus-display"
|
||||
className={css({
|
||||
filter: 'drop-shadow(0 4px 12px rgba(251, 191, 36, 0.2))',
|
||||
width: '100%',
|
||||
height: '100%',
|
||||
})}
|
||||
>
|
||||
<AbacusReact
|
||||
key="docked"
|
||||
value={dock.value ?? abacusValue}
|
||||
defaultValue={dock.defaultValue}
|
||||
columns={dock.columns ?? 5}
|
||||
scaleFactor={effectiveScaleFactor}
|
||||
beadShape={appConfig.beadShape}
|
||||
showNumbers={dock.showNumbers ?? true}
|
||||
interactive={dock.interactive ?? true}
|
||||
animated={dock.animated ?? true}
|
||||
customStyles={structuralStyles}
|
||||
onValueChange={(newValue: number | bigint) => {
|
||||
const numValue = Number(newValue)
|
||||
// Update the appropriate state based on dock mode
|
||||
// (unless dock provides its own value prop for full control)
|
||||
if (dock.value === undefined) {
|
||||
// When docked by user, update context value; otherwise update local/hero
|
||||
if (isDockedByUser) {
|
||||
setDockedValue(numValue)
|
||||
} else {
|
||||
setAbacusValue(numValue)
|
||||
{/* Show vision feed when enabled, otherwise show digital abacus */}
|
||||
{visionConfig.enabled && isVisionSetupComplete ? (
|
||||
<DockedVisionFeed
|
||||
columnCount={dock.columns ?? 5}
|
||||
onValueDetected={(value) => {
|
||||
// Update the appropriate state based on dock mode
|
||||
if (dock.value === undefined) {
|
||||
if (isDockedByUser) {
|
||||
setDockedValue(value)
|
||||
} else {
|
||||
setAbacusValue(value)
|
||||
}
|
||||
}
|
||||
}
|
||||
// Also call dock's callback if provided
|
||||
if (dock.onValueChange) {
|
||||
dock.onValueChange(numValue)
|
||||
}
|
||||
}}
|
||||
enhanced3d="realistic"
|
||||
material3d={{
|
||||
heavenBeads: 'glossy',
|
||||
earthBeads: 'satin',
|
||||
lighting: 'dramatic',
|
||||
woodGrain: true,
|
||||
}}
|
||||
/>
|
||||
// Also call dock's callback if provided
|
||||
if (dock.onValueChange) {
|
||||
dock.onValueChange(value)
|
||||
}
|
||||
}}
|
||||
/>
|
||||
) : (
|
||||
<AbacusReact
|
||||
key="docked"
|
||||
value={dock.value ?? abacusValue}
|
||||
defaultValue={dock.defaultValue}
|
||||
columns={dock.columns ?? 5}
|
||||
scaleFactor={effectiveScaleFactor}
|
||||
beadShape={appConfig.beadShape}
|
||||
showNumbers={dock.showNumbers ?? true}
|
||||
interactive={dock.interactive ?? true}
|
||||
animated={dock.animated ?? true}
|
||||
customStyles={structuralStyles}
|
||||
onValueChange={(newValue: number | bigint) => {
|
||||
const numValue = Number(newValue)
|
||||
// Update the appropriate state based on dock mode
|
||||
// (unless dock provides its own value prop for full control)
|
||||
if (dock.value === undefined) {
|
||||
// When docked by user, update context value; otherwise update local/hero
|
||||
if (isDockedByUser) {
|
||||
setDockedValue(numValue)
|
||||
} else {
|
||||
setAbacusValue(numValue)
|
||||
}
|
||||
}
|
||||
// Also call dock's callback if provided
|
||||
if (dock.onValueChange) {
|
||||
dock.onValueChange(numValue)
|
||||
}
|
||||
}}
|
||||
enhanced3d="realistic"
|
||||
material3d={{
|
||||
heavenBeads: 'glossy',
|
||||
earthBeads: 'satin',
|
||||
lighting: 'dramatic',
|
||||
woodGrain: true,
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
</div>,
|
||||
dock.element
|
||||
@@ -820,6 +851,9 @@ export function MyAbacus() {
|
||||
`,
|
||||
}}
|
||||
/>
|
||||
|
||||
{/* Vision setup modal - controlled by context state */}
|
||||
<VisionSetupModal />
|
||||
</>
|
||||
)
|
||||
}
|
||||
|
||||
@@ -19,6 +19,18 @@ import { VisionStatusIndicator } from './VisionStatusIndicator'
|
||||
|
||||
type CameraSource = 'local' | 'phone'
|
||||
|
||||
/**
|
||||
* Configuration change payload for onConfigurationChange callback
|
||||
*/
|
||||
export interface VisionConfigurationChange {
|
||||
/** Camera device ID (for local camera) */
|
||||
cameraDeviceId?: string | null
|
||||
/** Calibration grid */
|
||||
calibration?: import('@/types/vision').CalibrationGrid | null
|
||||
/** Remote camera session ID (for phone camera) */
|
||||
remoteCameraSessionId?: string | null
|
||||
}
|
||||
|
||||
export interface AbacusVisionBridgeProps {
|
||||
/** Number of abacus columns to detect */
|
||||
columnCount: number
|
||||
@@ -28,6 +40,8 @@ export interface AbacusVisionBridgeProps {
|
||||
onClose: () => void
|
||||
/** Called on error */
|
||||
onError?: (error: string) => void
|
||||
/** Called when configuration changes (camera, calibration, or remote session) */
|
||||
onConfigurationChange?: (config: VisionConfigurationChange) => void
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -44,6 +58,7 @@ export function AbacusVisionBridge({
|
||||
onValueDetected,
|
||||
onClose,
|
||||
onError,
|
||||
onConfigurationChange,
|
||||
}: AbacusVisionBridgeProps): ReactNode {
|
||||
const [videoDimensions, setVideoDimensions] = useState<{
|
||||
width: number
|
||||
@@ -110,6 +125,11 @@ export function AbacusVisionBridge({
|
||||
const lastRemoteInferenceTimeRef = useRef<number>(0)
|
||||
const REMOTE_INFERENCE_INTERVAL_MS = 100 // 10fps
|
||||
|
||||
// Track last reported configuration to avoid redundant callbacks
|
||||
const lastReportedCameraRef = useRef<string | null>(null)
|
||||
const lastReportedCalibrationRef = useRef<CalibrationGrid | null>(null)
|
||||
const lastReportedRemoteSessionRef = useRef<string | null>(null)
|
||||
|
||||
// Handle switching to phone camera
|
||||
const handleCameraSourceChange = useCallback(
|
||||
(source: CameraSource) => {
|
||||
@@ -208,6 +228,54 @@ export function AbacusVisionBridge({
|
||||
}
|
||||
}, [vision.cameraError, onError])
|
||||
|
||||
// Notify about local camera device changes
|
||||
useEffect(() => {
|
||||
if (
|
||||
cameraSource === 'local' &&
|
||||
vision.selectedDeviceId &&
|
||||
vision.selectedDeviceId !== lastReportedCameraRef.current
|
||||
) {
|
||||
lastReportedCameraRef.current = vision.selectedDeviceId
|
||||
onConfigurationChange?.({ cameraDeviceId: vision.selectedDeviceId })
|
||||
}
|
||||
}, [cameraSource, vision.selectedDeviceId, onConfigurationChange])
|
||||
|
||||
// Notify about local calibration changes
|
||||
useEffect(() => {
|
||||
if (
|
||||
cameraSource === 'local' &&
|
||||
vision.calibrationGrid &&
|
||||
vision.calibrationGrid !== lastReportedCalibrationRef.current
|
||||
) {
|
||||
lastReportedCalibrationRef.current = vision.calibrationGrid
|
||||
onConfigurationChange?.({ calibration: vision.calibrationGrid })
|
||||
}
|
||||
}, [cameraSource, vision.calibrationGrid, onConfigurationChange])
|
||||
|
||||
// Notify about remote camera session changes
|
||||
useEffect(() => {
|
||||
if (
|
||||
cameraSource === 'phone' &&
|
||||
remoteCameraSessionId &&
|
||||
remoteCameraSessionId !== lastReportedRemoteSessionRef.current
|
||||
) {
|
||||
lastReportedRemoteSessionRef.current = remoteCameraSessionId
|
||||
onConfigurationChange?.({ remoteCameraSessionId })
|
||||
}
|
||||
}, [cameraSource, remoteCameraSessionId, onConfigurationChange])
|
||||
|
||||
// Notify about remote calibration changes (manual mode)
|
||||
useEffect(() => {
|
||||
if (
|
||||
cameraSource === 'phone' &&
|
||||
remoteCalibration &&
|
||||
remoteCalibration !== lastReportedCalibrationRef.current
|
||||
) {
|
||||
lastReportedCalibrationRef.current = remoteCalibration
|
||||
onConfigurationChange?.({ calibration: remoteCalibration })
|
||||
}
|
||||
}, [cameraSource, remoteCalibration, onConfigurationChange])
|
||||
|
||||
// Process remote camera frames through CV pipeline
|
||||
useEffect(() => {
|
||||
// Only process when using phone camera and connected
|
||||
|
||||
555
apps/web/src/components/vision/DockedVisionFeed.tsx
Normal file
555
apps/web/src/components/vision/DockedVisionFeed.tsx
Normal file
@@ -0,0 +1,555 @@
|
||||
'use client'
|
||||
|
||||
import { useCallback, useEffect, useRef, useState } from 'react'
|
||||
import { useMyAbacus } from '@/contexts/MyAbacusContext'
|
||||
import { useRemoteCameraDesktop } from '@/hooks/useRemoteCameraDesktop'
|
||||
import { analyzeColumns, analysesToDigits, digitsToNumber } from '@/lib/vision/beadDetector'
|
||||
import { processVideoFrame, processImageFrame } from '@/lib/vision/frameProcessor'
|
||||
import {
|
||||
cleanupArucoDetector,
|
||||
detectMarkers,
|
||||
initArucoDetector,
|
||||
isArucoAvailable,
|
||||
loadAruco,
|
||||
} from '@/lib/vision/arucoDetection'
|
||||
import { useFrameStability } from '@/hooks/useFrameStability'
|
||||
import { VisionCameraFeed } from './VisionCameraFeed'
|
||||
import { css } from '../../../styled-system/css'
|
||||
import type { CalibrationGrid } from '@/types/vision'
|
||||
|
||||
interface DockedVisionFeedProps {
|
||||
/** Called when a stable value is detected */
|
||||
onValueDetected?: (value: number) => void
|
||||
/** Number of columns to detect */
|
||||
columnCount?: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Renders the processed camera feed in place of the docked abacus
|
||||
*
|
||||
* When vision is enabled in MyAbacusContext, this component:
|
||||
* - For local camera: Opens the saved camera, applies calibration, runs detection
|
||||
* - For remote camera: Receives frames from phone, runs detection
|
||||
* - Shows the video feed with detection overlay
|
||||
*/
|
||||
export function DockedVisionFeed({ onValueDetected, columnCount = 5 }: DockedVisionFeedProps) {
|
||||
const { visionConfig, setDockedValue, setVisionEnabled, setVisionCalibration } = useMyAbacus()
|
||||
|
||||
const videoRef = useRef<HTMLVideoElement>(null)
|
||||
const remoteImageRef = useRef<HTMLImageElement>(null)
|
||||
const animationFrameRef = useRef<number | null>(null)
|
||||
const markerDetectionFrameRef = useRef<number | null>(null)
|
||||
const lastInferenceTimeRef = useRef<number>(0)
|
||||
|
||||
const [videoStream, setVideoStream] = useState<MediaStream | null>(null)
|
||||
const [error, setError] = useState<string | null>(null)
|
||||
const [isLoading, setIsLoading] = useState(true)
|
||||
const [detectedValue, setDetectedValue] = useState<number | null>(null)
|
||||
const [confidence, setConfidence] = useState(0)
|
||||
const [isArucoReady, setIsArucoReady] = useState(false)
|
||||
const [markersFound, setMarkersFound] = useState(0)
|
||||
|
||||
const stability = useFrameStability()
|
||||
|
||||
// Determine camera source (must be before effects that use these)
|
||||
// Prioritize local camera if configured - remote camera only if no local camera
|
||||
const isLocalCamera = visionConfig.cameraDeviceId !== null
|
||||
const isRemoteCamera = !isLocalCamera && visionConfig.remoteCameraSessionId !== null
|
||||
|
||||
// Load and initialize ArUco on mount (for local camera auto-calibration)
|
||||
useEffect(() => {
|
||||
if (!isLocalCamera) return
|
||||
|
||||
let cancelled = false
|
||||
|
||||
const initAruco = async () => {
|
||||
try {
|
||||
await loadAruco()
|
||||
if (cancelled) return
|
||||
|
||||
const available = isArucoAvailable()
|
||||
if (available) {
|
||||
initArucoDetector()
|
||||
setIsArucoReady(true)
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('[DockedVisionFeed] Failed to load ArUco:', err)
|
||||
}
|
||||
}
|
||||
|
||||
initAruco()
|
||||
return () => {
|
||||
cancelled = true
|
||||
}
|
||||
}, [isLocalCamera])
|
||||
|
||||
// Cleanup ArUco detector on unmount
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
cleanupArucoDetector()
|
||||
}
|
||||
}, [])
|
||||
|
||||
// Auto-calibration loop using ArUco markers (for local camera)
|
||||
useEffect(() => {
|
||||
if (!visionConfig.enabled || !isLocalCamera || !videoStream || !isArucoReady) {
|
||||
if (markerDetectionFrameRef.current) {
|
||||
cancelAnimationFrame(markerDetectionFrameRef.current)
|
||||
markerDetectionFrameRef.current = null
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
const video = videoRef.current
|
||||
if (!video) return
|
||||
|
||||
let running = true
|
||||
|
||||
const detectLoop = () => {
|
||||
if (!running || !video || video.readyState < 2) {
|
||||
if (running) {
|
||||
markerDetectionFrameRef.current = requestAnimationFrame(detectLoop)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
const result = detectMarkers(video)
|
||||
setMarkersFound(result.markersFound)
|
||||
|
||||
// Auto-update calibration when all 4 markers found
|
||||
if (result.allMarkersFound && result.quadCorners) {
|
||||
const grid: CalibrationGrid = {
|
||||
roi: {
|
||||
x: Math.min(result.quadCorners.topLeft.x, result.quadCorners.bottomLeft.x),
|
||||
y: Math.min(result.quadCorners.topLeft.y, result.quadCorners.topRight.y),
|
||||
width:
|
||||
Math.max(result.quadCorners.topRight.x, result.quadCorners.bottomRight.x) -
|
||||
Math.min(result.quadCorners.topLeft.x, result.quadCorners.bottomLeft.x),
|
||||
height:
|
||||
Math.max(result.quadCorners.bottomLeft.y, result.quadCorners.bottomRight.y) -
|
||||
Math.min(result.quadCorners.topLeft.y, result.quadCorners.topRight.y),
|
||||
},
|
||||
corners: result.quadCorners,
|
||||
columnCount,
|
||||
columnDividers: Array.from({ length: columnCount - 1 }, (_, i) => (i + 1) / columnCount),
|
||||
rotation: 0,
|
||||
}
|
||||
// Update calibration in context
|
||||
setVisionCalibration(grid)
|
||||
}
|
||||
|
||||
markerDetectionFrameRef.current = requestAnimationFrame(detectLoop)
|
||||
}
|
||||
|
||||
detectLoop()
|
||||
|
||||
return () => {
|
||||
running = false
|
||||
if (markerDetectionFrameRef.current) {
|
||||
cancelAnimationFrame(markerDetectionFrameRef.current)
|
||||
markerDetectionFrameRef.current = null
|
||||
}
|
||||
}
|
||||
}, [visionConfig.enabled, isLocalCamera, videoStream, isArucoReady, columnCount, setVisionCalibration])
|
||||
|
||||
// Remote camera hook
|
||||
const {
|
||||
isPhoneConnected: remoteIsPhoneConnected,
|
||||
latestFrame: remoteLatestFrame,
|
||||
subscribe: remoteSubscribe,
|
||||
unsubscribe: remoteUnsubscribe,
|
||||
} = useRemoteCameraDesktop()
|
||||
|
||||
const INFERENCE_INTERVAL_MS = 100 // 10fps
|
||||
|
||||
// Start local camera when component mounts (only for local camera)
|
||||
useEffect(() => {
|
||||
if (!visionConfig.enabled || !isLocalCamera || !visionConfig.cameraDeviceId) {
|
||||
return
|
||||
}
|
||||
|
||||
let cancelled = false
|
||||
setIsLoading(true)
|
||||
setError(null)
|
||||
|
||||
const startCamera = async () => {
|
||||
try {
|
||||
const stream = await navigator.mediaDevices.getUserMedia({
|
||||
video: {
|
||||
deviceId: { exact: visionConfig.cameraDeviceId! },
|
||||
width: { ideal: 1280 },
|
||||
height: { ideal: 720 },
|
||||
},
|
||||
})
|
||||
|
||||
if (cancelled) {
|
||||
stream.getTracks().forEach((track) => track.stop())
|
||||
return
|
||||
}
|
||||
|
||||
setVideoStream(stream)
|
||||
setIsLoading(false)
|
||||
} catch (err) {
|
||||
if (cancelled) return
|
||||
console.error('[DockedVisionFeed] Failed to start camera:', err)
|
||||
setError('Failed to access camera')
|
||||
setIsLoading(false)
|
||||
}
|
||||
}
|
||||
|
||||
startCamera()
|
||||
|
||||
return () => {
|
||||
cancelled = true
|
||||
}
|
||||
}, [visionConfig.enabled, isLocalCamera, visionConfig.cameraDeviceId])
|
||||
|
||||
// Stop camera when stream changes or component unmounts
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
if (videoStream) {
|
||||
videoStream.getTracks().forEach((track) => track.stop())
|
||||
}
|
||||
}
|
||||
}, [videoStream])
|
||||
|
||||
// Attach stream to video element
|
||||
useEffect(() => {
|
||||
if (videoRef.current && videoStream) {
|
||||
videoRef.current.srcObject = videoStream
|
||||
}
|
||||
}, [videoStream])
|
||||
|
||||
// Subscribe to remote camera session
|
||||
useEffect(() => {
|
||||
if (!visionConfig.enabled || !isRemoteCamera || !visionConfig.remoteCameraSessionId) {
|
||||
return
|
||||
}
|
||||
|
||||
setIsLoading(true)
|
||||
remoteSubscribe(visionConfig.remoteCameraSessionId)
|
||||
|
||||
return () => {
|
||||
remoteUnsubscribe()
|
||||
}
|
||||
}, [visionConfig.enabled, isRemoteCamera, visionConfig.remoteCameraSessionId, remoteSubscribe, remoteUnsubscribe])
|
||||
|
||||
// Update loading state when remote camera connects
|
||||
useEffect(() => {
|
||||
if (isRemoteCamera && remoteIsPhoneConnected) {
|
||||
setIsLoading(false)
|
||||
}
|
||||
}, [isRemoteCamera, remoteIsPhoneConnected])
|
||||
|
||||
// Process local camera frames for detection
|
||||
const processLocalFrame = useCallback(() => {
|
||||
const now = performance.now()
|
||||
if (now - lastInferenceTimeRef.current < INFERENCE_INTERVAL_MS) {
|
||||
return
|
||||
}
|
||||
lastInferenceTimeRef.current = now
|
||||
|
||||
const video = videoRef.current
|
||||
if (!video || video.readyState < 2) return
|
||||
if (!visionConfig.calibration) return
|
||||
|
||||
// Process video frame into column strips
|
||||
const columnImages = processVideoFrame(video, visionConfig.calibration)
|
||||
if (columnImages.length === 0) return
|
||||
|
||||
// Use CV-based bead detection
|
||||
const analyses = analyzeColumns(columnImages)
|
||||
const { digits, minConfidence } = analysesToDigits(analyses)
|
||||
|
||||
// Convert to number
|
||||
const value = digitsToNumber(digits)
|
||||
|
||||
// Push to stability buffer
|
||||
stability.pushFrame(value, minConfidence)
|
||||
}, [visionConfig.calibration, stability])
|
||||
|
||||
// Process remote camera frames for detection
|
||||
useEffect(() => {
|
||||
if (!isRemoteCamera || !remoteIsPhoneConnected || !remoteLatestFrame) {
|
||||
return
|
||||
}
|
||||
|
||||
const now = performance.now()
|
||||
if (now - lastInferenceTimeRef.current < INFERENCE_INTERVAL_MS) {
|
||||
return
|
||||
}
|
||||
lastInferenceTimeRef.current = now
|
||||
|
||||
const image = remoteImageRef.current
|
||||
if (!image || !image.complete || image.naturalWidth === 0) {
|
||||
return
|
||||
}
|
||||
|
||||
// Phone sends pre-cropped frames in auto mode, so no calibration needed
|
||||
const columnImages = processImageFrame(image, null, columnCount)
|
||||
if (columnImages.length === 0) return
|
||||
|
||||
// Use CV-based bead detection
|
||||
const analyses = analyzeColumns(columnImages)
|
||||
const { digits, minConfidence } = analysesToDigits(analyses)
|
||||
|
||||
// Convert to number
|
||||
const value = digitsToNumber(digits)
|
||||
|
||||
// Push to stability buffer
|
||||
stability.pushFrame(value, minConfidence)
|
||||
}, [isRemoteCamera, remoteIsPhoneConnected, remoteLatestFrame, columnCount, stability])
|
||||
|
||||
// Local camera detection loop
|
||||
useEffect(() => {
|
||||
if (!visionConfig.enabled || !isLocalCamera || !videoStream || !visionConfig.calibration) {
|
||||
return
|
||||
}
|
||||
|
||||
let running = true
|
||||
|
||||
const loop = () => {
|
||||
if (!running) return
|
||||
|
||||
processLocalFrame()
|
||||
animationFrameRef.current = requestAnimationFrame(loop)
|
||||
}
|
||||
|
||||
loop()
|
||||
|
||||
return () => {
|
||||
running = false
|
||||
if (animationFrameRef.current) {
|
||||
cancelAnimationFrame(animationFrameRef.current)
|
||||
animationFrameRef.current = null
|
||||
}
|
||||
}
|
||||
}, [visionConfig.enabled, isLocalCamera, videoStream, visionConfig.calibration, processLocalFrame])
|
||||
|
||||
// Handle stable value changes
|
||||
useEffect(() => {
|
||||
if (stability.stableValue !== null && stability.stableValue !== detectedValue) {
|
||||
setDetectedValue(stability.stableValue)
|
||||
setConfidence(stability.currentConfidence)
|
||||
setDockedValue(stability.stableValue)
|
||||
onValueDetected?.(stability.stableValue)
|
||||
}
|
||||
}, [stability.stableValue, stability.currentConfidence, detectedValue, setDockedValue, onValueDetected])
|
||||
|
||||
const handleDisableVision = (e: React.MouseEvent) => {
|
||||
e.stopPropagation()
|
||||
setVisionEnabled(false)
|
||||
if (videoStream) {
|
||||
videoStream.getTracks().forEach((track) => track.stop())
|
||||
}
|
||||
}
|
||||
|
||||
if (error) {
|
||||
return (
|
||||
<div
|
||||
data-component="docked-vision-feed"
|
||||
data-status="error"
|
||||
className={css({
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
gap: 2,
|
||||
p: 4,
|
||||
bg: 'red.900/30',
|
||||
borderRadius: 'lg',
|
||||
color: 'red.400',
|
||||
textAlign: 'center',
|
||||
})}
|
||||
>
|
||||
<span className={css({ fontSize: 'xl' })}>⚠️</span>
|
||||
<span className={css({ fontSize: 'sm' })}>{error}</span>
|
||||
<button
|
||||
type="button"
|
||||
onClick={handleDisableVision}
|
||||
className={css({
|
||||
mt: 2,
|
||||
px: 3,
|
||||
py: 1,
|
||||
bg: 'gray.700',
|
||||
color: 'white',
|
||||
borderRadius: 'md',
|
||||
fontSize: 'xs',
|
||||
border: 'none',
|
||||
cursor: 'pointer',
|
||||
})}
|
||||
>
|
||||
Disable Vision
|
||||
</button>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
if (isLoading) {
|
||||
return (
|
||||
<div
|
||||
data-component="docked-vision-feed"
|
||||
data-status="loading"
|
||||
className={css({
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
gap: 2,
|
||||
p: 4,
|
||||
bg: 'gray.800/50',
|
||||
borderRadius: 'lg',
|
||||
color: 'gray.400',
|
||||
})}
|
||||
>
|
||||
<span className={css({ fontSize: 'xl' })}>📷</span>
|
||||
<span className={css({ fontSize: 'sm' })}>
|
||||
{isRemoteCamera ? 'Connecting to phone...' : 'Starting camera...'}
|
||||
</span>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
return (
|
||||
<div
|
||||
data-component="docked-vision-feed"
|
||||
data-status="active"
|
||||
data-source={isRemoteCamera ? 'remote' : 'local'}
|
||||
className={css({
|
||||
position: 'relative',
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
alignItems: 'center',
|
||||
overflow: 'hidden',
|
||||
borderRadius: 'lg',
|
||||
bg: 'black',
|
||||
width: '100%',
|
||||
height: '100%',
|
||||
})}
|
||||
>
|
||||
{/* Rectified video feed - local camera */}
|
||||
{isLocalCamera && (
|
||||
<VisionCameraFeed
|
||||
videoStream={videoStream}
|
||||
calibration={visionConfig.calibration}
|
||||
showRectifiedView={true}
|
||||
videoRef={(el) => {
|
||||
videoRef.current = el
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
|
||||
{/* Remote camera feed */}
|
||||
{isRemoteCamera && remoteLatestFrame && (
|
||||
<img
|
||||
ref={remoteImageRef}
|
||||
src={`data:image/jpeg;base64,${remoteLatestFrame.imageData}`}
|
||||
alt="Phone camera view"
|
||||
className={css({
|
||||
width: '100%',
|
||||
height: 'auto',
|
||||
objectFit: 'contain',
|
||||
})}
|
||||
/>
|
||||
)}
|
||||
|
||||
{/* Waiting for remote frames */}
|
||||
{isRemoteCamera && !remoteLatestFrame && remoteIsPhoneConnected && (
|
||||
<div
|
||||
className={css({
|
||||
width: '100%',
|
||||
aspectRatio: '2/1',
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
color: 'gray.400',
|
||||
fontSize: 'sm',
|
||||
})}
|
||||
>
|
||||
Waiting for frames...
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Detection overlay */}
|
||||
<div
|
||||
data-element="detection-overlay"
|
||||
className={css({
|
||||
position: 'absolute',
|
||||
bottom: 0,
|
||||
left: 0,
|
||||
right: 0,
|
||||
display: 'flex',
|
||||
justifyContent: 'space-between',
|
||||
alignItems: 'center',
|
||||
p: 2,
|
||||
bg: 'rgba(0, 0, 0, 0.7)',
|
||||
backdropFilter: 'blur(4px)',
|
||||
})}
|
||||
>
|
||||
{/* Detected value */}
|
||||
<div className={css({ display: 'flex', alignItems: 'center', gap: 2 })}>
|
||||
<span className={css({ fontSize: 'lg', fontWeight: 'bold', color: 'white', fontFamily: 'mono' })}>
|
||||
{detectedValue !== null ? detectedValue : '---'}
|
||||
</span>
|
||||
{detectedValue !== null && (
|
||||
<span className={css({ fontSize: 'xs', color: 'gray.400' })}>
|
||||
{Math.round(confidence * 100)}%
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Stability indicator */}
|
||||
<div className={css({ display: 'flex', alignItems: 'center', gap: 1 })}>
|
||||
{stability.consecutiveFrames > 0 && (
|
||||
<div className={css({ display: 'flex', gap: 0.5 })}>
|
||||
{Array.from({ length: 3 }).map((_, i) => (
|
||||
<div
|
||||
key={i}
|
||||
className={css({
|
||||
w: '6px',
|
||||
h: '6px',
|
||||
borderRadius: 'full',
|
||||
bg: i < stability.consecutiveFrames ? 'green.500' : 'gray.600',
|
||||
})}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Disable button */}
|
||||
<button
|
||||
type="button"
|
||||
data-action="disable-vision"
|
||||
onClick={handleDisableVision}
|
||||
title="Disable vision mode"
|
||||
className={css({
|
||||
position: 'absolute',
|
||||
top: '4px',
|
||||
right: '4px',
|
||||
w: '24px',
|
||||
h: '24px',
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
bg: 'rgba(0, 0, 0, 0.5)',
|
||||
backdropFilter: 'blur(4px)',
|
||||
border: '1px solid rgba(255, 255, 255, 0.3)',
|
||||
borderRadius: 'md',
|
||||
color: 'white',
|
||||
fontSize: 'xs',
|
||||
cursor: 'pointer',
|
||||
zIndex: 10,
|
||||
opacity: 0.7,
|
||||
_hover: {
|
||||
bg: 'rgba(239, 68, 68, 0.8)',
|
||||
opacity: 1,
|
||||
},
|
||||
})}
|
||||
>
|
||||
✕
|
||||
</button>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
123
apps/web/src/components/vision/VisionIndicator.tsx
Normal file
123
apps/web/src/components/vision/VisionIndicator.tsx
Normal file
@@ -0,0 +1,123 @@
|
||||
'use client'
|
||||
|
||||
import { useMyAbacus } from '@/contexts/MyAbacusContext'
|
||||
import { css } from '../../../styled-system/css'
|
||||
|
||||
interface VisionIndicatorProps {
|
||||
/** Size variant */
|
||||
size?: 'small' | 'medium'
|
||||
/** Position for absolute placement */
|
||||
position?: 'top-left' | 'bottom-right'
|
||||
}
|
||||
|
||||
/**
|
||||
* Camera icon indicator for abacus vision mode
|
||||
*
|
||||
* Shows:
|
||||
* - 🔴 Red dot = not configured (no camera or no calibration)
|
||||
* - 🟢 Green dot = configured and enabled
|
||||
* - ⚪ Gray = configured but disabled
|
||||
*
|
||||
* Click behavior:
|
||||
* - If not configured: opens setup modal
|
||||
* - If configured: toggles vision on/off
|
||||
*/
|
||||
export function VisionIndicator({ size = 'medium', position = 'bottom-right' }: VisionIndicatorProps) {
|
||||
const { visionConfig, isVisionSetupComplete, openVisionSetup } = useMyAbacus()
|
||||
|
||||
const handleClick = (e: React.MouseEvent) => {
|
||||
e.stopPropagation()
|
||||
console.log('[VisionIndicator] Click detected, isVisionSetupComplete:', isVisionSetupComplete)
|
||||
|
||||
// Always open setup modal on click for now
|
||||
// This gives users easy access to vision settings
|
||||
// In the future, we could add quick-toggle behavior
|
||||
console.log('[VisionIndicator] Opening setup modal...')
|
||||
openVisionSetup()
|
||||
}
|
||||
|
||||
const handleContextMenu = (e: React.MouseEvent) => {
|
||||
e.preventDefault()
|
||||
e.stopPropagation()
|
||||
// Right-click always opens setup
|
||||
openVisionSetup()
|
||||
}
|
||||
|
||||
// Determine status indicator color
|
||||
const statusColor = !isVisionSetupComplete
|
||||
? 'red.500' // Not configured
|
||||
: visionConfig.enabled
|
||||
? 'green.500' // Enabled
|
||||
: 'gray.400' // Configured but disabled
|
||||
|
||||
const statusLabel = !isVisionSetupComplete
|
||||
? 'Vision not configured'
|
||||
: visionConfig.enabled
|
||||
? 'Vision enabled'
|
||||
: 'Vision disabled'
|
||||
|
||||
const sizeStyles =
|
||||
size === 'small'
|
||||
? { w: '20px', h: '20px', fontSize: '10px' }
|
||||
: { w: '28px', h: '28px', fontSize: '14px' }
|
||||
|
||||
const positionStyles =
|
||||
position === 'top-left'
|
||||
? { top: 0, left: 0, margin: '4px' }
|
||||
: { bottom: 0, right: 0, margin: '4px' }
|
||||
|
||||
return (
|
||||
<button
|
||||
type="button"
|
||||
data-action="toggle-vision"
|
||||
data-vision-status={!isVisionSetupComplete ? 'not-configured' : visionConfig.enabled ? 'enabled' : 'disabled'}
|
||||
onClick={handleClick}
|
||||
onContextMenu={handleContextMenu}
|
||||
title={`${statusLabel} (right-click for settings)`}
|
||||
style={{
|
||||
position: 'absolute',
|
||||
...positionStyles,
|
||||
}}
|
||||
className={css({
|
||||
...sizeStyles,
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
bg: 'rgba(0, 0, 0, 0.5)',
|
||||
backdropFilter: 'blur(4px)',
|
||||
border: '1px solid rgba(255, 255, 255, 0.3)',
|
||||
borderRadius: 'md',
|
||||
color: 'white',
|
||||
cursor: 'pointer',
|
||||
transition: 'all 0.2s',
|
||||
zIndex: 10,
|
||||
opacity: 0.8,
|
||||
_hover: {
|
||||
bg: 'rgba(0, 0, 0, 0.7)',
|
||||
opacity: 1,
|
||||
transform: 'scale(1.1)',
|
||||
},
|
||||
})}
|
||||
>
|
||||
{/* Camera icon */}
|
||||
<span style={{ position: 'relative' }}>
|
||||
📷
|
||||
{/* Status dot */}
|
||||
<span
|
||||
data-element="vision-status-dot"
|
||||
className={css({
|
||||
position: 'absolute',
|
||||
top: '-2px',
|
||||
right: '-4px',
|
||||
w: '8px',
|
||||
h: '8px',
|
||||
borderRadius: 'full',
|
||||
bg: statusColor,
|
||||
border: '1px solid white',
|
||||
boxShadow: '0 1px 2px rgba(0,0,0,0.3)',
|
||||
})}
|
||||
/>
|
||||
</span>
|
||||
</button>
|
||||
)
|
||||
}
|
||||
253
apps/web/src/components/vision/VisionSetupModal.tsx
Normal file
253
apps/web/src/components/vision/VisionSetupModal.tsx
Normal file
@@ -0,0 +1,253 @@
|
||||
'use client'
|
||||
|
||||
import { useState } from 'react'
|
||||
import { useMyAbacus } from '@/contexts/MyAbacusContext'
|
||||
import { Modal, ModalContent } from '@/components/common/Modal'
|
||||
import { AbacusVisionBridge } from './AbacusVisionBridge'
|
||||
import { css } from '../../../styled-system/css'
|
||||
|
||||
/**
|
||||
* Modal for configuring abacus vision settings
|
||||
*
|
||||
* Shows status and launches AbacusVisionBridge for configuration.
|
||||
* AbacusVisionBridge saves camera/calibration to MyAbacusContext.
|
||||
*/
|
||||
export function VisionSetupModal() {
|
||||
const {
|
||||
isVisionSetupOpen,
|
||||
closeVisionSetup,
|
||||
visionConfig,
|
||||
isVisionSetupComplete,
|
||||
setVisionEnabled,
|
||||
setVisionCamera,
|
||||
setVisionCalibration,
|
||||
setVisionRemoteSession,
|
||||
dock,
|
||||
} = useMyAbacus()
|
||||
|
||||
// State for showing the configuration UI
|
||||
const [isConfiguring, setIsConfiguring] = useState(false)
|
||||
|
||||
const handleClearSettings = () => {
|
||||
setVisionCamera(null)
|
||||
setVisionCalibration(null)
|
||||
setVisionRemoteSession(null)
|
||||
setVisionEnabled(false)
|
||||
}
|
||||
|
||||
return (
|
||||
<Modal isOpen={isVisionSetupOpen} onClose={closeVisionSetup}>
|
||||
<ModalContent
|
||||
title="📷 Abacus Vision"
|
||||
description="Use a camera to detect your physical abacus"
|
||||
borderColor="rgba(34, 211, 238, 0.3)"
|
||||
titleColor="rgba(34, 211, 238, 1)"
|
||||
>
|
||||
<div className={css({ display: 'flex', flexDirection: 'column', gap: 4 })}>
|
||||
{/* Status display */}
|
||||
<div
|
||||
data-element="vision-status"
|
||||
className={css({
|
||||
bg: 'rgba(0, 0, 0, 0.3)',
|
||||
borderRadius: 'lg',
|
||||
p: 4,
|
||||
})}
|
||||
>
|
||||
<h3 className={css({ fontSize: 'sm', fontWeight: 'bold', color: 'gray.300', mb: 2 })}>
|
||||
Status
|
||||
</h3>
|
||||
<div className={css({ display: 'flex', flexDirection: 'column', gap: 2 })}>
|
||||
<StatusRow
|
||||
label="Camera"
|
||||
value={visionConfig.cameraDeviceId ? 'Configured' : 'Not configured'}
|
||||
isConfigured={visionConfig.cameraDeviceId !== null}
|
||||
/>
|
||||
<StatusRow
|
||||
label="Calibration"
|
||||
value={visionConfig.calibration ? 'Configured' : 'Not configured'}
|
||||
isConfigured={visionConfig.calibration !== null}
|
||||
/>
|
||||
<StatusRow
|
||||
label="Remote Phone"
|
||||
value={visionConfig.remoteCameraSessionId ? 'Connected' : 'Not connected'}
|
||||
isConfigured={visionConfig.remoteCameraSessionId !== null}
|
||||
/>
|
||||
<StatusRow
|
||||
label="Vision Mode"
|
||||
value={visionConfig.enabled ? 'Enabled' : 'Disabled'}
|
||||
isConfigured={visionConfig.enabled}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Actions */}
|
||||
<div className={css({ display: 'flex', flexDirection: 'column', gap: 2 })}>
|
||||
{isVisionSetupComplete && (
|
||||
<button
|
||||
type="button"
|
||||
data-action="toggle-vision"
|
||||
onClick={() => {
|
||||
setVisionEnabled(!visionConfig.enabled)
|
||||
}}
|
||||
className={css({
|
||||
px: 4,
|
||||
py: 3,
|
||||
bg: visionConfig.enabled ? 'red.600' : 'green.600',
|
||||
color: 'white',
|
||||
borderRadius: 'lg',
|
||||
fontWeight: 'semibold',
|
||||
border: 'none',
|
||||
cursor: 'pointer',
|
||||
transition: 'all 0.2s',
|
||||
_hover: {
|
||||
bg: visionConfig.enabled ? 'red.700' : 'green.700',
|
||||
transform: 'scale(1.02)',
|
||||
},
|
||||
})}
|
||||
>
|
||||
{visionConfig.enabled ? 'Disable Vision' : 'Enable Vision'}
|
||||
</button>
|
||||
)}
|
||||
|
||||
<button
|
||||
type="button"
|
||||
data-action="configure-camera"
|
||||
onClick={() => setIsConfiguring(true)}
|
||||
className={css({
|
||||
px: 4,
|
||||
py: 3,
|
||||
bg: 'cyan.600',
|
||||
color: 'white',
|
||||
borderRadius: 'lg',
|
||||
fontWeight: 'semibold',
|
||||
border: 'none',
|
||||
cursor: 'pointer',
|
||||
transition: 'all 0.2s',
|
||||
_hover: {
|
||||
bg: 'cyan.700',
|
||||
transform: 'scale(1.02)',
|
||||
},
|
||||
})}
|
||||
>
|
||||
{isVisionSetupComplete ? 'Reconfigure Camera' : 'Configure Camera & Calibration'}
|
||||
</button>
|
||||
|
||||
{isVisionSetupComplete && (
|
||||
<button
|
||||
type="button"
|
||||
data-action="clear-settings"
|
||||
onClick={handleClearSettings}
|
||||
className={css({
|
||||
px: 4,
|
||||
py: 2,
|
||||
bg: 'transparent',
|
||||
color: 'gray.400',
|
||||
borderRadius: 'lg',
|
||||
fontWeight: 'medium',
|
||||
border: '1px solid',
|
||||
borderColor: 'gray.600',
|
||||
cursor: 'pointer',
|
||||
transition: 'all 0.2s',
|
||||
_hover: {
|
||||
borderColor: 'gray.500',
|
||||
color: 'gray.300',
|
||||
},
|
||||
})}
|
||||
>
|
||||
Clear All Settings
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Close button */}
|
||||
<button
|
||||
type="button"
|
||||
data-action="close-modal"
|
||||
onClick={closeVisionSetup}
|
||||
className={css({
|
||||
mt: 2,
|
||||
px: 4,
|
||||
py: 2,
|
||||
bg: 'gray.700',
|
||||
color: 'white',
|
||||
borderRadius: 'lg',
|
||||
border: 'none',
|
||||
cursor: 'pointer',
|
||||
transition: 'all 0.2s',
|
||||
_hover: {
|
||||
bg: 'gray.600',
|
||||
},
|
||||
})}
|
||||
>
|
||||
Close
|
||||
</button>
|
||||
</div>
|
||||
</ModalContent>
|
||||
|
||||
{/* AbacusVisionBridge overlay for configuration */}
|
||||
{isConfiguring && (
|
||||
<div
|
||||
data-element="vision-config-overlay"
|
||||
className={css({
|
||||
position: 'fixed',
|
||||
inset: 0,
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
bg: 'rgba(0, 0, 0, 0.8)',
|
||||
zIndex: 1000,
|
||||
})}
|
||||
>
|
||||
<AbacusVisionBridge
|
||||
columnCount={dock?.columns ?? 5}
|
||||
onValueDetected={(value) => {
|
||||
// Value detected - configuration is working
|
||||
console.log('[VisionSetupModal] Value detected:', value)
|
||||
}}
|
||||
onClose={() => setIsConfiguring(false)}
|
||||
onConfigurationChange={(config) => {
|
||||
// Save configuration to context as it changes
|
||||
if (config.cameraDeviceId !== undefined) {
|
||||
setVisionCamera(config.cameraDeviceId)
|
||||
}
|
||||
if (config.calibration !== undefined) {
|
||||
setVisionCalibration(config.calibration)
|
||||
}
|
||||
if (config.remoteCameraSessionId !== undefined) {
|
||||
setVisionRemoteSession(config.remoteCameraSessionId)
|
||||
}
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
</Modal>
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Status row component
|
||||
*/
|
||||
function StatusRow({
|
||||
label,
|
||||
value,
|
||||
isConfigured,
|
||||
}: {
|
||||
label: string
|
||||
value: string
|
||||
isConfigured: boolean
|
||||
}) {
|
||||
return (
|
||||
<div className={css({ display: 'flex', justifyContent: 'space-between', alignItems: 'center' })}>
|
||||
<span className={css({ color: 'gray.400', fontSize: 'sm' })}>{label}</span>
|
||||
<span
|
||||
className={css({
|
||||
fontSize: 'sm',
|
||||
fontWeight: 'medium',
|
||||
color: isConfigured ? 'green.400' : 'gray.500',
|
||||
})}
|
||||
>
|
||||
{value}
|
||||
</span>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -6,9 +6,68 @@ import {
|
||||
type MutableRefObject,
|
||||
useCallback,
|
||||
useContext,
|
||||
useEffect,
|
||||
useRef,
|
||||
useState,
|
||||
} from 'react'
|
||||
import type { CalibrationGrid } from '@/types/vision'
|
||||
|
||||
/**
|
||||
* Configuration for abacus vision (camera-based input)
|
||||
*/
|
||||
export interface VisionConfig {
|
||||
/** Whether vision mode is enabled */
|
||||
enabled: boolean
|
||||
/** Selected camera device ID */
|
||||
cameraDeviceId: string | null
|
||||
/** Saved calibration grid for cropping */
|
||||
calibration: CalibrationGrid | null
|
||||
/** Remote phone camera session ID (for phone-as-camera mode) */
|
||||
remoteCameraSessionId: string | null
|
||||
}
|
||||
|
||||
const DEFAULT_VISION_CONFIG: VisionConfig = {
|
||||
enabled: false,
|
||||
cameraDeviceId: null,
|
||||
calibration: null,
|
||||
remoteCameraSessionId: null,
|
||||
}
|
||||
|
||||
const VISION_CONFIG_STORAGE_KEY = 'abacus-vision-config'
|
||||
|
||||
/**
|
||||
* Load vision config from localStorage
|
||||
*/
|
||||
function loadVisionConfig(): VisionConfig {
|
||||
if (typeof window === 'undefined') return DEFAULT_VISION_CONFIG
|
||||
try {
|
||||
const stored = localStorage.getItem(VISION_CONFIG_STORAGE_KEY)
|
||||
if (stored) {
|
||||
const parsed = JSON.parse(stored)
|
||||
return {
|
||||
...DEFAULT_VISION_CONFIG,
|
||||
...parsed,
|
||||
// Always start with vision disabled - user must re-enable
|
||||
enabled: false,
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('[MyAbacusContext] Failed to load vision config:', e)
|
||||
}
|
||||
return DEFAULT_VISION_CONFIG
|
||||
}
|
||||
|
||||
/**
|
||||
* Save vision config to localStorage
|
||||
*/
|
||||
function saveVisionConfig(config: VisionConfig): void {
|
||||
if (typeof window === 'undefined') return
|
||||
try {
|
||||
localStorage.setItem(VISION_CONFIG_STORAGE_KEY, JSON.stringify(config))
|
||||
} catch (e) {
|
||||
console.error('[MyAbacusContext] Failed to save vision config:', e)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Configuration for a docked abacus
|
||||
@@ -107,6 +166,25 @@ interface MyAbacusContextValue {
|
||||
setDockedValue: (value: number) => void
|
||||
/** Current abacus value (for reading) */
|
||||
abacusValue: number
|
||||
// Vision-related state
|
||||
/** Current vision configuration */
|
||||
visionConfig: VisionConfig
|
||||
/** Whether vision setup is complete (has camera and calibration) */
|
||||
isVisionSetupComplete: boolean
|
||||
/** Set whether vision is enabled */
|
||||
setVisionEnabled: (enabled: boolean) => void
|
||||
/** Set the selected camera device ID */
|
||||
setVisionCamera: (deviceId: string | null) => void
|
||||
/** Set the calibration grid */
|
||||
setVisionCalibration: (calibration: CalibrationGrid | null) => void
|
||||
/** Set the remote camera session ID */
|
||||
setVisionRemoteSession: (sessionId: string | null) => void
|
||||
/** Whether the vision setup modal is open */
|
||||
isVisionSetupOpen: boolean
|
||||
/** Open the vision setup modal */
|
||||
openVisionSetup: () => void
|
||||
/** Close the vision setup modal */
|
||||
closeVisionSetup: () => void
|
||||
}
|
||||
|
||||
const MyAbacusContext = createContext<MyAbacusContextValue | undefined>(undefined)
|
||||
@@ -124,6 +202,16 @@ export function MyAbacusProvider({ children }: { children: React.ReactNode }) {
|
||||
const [pendingDockRequest, setPendingDockRequest] = useState(false)
|
||||
const [abacusValue, setAbacusValue] = useState(0)
|
||||
|
||||
// Vision state
|
||||
const [visionConfig, setVisionConfig] = useState<VisionConfig>(DEFAULT_VISION_CONFIG)
|
||||
const [isVisionSetupOpen, setIsVisionSetupOpen] = useState(false)
|
||||
|
||||
// Load vision config from localStorage on mount
|
||||
useEffect(() => {
|
||||
const loaded = loadVisionConfig()
|
||||
setVisionConfig(loaded)
|
||||
}, [])
|
||||
|
||||
const open = useCallback(() => setIsOpen(true), [])
|
||||
const close = useCallback(() => setIsOpen(false), [])
|
||||
const toggle = useCallback(() => setIsOpen((prev) => !prev), [])
|
||||
@@ -200,6 +288,51 @@ export function MyAbacusProvider({ children }: { children: React.ReactNode }) {
|
||||
setAbacusValue(value)
|
||||
}, [])
|
||||
|
||||
// Vision callbacks
|
||||
const isVisionSetupComplete =
|
||||
visionConfig.cameraDeviceId !== null && visionConfig.calibration !== null
|
||||
|
||||
const setVisionEnabled = useCallback((enabled: boolean) => {
|
||||
setVisionConfig((prev) => {
|
||||
const updated = { ...prev, enabled }
|
||||
saveVisionConfig(updated)
|
||||
return updated
|
||||
})
|
||||
}, [])
|
||||
|
||||
const setVisionCamera = useCallback((deviceId: string | null) => {
|
||||
setVisionConfig((prev) => {
|
||||
const updated = { ...prev, cameraDeviceId: deviceId }
|
||||
saveVisionConfig(updated)
|
||||
return updated
|
||||
})
|
||||
}, [])
|
||||
|
||||
const setVisionCalibration = useCallback((calibration: CalibrationGrid | null) => {
|
||||
setVisionConfig((prev) => {
|
||||
const updated = { ...prev, calibration }
|
||||
saveVisionConfig(updated)
|
||||
return updated
|
||||
})
|
||||
}, [])
|
||||
|
||||
const setVisionRemoteSession = useCallback((sessionId: string | null) => {
|
||||
setVisionConfig((prev) => {
|
||||
const updated = { ...prev, remoteCameraSessionId: sessionId }
|
||||
saveVisionConfig(updated)
|
||||
return updated
|
||||
})
|
||||
}, [])
|
||||
|
||||
const openVisionSetup = useCallback(() => {
|
||||
console.log('[MyAbacusContext] openVisionSetup called')
|
||||
setIsVisionSetupOpen(true)
|
||||
}, [])
|
||||
const closeVisionSetup = useCallback(() => {
|
||||
console.log('[MyAbacusContext] closeVisionSetup called')
|
||||
setIsVisionSetupOpen(false)
|
||||
}, [])
|
||||
|
||||
return (
|
||||
<MyAbacusContext.Provider
|
||||
value={{
|
||||
@@ -233,6 +366,16 @@ export function MyAbacusProvider({ children }: { children: React.ReactNode }) {
|
||||
clearDockRequest,
|
||||
setDockedValue,
|
||||
abacusValue,
|
||||
// Vision
|
||||
visionConfig,
|
||||
isVisionSetupComplete,
|
||||
setVisionEnabled,
|
||||
setVisionCamera,
|
||||
setVisionCalibration,
|
||||
setVisionRemoteSession,
|
||||
isVisionSetupOpen,
|
||||
openVisionSetup,
|
||||
closeVisionSetup,
|
||||
}}
|
||||
>
|
||||
{children}
|
||||
|
||||
Reference in New Issue
Block a user