feat(vision): add AbacusVisionBridge for physical soroban detection
Implements camera-based detection system for physical abacus: - VisionCameraFeed: Camera display with Desk View auto-detection - CalibrationOverlay: Interactive quad corner calibration with proper letterbox handling and bounds clamping - Perspective transform using OpenCV.js for rectification - Live rectified preview during calibration - Hooks: useAbacusVision, useDeskViewCamera, useCameraCalibration, useFrameStability 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Thomas Hallock
parent
156a0dfe96
commit
47088e4850
|
|
@ -0,0 +1,417 @@
|
|||
'use client'
|
||||
|
||||
import type { ReactNode } from 'react'
|
||||
import { useCallback, useEffect, useRef, useState } from 'react'
|
||||
import { css } from '../../../styled-system/css'
|
||||
import { useAbacusVision } from '@/hooks/useAbacusVision'
|
||||
import type { QuadCorners } from '@/types/vision'
|
||||
import { DEFAULT_STABILITY_CONFIG } from '@/types/vision'
|
||||
import { isOpenCVReady, loadOpenCV, rectifyQuadrilateral } from '@/lib/vision/perspectiveTransform'
|
||||
import { CalibrationOverlay } from './CalibrationOverlay'
|
||||
import { VisionCameraFeed } from './VisionCameraFeed'
|
||||
import { VisionStatusIndicator } from './VisionStatusIndicator'
|
||||
|
||||
export interface AbacusVisionBridgeProps {
|
||||
/** Number of abacus columns to detect */
|
||||
columnCount: number
|
||||
/** Called when a stable value is detected */
|
||||
onValueDetected: (value: number) => void
|
||||
/** Called when vision mode is closed */
|
||||
onClose: () => void
|
||||
/** Called on error */
|
||||
onError?: (error: string) => void
|
||||
}
|
||||
|
||||
/**
|
||||
* AbacusVisionBridge - Main UI for physical abacus detection
|
||||
*
|
||||
* Provides:
|
||||
* - Camera feed with Desk View auto-detection
|
||||
* - Interactive calibration UI
|
||||
* - Real-time detection status
|
||||
* - Stable value output to control digital abacus
|
||||
*/
|
||||
export function AbacusVisionBridge({
|
||||
columnCount,
|
||||
onValueDetected,
|
||||
onClose,
|
||||
onError,
|
||||
}: AbacusVisionBridgeProps): ReactNode {
|
||||
const [videoDimensions, setVideoDimensions] = useState<{
|
||||
width: number
|
||||
height: number
|
||||
containerWidth: number
|
||||
containerHeight: number
|
||||
} | null>(null)
|
||||
|
||||
const containerRef = useRef<HTMLDivElement>(null)
|
||||
const cameraFeedContainerRef = useRef<HTMLDivElement>(null)
|
||||
const videoRef = useRef<HTMLVideoElement | null>(null)
|
||||
const previewCanvasRef = useRef<HTMLCanvasElement>(null)
|
||||
|
||||
const [calibrationCorners, setCalibrationCorners] = useState<QuadCorners | null>(null)
|
||||
const [opencvReady, setOpencvReady] = useState(false)
|
||||
|
||||
const vision = useAbacusVision({
|
||||
columnCount,
|
||||
onValueDetected,
|
||||
})
|
||||
|
||||
// Start camera on mount
|
||||
useEffect(() => {
|
||||
vision.enable()
|
||||
return () => vision.disable()
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, []) // Only run on mount/unmount - vision functions are stable
|
||||
|
||||
// Report errors
|
||||
useEffect(() => {
|
||||
if (vision.cameraError && onError) {
|
||||
onError(vision.cameraError)
|
||||
}
|
||||
}, [vision.cameraError, onError])
|
||||
|
||||
// Load OpenCV when calibrating
|
||||
useEffect(() => {
|
||||
if (vision.isCalibrating && !opencvReady) {
|
||||
loadOpenCV()
|
||||
.then(() => setOpencvReady(true))
|
||||
.catch((err) => console.error('Failed to load OpenCV:', err))
|
||||
}
|
||||
}, [vision.isCalibrating, opencvReady])
|
||||
|
||||
// Render preview when calibrating
|
||||
useEffect(() => {
|
||||
if (!vision.isCalibrating || !calibrationCorners || !videoRef.current || !previewCanvasRef.current) {
|
||||
return
|
||||
}
|
||||
|
||||
let running = true
|
||||
const video = videoRef.current
|
||||
const canvas = previewCanvasRef.current
|
||||
|
||||
const drawPreview = () => {
|
||||
if (!running || video.readyState < 2) {
|
||||
if (running) requestAnimationFrame(drawPreview)
|
||||
return
|
||||
}
|
||||
|
||||
if (opencvReady && isOpenCVReady()) {
|
||||
rectifyQuadrilateral(video, calibrationCorners, canvas, {
|
||||
outputWidth: 300,
|
||||
outputHeight: 200,
|
||||
})
|
||||
}
|
||||
|
||||
requestAnimationFrame(drawPreview)
|
||||
}
|
||||
|
||||
drawPreview()
|
||||
return () => { running = false }
|
||||
}, [vision.isCalibrating, calibrationCorners, opencvReady])
|
||||
|
||||
// Handle video ready - get dimensions
|
||||
const handleVideoReady = useCallback(
|
||||
(width: number, height: number) => {
|
||||
const feedContainer = cameraFeedContainerRef.current
|
||||
if (!feedContainer) return
|
||||
|
||||
const rect = feedContainer.getBoundingClientRect()
|
||||
setVideoDimensions({
|
||||
width,
|
||||
height,
|
||||
containerWidth: rect.width,
|
||||
containerHeight: rect.height,
|
||||
})
|
||||
|
||||
// Initialize calibration with default grid if not already calibrated
|
||||
if (!vision.isCalibrated) {
|
||||
// Will start calibration on user action
|
||||
}
|
||||
},
|
||||
[vision.isCalibrated]
|
||||
)
|
||||
|
||||
// Handle calibration complete
|
||||
const handleCalibrationComplete = useCallback(
|
||||
(grid: Parameters<typeof vision.finishCalibration>[0]) => {
|
||||
vision.finishCalibration(grid)
|
||||
},
|
||||
[vision]
|
||||
)
|
||||
|
||||
// Camera selector
|
||||
const handleCameraSelect = useCallback(
|
||||
(e: React.ChangeEvent<HTMLSelectElement>) => {
|
||||
vision.selectCamera(e.target.value)
|
||||
},
|
||||
[vision]
|
||||
)
|
||||
|
||||
return (
|
||||
<div
|
||||
ref={containerRef}
|
||||
data-component="abacus-vision-bridge"
|
||||
className={css({
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
gap: 3,
|
||||
p: 3,
|
||||
bg: 'gray.900',
|
||||
borderRadius: 'xl',
|
||||
maxWidth: '400px',
|
||||
width: '100%',
|
||||
})}
|
||||
>
|
||||
{/* Header */}
|
||||
<div
|
||||
data-element="header"
|
||||
className={css({
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'space-between',
|
||||
})}
|
||||
>
|
||||
<div className={css({ display: 'flex', alignItems: 'center', gap: 2 })}>
|
||||
<span className={css({ fontSize: 'lg' })}>📷</span>
|
||||
<span className={css({ color: 'white', fontWeight: 'medium' })}>Abacus Vision</span>
|
||||
{vision.isDeskViewDetected && (
|
||||
<span
|
||||
className={css({
|
||||
px: 2,
|
||||
py: 0.5,
|
||||
bg: 'green.900',
|
||||
color: 'green.300',
|
||||
fontSize: 'xs',
|
||||
borderRadius: 'full',
|
||||
})}
|
||||
>
|
||||
Desk View
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
<button
|
||||
type="button"
|
||||
onClick={onClose}
|
||||
className={css({
|
||||
width: '32px',
|
||||
height: '32px',
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
bg: 'gray.700',
|
||||
color: 'white',
|
||||
border: 'none',
|
||||
borderRadius: 'full',
|
||||
cursor: 'pointer',
|
||||
fontSize: 'lg',
|
||||
_hover: { bg: 'gray.600' },
|
||||
})}
|
||||
aria-label="Close"
|
||||
>
|
||||
×
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{/* Camera selector (if multiple cameras) */}
|
||||
{vision.availableDevices.length > 1 && (
|
||||
<select
|
||||
data-element="camera-selector"
|
||||
value={vision.selectedDeviceId ?? ''}
|
||||
onChange={handleCameraSelect}
|
||||
className={css({
|
||||
p: 2,
|
||||
bg: 'gray.800',
|
||||
color: 'white',
|
||||
border: '1px solid',
|
||||
borderColor: 'gray.600',
|
||||
borderRadius: 'md',
|
||||
fontSize: 'sm',
|
||||
})}
|
||||
>
|
||||
{vision.availableDevices.map((device) => (
|
||||
<option key={device.deviceId} value={device.deviceId}>
|
||||
{device.label || `Camera ${device.deviceId.slice(0, 8)}`}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
)}
|
||||
|
||||
{/* Camera feed */}
|
||||
<div ref={cameraFeedContainerRef} className={css({ position: 'relative' })}>
|
||||
<VisionCameraFeed
|
||||
videoStream={vision.videoStream}
|
||||
isLoading={vision.isCameraLoading}
|
||||
calibration={vision.calibrationGrid}
|
||||
showCalibrationGrid={!vision.isCalibrating && !vision.isCalibrated}
|
||||
showRectifiedView={vision.isCalibrated && !vision.isCalibrating}
|
||||
videoRef={(el) => {
|
||||
videoRef.current = el
|
||||
}}
|
||||
onVideoReady={handleVideoReady}
|
||||
>
|
||||
{/* Calibration overlay when calibrating */}
|
||||
{vision.isCalibrating && videoDimensions && (
|
||||
<CalibrationOverlay
|
||||
columnCount={columnCount}
|
||||
videoWidth={videoDimensions.width}
|
||||
videoHeight={videoDimensions.height}
|
||||
containerWidth={videoDimensions.containerWidth}
|
||||
containerHeight={videoDimensions.containerHeight}
|
||||
initialCalibration={vision.calibrationGrid}
|
||||
onComplete={handleCalibrationComplete}
|
||||
onCancel={vision.cancelCalibration}
|
||||
videoElement={videoRef.current}
|
||||
onCornersChange={setCalibrationCorners}
|
||||
/>
|
||||
)}
|
||||
</VisionCameraFeed>
|
||||
|
||||
{/* Status indicator */}
|
||||
{vision.isEnabled && !vision.isCalibrating && (
|
||||
<div
|
||||
className={css({
|
||||
position: 'absolute',
|
||||
top: 2,
|
||||
left: 2,
|
||||
})}
|
||||
>
|
||||
<VisionStatusIndicator
|
||||
isCalibrated={vision.isCalibrated}
|
||||
isDetecting={vision.isDetecting}
|
||||
confidence={vision.confidence}
|
||||
handDetected={vision.isHandDetected}
|
||||
detectedValue={vision.currentDetectedValue}
|
||||
consecutiveFrames={vision.consecutiveFrames}
|
||||
minFrames={DEFAULT_STABILITY_CONFIG.minConsecutiveFrames}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Rectified preview during calibration */}
|
||||
{vision.isCalibrating && (
|
||||
<div
|
||||
data-element="calibration-preview"
|
||||
className={css({
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
alignItems: 'center',
|
||||
gap: 1,
|
||||
})}
|
||||
>
|
||||
<span className={css({ color: 'gray.400', fontSize: 'xs' })}>
|
||||
Rectified Preview
|
||||
</span>
|
||||
<canvas
|
||||
ref={previewCanvasRef}
|
||||
className={css({
|
||||
borderRadius: 'md',
|
||||
border: '1px solid',
|
||||
borderColor: 'gray.600',
|
||||
bg: 'gray.800',
|
||||
})}
|
||||
width={300}
|
||||
height={200}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Actions */}
|
||||
<div
|
||||
data-element="actions"
|
||||
className={css({
|
||||
display: 'flex',
|
||||
gap: 2,
|
||||
})}
|
||||
>
|
||||
{!vision.isCalibrated ? (
|
||||
<button
|
||||
type="button"
|
||||
onClick={vision.startCalibration}
|
||||
disabled={!videoDimensions}
|
||||
className={css({
|
||||
flex: 1,
|
||||
py: 2,
|
||||
bg: 'blue.600',
|
||||
color: 'white',
|
||||
border: 'none',
|
||||
borderRadius: 'md',
|
||||
fontWeight: 'medium',
|
||||
cursor: 'pointer',
|
||||
_hover: { bg: 'blue.500' },
|
||||
_disabled: { opacity: 0.5, cursor: 'not-allowed' },
|
||||
})}
|
||||
>
|
||||
Calibrate
|
||||
</button>
|
||||
) : (
|
||||
<>
|
||||
<button
|
||||
type="button"
|
||||
onClick={vision.startCalibration}
|
||||
className={css({
|
||||
flex: 1,
|
||||
py: 2,
|
||||
bg: 'gray.700',
|
||||
color: 'white',
|
||||
border: 'none',
|
||||
borderRadius: 'md',
|
||||
cursor: 'pointer',
|
||||
_hover: { bg: 'gray.600' },
|
||||
})}
|
||||
>
|
||||
Recalibrate
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={vision.resetCalibration}
|
||||
className={css({
|
||||
py: 2,
|
||||
px: 3,
|
||||
bg: 'red.700',
|
||||
color: 'white',
|
||||
border: 'none',
|
||||
borderRadius: 'md',
|
||||
cursor: 'pointer',
|
||||
_hover: { bg: 'red.600' },
|
||||
})}
|
||||
>
|
||||
Reset
|
||||
</button>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Instructions */}
|
||||
{!vision.isCalibrated && !vision.isCalibrating && (
|
||||
<p
|
||||
className={css({
|
||||
color: 'gray.400',
|
||||
fontSize: 'sm',
|
||||
textAlign: 'center',
|
||||
})}
|
||||
>
|
||||
Point your camera at a soroban and click Calibrate to set up detection
|
||||
</p>
|
||||
)}
|
||||
|
||||
{/* Error display */}
|
||||
{vision.cameraError && (
|
||||
<div
|
||||
className={css({
|
||||
p: 3,
|
||||
bg: 'red.900',
|
||||
color: 'red.200',
|
||||
borderRadius: 'md',
|
||||
fontSize: 'sm',
|
||||
})}
|
||||
>
|
||||
{vision.cameraError}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default AbacusVisionBridge
|
||||
|
|
@ -0,0 +1,560 @@
|
|||
'use client'
|
||||
|
||||
import type { ReactNode } from 'react'
|
||||
import { useCallback, useEffect, useRef, useState } from 'react'
|
||||
import { css } from '../../../styled-system/css'
|
||||
import type { CalibrationGrid, Point, QuadCorners, ROI } from '@/types/vision'
|
||||
|
||||
export interface CalibrationOverlayProps {
|
||||
/** Number of columns to configure */
|
||||
columnCount: number
|
||||
/** Video dimensions */
|
||||
videoWidth: number
|
||||
videoHeight: number
|
||||
/** Container dimensions (displayed size) */
|
||||
containerWidth: number
|
||||
containerHeight: number
|
||||
/** Current calibration (if any) */
|
||||
initialCalibration?: CalibrationGrid | null
|
||||
/** Called when calibration is completed */
|
||||
onComplete: (grid: CalibrationGrid) => void
|
||||
/** Called when calibration is cancelled */
|
||||
onCancel: () => void
|
||||
/** Video element for live preview */
|
||||
videoElement?: HTMLVideoElement | null
|
||||
/** Called when corners change (for external preview) */
|
||||
onCornersChange?: (corners: QuadCorners) => void
|
||||
}
|
||||
|
||||
type CornerKey = 'topLeft' | 'topRight' | 'bottomLeft' | 'bottomRight'
|
||||
type DragTarget = CornerKey | 'quad' | `divider-${number}` | null
|
||||
|
||||
/**
|
||||
* Convert QuadCorners to legacy ROI format (bounding box)
|
||||
*/
|
||||
function cornersToROI(corners: QuadCorners): ROI {
|
||||
const minX = Math.min(corners.topLeft.x, corners.bottomLeft.x)
|
||||
const maxX = Math.max(corners.topRight.x, corners.bottomRight.x)
|
||||
const minY = Math.min(corners.topLeft.y, corners.topRight.y)
|
||||
const maxY = Math.max(corners.bottomLeft.y, corners.bottomRight.y)
|
||||
return {
|
||||
x: minX,
|
||||
y: minY,
|
||||
width: maxX - minX,
|
||||
height: maxY - minY,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Interpolate a point along the edge of the quadrilateral
|
||||
* @param t - Position along the top/bottom edge (0 = left, 1 = right)
|
||||
* @param corners - The quadrilateral corners
|
||||
* @returns Top and bottom points for a vertical line at position t
|
||||
*/
|
||||
function getColumnLine(t: number, corners: QuadCorners): { top: Point; bottom: Point } {
|
||||
return {
|
||||
top: {
|
||||
x: corners.topLeft.x + t * (corners.topRight.x - corners.topLeft.x),
|
||||
y: corners.topLeft.y + t * (corners.topRight.y - corners.topLeft.y),
|
||||
},
|
||||
bottom: {
|
||||
x: corners.bottomLeft.x + t * (corners.bottomRight.x - corners.bottomLeft.x),
|
||||
y: corners.bottomLeft.y + t * (corners.bottomRight.y - corners.bottomLeft.y),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* CalibrationOverlay - Interactive quadrilateral calibration editor
|
||||
*
|
||||
* Allows users to:
|
||||
* 1. Drag 4 corners independently to match perspective
|
||||
* 2. Adjust column dividers within the quadrilateral
|
||||
* 3. Align the virtual grid over their physical abacus
|
||||
*/
|
||||
export function CalibrationOverlay({
|
||||
columnCount,
|
||||
videoWidth,
|
||||
videoHeight,
|
||||
containerWidth,
|
||||
containerHeight,
|
||||
initialCalibration,
|
||||
onComplete,
|
||||
onCancel,
|
||||
videoElement,
|
||||
onCornersChange,
|
||||
}: CalibrationOverlayProps): ReactNode {
|
||||
// Calculate actual visible video bounds (accounting for object-fit: contain letterboxing)
|
||||
const videoAspect = videoWidth / videoHeight
|
||||
const containerAspect = containerWidth / containerHeight
|
||||
|
||||
let displayedVideoWidth: number
|
||||
let displayedVideoHeight: number
|
||||
let videoOffsetX: number
|
||||
let videoOffsetY: number
|
||||
|
||||
if (videoAspect > containerAspect) {
|
||||
// Video is wider than container - letterbox top/bottom
|
||||
displayedVideoWidth = containerWidth
|
||||
displayedVideoHeight = containerWidth / videoAspect
|
||||
videoOffsetX = 0
|
||||
videoOffsetY = (containerHeight - displayedVideoHeight) / 2
|
||||
} else {
|
||||
// Video is taller than container - letterbox left/right
|
||||
displayedVideoHeight = containerHeight
|
||||
displayedVideoWidth = containerHeight * videoAspect
|
||||
videoOffsetX = (containerWidth - displayedVideoWidth) / 2
|
||||
videoOffsetY = 0
|
||||
}
|
||||
|
||||
// Uniform scale factor (maintains aspect ratio)
|
||||
const scale = displayedVideoWidth / videoWidth
|
||||
|
||||
// Initialize corners state
|
||||
const getDefaultCorners = (): QuadCorners => {
|
||||
// Default to a slightly trapezoidal shape (wider at bottom for typical desk perspective)
|
||||
// Use larger margins to ensure all corners are visible and draggable
|
||||
const topMargin = 0.15
|
||||
const bottomMargin = 0.2 // Larger margin at bottom to keep handles visible
|
||||
const sideMargin = 0.15
|
||||
const topInset = 0.03 // Make top slightly narrower than bottom for perspective
|
||||
return {
|
||||
topLeft: { x: videoWidth * (sideMargin + topInset), y: videoHeight * topMargin },
|
||||
topRight: { x: videoWidth * (1 - sideMargin - topInset), y: videoHeight * topMargin },
|
||||
bottomLeft: { x: videoWidth * sideMargin, y: videoHeight * (1 - bottomMargin) },
|
||||
bottomRight: { x: videoWidth * (1 - sideMargin), y: videoHeight * (1 - bottomMargin) },
|
||||
}
|
||||
}
|
||||
|
||||
const [corners, setCorners] = useState<QuadCorners>(() => {
|
||||
if (initialCalibration?.corners) {
|
||||
return initialCalibration.corners
|
||||
}
|
||||
// Convert from legacy ROI if available
|
||||
if (initialCalibration?.roi) {
|
||||
const roi = initialCalibration.roi
|
||||
return {
|
||||
topLeft: { x: roi.x, y: roi.y },
|
||||
topRight: { x: roi.x + roi.width, y: roi.y },
|
||||
bottomLeft: { x: roi.x, y: roi.y + roi.height },
|
||||
bottomRight: { x: roi.x + roi.width, y: roi.y + roi.height },
|
||||
}
|
||||
}
|
||||
return getDefaultCorners()
|
||||
})
|
||||
|
||||
// Initialize column dividers (evenly spaced)
|
||||
const getDefaultDividers = (): number[] => {
|
||||
const dividers: number[] = []
|
||||
for (let i = 1; i < columnCount; i++) {
|
||||
dividers.push(i / columnCount)
|
||||
}
|
||||
return dividers
|
||||
}
|
||||
|
||||
const [dividers, setDividers] = useState<number[]>(
|
||||
initialCalibration?.columnDividers ?? getDefaultDividers()
|
||||
)
|
||||
|
||||
// Instructions visibility
|
||||
const [showInstructions, setShowInstructions] = useState(true)
|
||||
|
||||
// Drag state
|
||||
const [dragTarget, setDragTarget] = useState<DragTarget>(null)
|
||||
const dragStartRef = useRef<{
|
||||
x: number
|
||||
y: number
|
||||
corners: QuadCorners
|
||||
dividers: number[]
|
||||
} | null>(null)
|
||||
|
||||
// Notify parent when corners change
|
||||
useEffect(() => {
|
||||
onCornersChange?.(corners)
|
||||
}, [corners, onCornersChange])
|
||||
|
||||
// Handle pointer down on corners or dividers
|
||||
const handlePointerDown = useCallback(
|
||||
(e: React.PointerEvent, target: DragTarget) => {
|
||||
e.preventDefault()
|
||||
e.stopPropagation()
|
||||
setDragTarget(target)
|
||||
dragStartRef.current = {
|
||||
x: e.clientX,
|
||||
y: e.clientY,
|
||||
corners: { ...corners },
|
||||
dividers: [...dividers],
|
||||
}
|
||||
;(e.target as HTMLElement).setPointerCapture(e.pointerId)
|
||||
},
|
||||
[corners, dividers]
|
||||
)
|
||||
|
||||
// Handle pointer move during drag
|
||||
const handlePointerMove = useCallback(
|
||||
(e: React.PointerEvent) => {
|
||||
if (!dragTarget || !dragStartRef.current) return
|
||||
|
||||
const dx = (e.clientX - dragStartRef.current.x) / scale
|
||||
const dy = (e.clientY - dragStartRef.current.y) / scale
|
||||
const startCorners = dragStartRef.current.corners
|
||||
|
||||
if (dragTarget === 'quad') {
|
||||
// Move entire quadrilateral
|
||||
// Calculate bounds to keep all corners within video
|
||||
const minX = Math.min(startCorners.topLeft.x, startCorners.bottomLeft.x)
|
||||
const maxX = Math.max(startCorners.topRight.x, startCorners.bottomRight.x)
|
||||
const minY = Math.min(startCorners.topLeft.y, startCorners.topRight.y)
|
||||
const maxY = Math.max(startCorners.bottomLeft.y, startCorners.bottomRight.y)
|
||||
|
||||
// Clamp movement to keep quad within video bounds
|
||||
const clampedDx = Math.max(-minX, Math.min(videoWidth - maxX, dx))
|
||||
const clampedDy = Math.max(-minY, Math.min(videoHeight - maxY, dy))
|
||||
|
||||
setCorners({
|
||||
topLeft: { x: startCorners.topLeft.x + clampedDx, y: startCorners.topLeft.y + clampedDy },
|
||||
topRight: {
|
||||
x: startCorners.topRight.x + clampedDx,
|
||||
y: startCorners.topRight.y + clampedDy,
|
||||
},
|
||||
bottomLeft: {
|
||||
x: startCorners.bottomLeft.x + clampedDx,
|
||||
y: startCorners.bottomLeft.y + clampedDy,
|
||||
},
|
||||
bottomRight: {
|
||||
x: startCorners.bottomRight.x + clampedDx,
|
||||
y: startCorners.bottomRight.y + clampedDy,
|
||||
},
|
||||
})
|
||||
} else if (
|
||||
dragTarget === 'topLeft' ||
|
||||
dragTarget === 'topRight' ||
|
||||
dragTarget === 'bottomLeft' ||
|
||||
dragTarget === 'bottomRight'
|
||||
) {
|
||||
// Move single corner
|
||||
const startPoint = startCorners[dragTarget]
|
||||
const newPoint: Point = {
|
||||
x: Math.max(0, Math.min(videoWidth, startPoint.x + dx)),
|
||||
y: Math.max(0, Math.min(videoHeight, startPoint.y + dy)),
|
||||
}
|
||||
setCorners((prev) => ({
|
||||
...prev,
|
||||
[dragTarget]: newPoint,
|
||||
}))
|
||||
} else if (dragTarget.startsWith('divider-')) {
|
||||
// Move divider
|
||||
const index = Number.parseInt(dragTarget.split('-')[1], 10)
|
||||
const startDividers = dragStartRef.current.dividers
|
||||
|
||||
// Calculate dx as fraction of quad width (average of top and bottom widths)
|
||||
const topWidth = startCorners.topRight.x - startCorners.topLeft.x
|
||||
const bottomWidth = startCorners.bottomRight.x - startCorners.bottomLeft.x
|
||||
const avgWidth = (topWidth + bottomWidth) / 2
|
||||
const dxFraction = dx / avgWidth
|
||||
|
||||
const newDividers = [...startDividers]
|
||||
const minPos = index === 0 ? 0.05 : startDividers[index - 1] + 0.05
|
||||
const maxPos = index === startDividers.length - 1 ? 0.95 : startDividers[index + 1] - 0.05
|
||||
newDividers[index] = Math.max(minPos, Math.min(maxPos, startDividers[index] + dxFraction))
|
||||
setDividers(newDividers)
|
||||
}
|
||||
},
|
||||
[dragTarget, scale, videoWidth, videoHeight]
|
||||
)
|
||||
|
||||
// Handle pointer up
|
||||
const handlePointerUp = useCallback(() => {
|
||||
setDragTarget(null)
|
||||
dragStartRef.current = null
|
||||
}, [])
|
||||
|
||||
// Handle complete
|
||||
const handleComplete = useCallback(() => {
|
||||
const grid: CalibrationGrid = {
|
||||
roi: cornersToROI(corners),
|
||||
corners,
|
||||
columnCount,
|
||||
columnDividers: dividers,
|
||||
rotation: 0, // Deprecated - perspective handled by corners
|
||||
}
|
||||
onComplete(grid)
|
||||
}, [corners, columnCount, dividers, onComplete])
|
||||
|
||||
// Convert corners to display coordinates (accounting for letterbox offset)
|
||||
const displayCorners: QuadCorners = {
|
||||
topLeft: { x: corners.topLeft.x * scale + videoOffsetX, y: corners.topLeft.y * scale + videoOffsetY },
|
||||
topRight: { x: corners.topRight.x * scale + videoOffsetX, y: corners.topRight.y * scale + videoOffsetY },
|
||||
bottomLeft: { x: corners.bottomLeft.x * scale + videoOffsetX, y: corners.bottomLeft.y * scale + videoOffsetY },
|
||||
bottomRight: { x: corners.bottomRight.x * scale + videoOffsetX, y: corners.bottomRight.y * scale + videoOffsetY },
|
||||
}
|
||||
|
||||
// Create SVG path for the quadrilateral
|
||||
const quadPath = `M ${displayCorners.topLeft.x} ${displayCorners.topLeft.y}
|
||||
L ${displayCorners.topRight.x} ${displayCorners.topRight.y}
|
||||
L ${displayCorners.bottomRight.x} ${displayCorners.bottomRight.y}
|
||||
L ${displayCorners.bottomLeft.x} ${displayCorners.bottomLeft.y} Z`
|
||||
|
||||
const handleSize = 16
|
||||
|
||||
// Corner positions for handles
|
||||
const cornerPositions: { key: CornerKey; point: Point }[] = [
|
||||
{ key: 'topLeft', point: displayCorners.topLeft },
|
||||
{ key: 'topRight', point: displayCorners.topRight },
|
||||
{ key: 'bottomLeft', point: displayCorners.bottomLeft },
|
||||
{ key: 'bottomRight', point: displayCorners.bottomRight },
|
||||
]
|
||||
|
||||
return (
|
||||
<div
|
||||
data-component="calibration-overlay"
|
||||
className={css({
|
||||
position: 'absolute',
|
||||
inset: 0,
|
||||
zIndex: 10,
|
||||
})}
|
||||
onPointerMove={handlePointerMove}
|
||||
onPointerUp={handlePointerUp}
|
||||
>
|
||||
{/* Semi-transparent overlay outside quadrilateral */}
|
||||
<svg
|
||||
width={containerWidth}
|
||||
height={containerHeight}
|
||||
className={css({ position: 'absolute', inset: 0 })}
|
||||
>
|
||||
{/* Darkened area outside quadrilateral */}
|
||||
<defs>
|
||||
<mask id="quad-mask">
|
||||
<rect width="100%" height="100%" fill="white" />
|
||||
<path d={quadPath} fill="black" />
|
||||
</mask>
|
||||
</defs>
|
||||
<rect
|
||||
width="100%"
|
||||
height="100%"
|
||||
fill="rgba(0, 0, 0, 0.5)"
|
||||
mask="url(#quad-mask)"
|
||||
style={{ pointerEvents: 'none' }}
|
||||
/>
|
||||
|
||||
{/* Clickable fill area inside quadrilateral - for moving the entire quad */}
|
||||
<path
|
||||
d={quadPath}
|
||||
fill="transparent"
|
||||
style={{ cursor: dragTarget === 'quad' ? 'grabbing' : 'grab' }}
|
||||
onPointerDown={(e) => handlePointerDown(e, 'quad')}
|
||||
/>
|
||||
|
||||
{/* Quadrilateral border */}
|
||||
<path
|
||||
d={quadPath}
|
||||
fill="none"
|
||||
stroke="#4ade80"
|
||||
strokeWidth="2"
|
||||
strokeDasharray="8,4"
|
||||
style={{ pointerEvents: 'none' }}
|
||||
/>
|
||||
|
||||
{/* Column divider lines */}
|
||||
{dividers.map((divider, i) => {
|
||||
const line = getColumnLine(divider, displayCorners)
|
||||
return (
|
||||
<line
|
||||
key={i}
|
||||
x1={line.top.x}
|
||||
y1={line.top.y}
|
||||
x2={line.bottom.x}
|
||||
y2={line.bottom.y}
|
||||
stroke="#facc15"
|
||||
strokeWidth="3"
|
||||
style={{ cursor: 'ew-resize' }}
|
||||
onPointerDown={(e) => handlePointerDown(e, `divider-${i}`)}
|
||||
/>
|
||||
)
|
||||
})}
|
||||
|
||||
{/* Beam indicator line (20% from top, interpolated) */}
|
||||
{(() => {
|
||||
const beamT = 0.2
|
||||
const leftPoint: Point = {
|
||||
x:
|
||||
displayCorners.topLeft.x +
|
||||
beamT * (displayCorners.bottomLeft.x - displayCorners.topLeft.x),
|
||||
y:
|
||||
displayCorners.topLeft.y +
|
||||
beamT * (displayCorners.bottomLeft.y - displayCorners.topLeft.y),
|
||||
}
|
||||
const rightPoint: Point = {
|
||||
x:
|
||||
displayCorners.topRight.x +
|
||||
beamT * (displayCorners.bottomRight.x - displayCorners.topRight.x),
|
||||
y:
|
||||
displayCorners.topRight.y +
|
||||
beamT * (displayCorners.bottomRight.y - displayCorners.topRight.y),
|
||||
}
|
||||
return (
|
||||
<line
|
||||
x1={leftPoint.x}
|
||||
y1={leftPoint.y}
|
||||
x2={rightPoint.x}
|
||||
y2={rightPoint.y}
|
||||
stroke="#22d3ee"
|
||||
strokeWidth="2"
|
||||
strokeDasharray="4,4"
|
||||
opacity="0.7"
|
||||
/>
|
||||
)
|
||||
})()}
|
||||
</svg>
|
||||
|
||||
{/* Corner drag handles */}
|
||||
{cornerPositions.map(({ key, point }) => (
|
||||
<div
|
||||
key={key}
|
||||
data-element={`handle-${key}`}
|
||||
className={css({
|
||||
position: 'absolute',
|
||||
width: `${handleSize}px`,
|
||||
height: `${handleSize}px`,
|
||||
bg: 'green.400',
|
||||
border: '2px solid white',
|
||||
borderRadius: 'full',
|
||||
cursor: 'move',
|
||||
transform: 'translate(-50%, -50%)',
|
||||
_hover: { bg: 'green.300', transform: 'translate(-50%, -50%) scale(1.2)' },
|
||||
})}
|
||||
style={{
|
||||
left: point.x,
|
||||
top: point.y,
|
||||
}}
|
||||
onPointerDown={(e) => handlePointerDown(e, key)}
|
||||
/>
|
||||
))}
|
||||
|
||||
{/* Compact instructions (collapsible) */}
|
||||
{showInstructions && (
|
||||
<div
|
||||
data-element="calibration-instructions"
|
||||
className={css({
|
||||
position: 'absolute',
|
||||
top: 2,
|
||||
left: 2,
|
||||
p: 2,
|
||||
bg: 'rgba(0, 0, 0, 0.75)',
|
||||
borderRadius: 'md',
|
||||
color: 'white',
|
||||
fontSize: 'xs',
|
||||
maxWidth: '180px',
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
gap: 1,
|
||||
})}
|
||||
>
|
||||
<div
|
||||
className={css({
|
||||
display: 'flex',
|
||||
justifyContent: 'space-between',
|
||||
alignItems: 'center',
|
||||
})}
|
||||
>
|
||||
<span className={css({ fontWeight: 'medium' })}>Calibration</span>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setShowInstructions(false)}
|
||||
className={css({
|
||||
bg: 'transparent',
|
||||
border: 'none',
|
||||
color: 'gray.400',
|
||||
cursor: 'pointer',
|
||||
p: 0,
|
||||
fontSize: 'sm',
|
||||
lineHeight: 1,
|
||||
_hover: { color: 'white' },
|
||||
})}
|
||||
aria-label="Hide instructions"
|
||||
>
|
||||
×
|
||||
</button>
|
||||
</div>
|
||||
<p className={css({ color: 'gray.300' })}>
|
||||
Drag inside to move. Drag corners to resize. Yellow = columns.
|
||||
</p>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Show instructions button (when hidden) */}
|
||||
{!showInstructions && (
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setShowInstructions(true)}
|
||||
className={css({
|
||||
position: 'absolute',
|
||||
top: 2,
|
||||
left: 2,
|
||||
px: 2,
|
||||
py: 1,
|
||||
bg: 'rgba(0, 0, 0, 0.6)',
|
||||
border: 'none',
|
||||
borderRadius: 'md',
|
||||
color: 'gray.300',
|
||||
fontSize: 'xs',
|
||||
cursor: 'pointer',
|
||||
_hover: { bg: 'rgba(0, 0, 0, 0.8)', color: 'white' },
|
||||
})}
|
||||
>
|
||||
?
|
||||
</button>
|
||||
)}
|
||||
|
||||
{/* Control buttons */}
|
||||
<div
|
||||
data-element="calibration-controls"
|
||||
className={css({
|
||||
position: 'absolute',
|
||||
top: 2,
|
||||
right: 2,
|
||||
display: 'flex',
|
||||
gap: 2,
|
||||
})}
|
||||
>
|
||||
<button
|
||||
type="button"
|
||||
onClick={onCancel}
|
||||
className={css({
|
||||
px: 3,
|
||||
py: 1.5,
|
||||
bg: 'gray.700',
|
||||
color: 'white',
|
||||
borderRadius: 'md',
|
||||
fontSize: 'sm',
|
||||
border: 'none',
|
||||
cursor: 'pointer',
|
||||
_hover: { bg: 'gray.600' },
|
||||
})}
|
||||
>
|
||||
Cancel
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={handleComplete}
|
||||
className={css({
|
||||
px: 3,
|
||||
py: 1.5,
|
||||
bg: 'green.600',
|
||||
color: 'white',
|
||||
borderRadius: 'md',
|
||||
fontSize: 'sm',
|
||||
fontWeight: 'medium',
|
||||
border: 'none',
|
||||
cursor: 'pointer',
|
||||
_hover: { bg: 'green.500' },
|
||||
})}
|
||||
>
|
||||
Done
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default CalibrationOverlay
|
||||
|
|
@ -0,0 +1,453 @@
|
|||
'use client'
|
||||
|
||||
import type { ReactNode } from 'react'
|
||||
import { useCallback, useEffect, useRef, useState } from 'react'
|
||||
import { css } from '../../../styled-system/css'
|
||||
import type { CalibrationGrid, QuadCorners } from '@/types/vision'
|
||||
import { isOpenCVReady, loadOpenCV, rectifyQuadrilateral } from '@/lib/vision/perspectiveTransform'
|
||||
|
||||
export interface VisionCameraFeedProps {
|
||||
/** Video stream to display */
|
||||
videoStream: MediaStream | null
|
||||
/** Whether camera is currently loading */
|
||||
isLoading?: boolean
|
||||
/** Calibration grid to visualize (optional) */
|
||||
calibration?: CalibrationGrid | null
|
||||
/** Whether to show the calibration grid overlay */
|
||||
showCalibrationGrid?: boolean
|
||||
/** Whether to show rectified (perspective-corrected) view */
|
||||
showRectifiedView?: boolean
|
||||
/** Video element ref callback for external access */
|
||||
videoRef?: (el: HTMLVideoElement | null) => void
|
||||
/** Called when video metadata is loaded (provides dimensions) */
|
||||
onVideoReady?: (width: number, height: number) => void
|
||||
/** Children rendered over the video (e.g., CalibrationOverlay) */
|
||||
children?: ReactNode
|
||||
}
|
||||
|
||||
/**
|
||||
* Get corners from calibration (prefer QuadCorners, fall back to ROI rectangle)
|
||||
*/
|
||||
function getCornersFromCalibration(calibration: CalibrationGrid): QuadCorners {
|
||||
if (calibration.corners) {
|
||||
return calibration.corners
|
||||
}
|
||||
const roi = calibration.roi
|
||||
return {
|
||||
topLeft: { x: roi.x, y: roi.y },
|
||||
topRight: { x: roi.x + roi.width, y: roi.y },
|
||||
bottomLeft: { x: roi.x, y: roi.y + roi.height },
|
||||
bottomRight: { x: roi.x + roi.width, y: roi.y + roi.height },
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* VisionCameraFeed - Displays camera video with optional calibration grid overlay
|
||||
*
|
||||
* Renders the video stream and optionally draws the calibration grid
|
||||
* showing the ROI and column dividers. Can also show a rectified
|
||||
* (perspective-corrected) view of the calibrated region.
|
||||
*/
|
||||
export function VisionCameraFeed({
|
||||
videoStream,
|
||||
isLoading = false,
|
||||
calibration,
|
||||
showCalibrationGrid = false,
|
||||
showRectifiedView = false,
|
||||
videoRef: externalVideoRef,
|
||||
onVideoReady,
|
||||
children,
|
||||
}: VisionCameraFeedProps): ReactNode {
|
||||
const internalVideoRef = useRef<HTMLVideoElement>(null)
|
||||
const canvasRef = useRef<HTMLCanvasElement>(null)
|
||||
const rectifiedCanvasRef = useRef<HTMLCanvasElement>(null)
|
||||
const containerRef = useRef<HTMLDivElement>(null)
|
||||
const animationFrameRef = useRef<number | null>(null)
|
||||
|
||||
const [opencvReady, setOpencvReady] = useState(false)
|
||||
|
||||
// Load OpenCV when rectified view is needed
|
||||
useEffect(() => {
|
||||
if (showRectifiedView && !opencvReady) {
|
||||
loadOpenCV()
|
||||
.then(() => setOpencvReady(true))
|
||||
.catch((err) => console.error('[VisionCameraFeed] Failed to load OpenCV:', err))
|
||||
}
|
||||
}, [showRectifiedView, opencvReady])
|
||||
|
||||
// Set video ref for external access
|
||||
useEffect(() => {
|
||||
if (externalVideoRef) {
|
||||
externalVideoRef(internalVideoRef.current)
|
||||
}
|
||||
}, [externalVideoRef])
|
||||
|
||||
// Attach stream to video element
|
||||
useEffect(() => {
|
||||
const video = internalVideoRef.current
|
||||
if (!video) return
|
||||
|
||||
let isMounted = true
|
||||
|
||||
if (videoStream) {
|
||||
video.srcObject = videoStream
|
||||
video.play().catch((err) => {
|
||||
// Ignore AbortError - happens when component unmounts during play()
|
||||
if (isMounted && err.name !== 'AbortError') {
|
||||
console.error('Failed to play video:', err)
|
||||
}
|
||||
})
|
||||
} else {
|
||||
video.srcObject = null
|
||||
}
|
||||
|
||||
return () => {
|
||||
isMounted = false
|
||||
}
|
||||
}, [videoStream])
|
||||
|
||||
// Handle video metadata loaded
|
||||
useEffect(() => {
|
||||
const video = internalVideoRef.current
|
||||
if (!video || !videoStream) return
|
||||
|
||||
const handleLoadedMetadata = () => {
|
||||
if (onVideoReady && video.videoWidth > 0 && video.videoHeight > 0) {
|
||||
onVideoReady(video.videoWidth, video.videoHeight)
|
||||
}
|
||||
}
|
||||
|
||||
// Check if already loaded (e.g., from cache or fast load)
|
||||
if (video.readyState >= 1 && video.videoWidth > 0) {
|
||||
handleLoadedMetadata()
|
||||
}
|
||||
|
||||
video.addEventListener('loadedmetadata', handleLoadedMetadata)
|
||||
return () => {
|
||||
video.removeEventListener('loadedmetadata', handleLoadedMetadata)
|
||||
}
|
||||
}, [videoStream, onVideoReady])
|
||||
|
||||
// Rectified view processing loop
|
||||
useEffect(() => {
|
||||
if (!showRectifiedView || !calibration || !opencvReady) {
|
||||
return
|
||||
}
|
||||
|
||||
const video = internalVideoRef.current
|
||||
const canvas = rectifiedCanvasRef.current
|
||||
if (!video || !canvas) return
|
||||
|
||||
const corners = getCornersFromCalibration(calibration)
|
||||
let running = true
|
||||
|
||||
const processFrame = () => {
|
||||
if (!running) return
|
||||
|
||||
if (video.readyState >= 2 && isOpenCVReady()) {
|
||||
rectifyQuadrilateral(video, corners, canvas)
|
||||
}
|
||||
|
||||
animationFrameRef.current = requestAnimationFrame(processFrame)
|
||||
}
|
||||
|
||||
processFrame()
|
||||
|
||||
return () => {
|
||||
running = false
|
||||
if (animationFrameRef.current !== null) {
|
||||
cancelAnimationFrame(animationFrameRef.current)
|
||||
animationFrameRef.current = null
|
||||
}
|
||||
}
|
||||
}, [showRectifiedView, calibration, opencvReady])
|
||||
|
||||
// Draw calibration grid overlay (only when not in rectified mode)
|
||||
useEffect(() => {
|
||||
if (!showCalibrationGrid || !calibration || showRectifiedView) return
|
||||
|
||||
const canvas = canvasRef.current
|
||||
const video = internalVideoRef.current
|
||||
if (!canvas || !video) return
|
||||
|
||||
const ctx = canvas.getContext('2d')
|
||||
if (!ctx) return
|
||||
|
||||
const drawGrid = () => {
|
||||
// Match canvas size to video display size
|
||||
const rect = video.getBoundingClientRect()
|
||||
canvas.width = rect.width
|
||||
canvas.height = rect.height
|
||||
|
||||
// Calculate scale factors
|
||||
const scaleX = rect.width / video.videoWidth
|
||||
const scaleY = rect.height / video.videoHeight
|
||||
|
||||
// Clear canvas
|
||||
ctx.clearRect(0, 0, canvas.width, canvas.height)
|
||||
|
||||
const corners = getCornersFromCalibration(calibration)
|
||||
|
||||
// Scale corners to display coordinates
|
||||
const displayCorners = {
|
||||
topLeft: { x: corners.topLeft.x * scaleX, y: corners.topLeft.y * scaleY },
|
||||
topRight: { x: corners.topRight.x * scaleX, y: corners.topRight.y * scaleY },
|
||||
bottomLeft: { x: corners.bottomLeft.x * scaleX, y: corners.bottomLeft.y * scaleY },
|
||||
bottomRight: { x: corners.bottomRight.x * scaleX, y: corners.bottomRight.y * scaleY },
|
||||
}
|
||||
|
||||
// Draw quadrilateral border
|
||||
ctx.strokeStyle = '#00ff00'
|
||||
ctx.lineWidth = 2
|
||||
ctx.setLineDash([5, 5])
|
||||
ctx.beginPath()
|
||||
ctx.moveTo(displayCorners.topLeft.x, displayCorners.topLeft.y)
|
||||
ctx.lineTo(displayCorners.topRight.x, displayCorners.topRight.y)
|
||||
ctx.lineTo(displayCorners.bottomRight.x, displayCorners.bottomRight.y)
|
||||
ctx.lineTo(displayCorners.bottomLeft.x, displayCorners.bottomLeft.y)
|
||||
ctx.closePath()
|
||||
ctx.stroke()
|
||||
|
||||
// Draw column dividers (interpolate along top and bottom edges)
|
||||
ctx.setLineDash([])
|
||||
ctx.strokeStyle = '#00ff00'
|
||||
ctx.lineWidth = 1
|
||||
|
||||
for (const divider of calibration.columnDividers) {
|
||||
const topX =
|
||||
displayCorners.topLeft.x +
|
||||
divider * (displayCorners.topRight.x - displayCorners.topLeft.x)
|
||||
const topY =
|
||||
displayCorners.topLeft.y +
|
||||
divider * (displayCorners.topRight.y - displayCorners.topLeft.y)
|
||||
const bottomX =
|
||||
displayCorners.bottomLeft.x +
|
||||
divider * (displayCorners.bottomRight.x - displayCorners.bottomLeft.x)
|
||||
const bottomY =
|
||||
displayCorners.bottomLeft.y +
|
||||
divider * (displayCorners.bottomRight.y - displayCorners.bottomLeft.y)
|
||||
|
||||
ctx.beginPath()
|
||||
ctx.moveTo(topX, topY)
|
||||
ctx.lineTo(bottomX, bottomY)
|
||||
ctx.stroke()
|
||||
}
|
||||
|
||||
// Draw beam line (20% from top, interpolated along left and right edges)
|
||||
ctx.strokeStyle = '#ffff00'
|
||||
ctx.setLineDash([3, 3])
|
||||
const beamT = 0.2
|
||||
const beamLeftX =
|
||||
displayCorners.topLeft.x + beamT * (displayCorners.bottomLeft.x - displayCorners.topLeft.x)
|
||||
const beamLeftY =
|
||||
displayCorners.topLeft.y + beamT * (displayCorners.bottomLeft.y - displayCorners.topLeft.y)
|
||||
const beamRightX =
|
||||
displayCorners.topRight.x +
|
||||
beamT * (displayCorners.bottomRight.x - displayCorners.topRight.x)
|
||||
const beamRightY =
|
||||
displayCorners.topRight.y +
|
||||
beamT * (displayCorners.bottomRight.y - displayCorners.topRight.y)
|
||||
ctx.beginPath()
|
||||
ctx.moveTo(beamLeftX, beamLeftY)
|
||||
ctx.lineTo(beamRightX, beamRightY)
|
||||
ctx.stroke()
|
||||
}
|
||||
|
||||
// Draw on next frame
|
||||
const animationId = requestAnimationFrame(drawGrid)
|
||||
return () => cancelAnimationFrame(animationId)
|
||||
}, [showCalibrationGrid, calibration, showRectifiedView])
|
||||
|
||||
// Draw column dividers on rectified view
|
||||
const drawRectifiedOverlay = useCallback(() => {
|
||||
if (!showRectifiedView || !calibration) return
|
||||
|
||||
const canvas = rectifiedCanvasRef.current
|
||||
if (!canvas) return
|
||||
|
||||
const ctx = canvas.getContext('2d')
|
||||
if (!ctx) return
|
||||
|
||||
const width = canvas.width
|
||||
const height = canvas.height
|
||||
|
||||
// Draw column dividers on rectified canvas
|
||||
ctx.strokeStyle = '#00ff00'
|
||||
ctx.lineWidth = 1
|
||||
ctx.setLineDash([])
|
||||
|
||||
for (const divider of calibration.columnDividers) {
|
||||
const x = divider * width
|
||||
ctx.beginPath()
|
||||
ctx.moveTo(x, 0)
|
||||
ctx.lineTo(x, height)
|
||||
ctx.stroke()
|
||||
}
|
||||
|
||||
// Draw beam line
|
||||
ctx.strokeStyle = '#ffff00'
|
||||
ctx.setLineDash([3, 3])
|
||||
const beamY = height * 0.2
|
||||
ctx.beginPath()
|
||||
ctx.moveTo(0, beamY)
|
||||
ctx.lineTo(width, beamY)
|
||||
ctx.stroke()
|
||||
}, [showRectifiedView, calibration])
|
||||
|
||||
// Draw overlay after each rectification frame
|
||||
useEffect(() => {
|
||||
if (!showRectifiedView || !calibration || !opencvReady) return
|
||||
|
||||
// Set up a loop to draw overlay after rectification
|
||||
let running = true
|
||||
const overlayLoop = () => {
|
||||
if (!running) return
|
||||
drawRectifiedOverlay()
|
||||
requestAnimationFrame(overlayLoop)
|
||||
}
|
||||
// Delay slightly to let rectification happen first
|
||||
const timeoutId = setTimeout(() => {
|
||||
overlayLoop()
|
||||
}, 100)
|
||||
|
||||
return () => {
|
||||
running = false
|
||||
clearTimeout(timeoutId)
|
||||
}
|
||||
}, [showRectifiedView, calibration, opencvReady, drawRectifiedOverlay])
|
||||
|
||||
if (!videoStream) {
|
||||
return (
|
||||
<div
|
||||
data-component="vision-camera-feed"
|
||||
data-status={isLoading ? 'loading' : 'no-stream'}
|
||||
className={css({
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
gap: 2,
|
||||
bg: 'gray.900',
|
||||
color: 'gray.400',
|
||||
borderRadius: 'lg',
|
||||
aspectRatio: '4/3',
|
||||
fontSize: 'sm',
|
||||
})}
|
||||
>
|
||||
{isLoading ? (
|
||||
<>
|
||||
<div
|
||||
className={css({
|
||||
width: '24px',
|
||||
height: '24px',
|
||||
border: '2px solid',
|
||||
borderColor: 'gray.600',
|
||||
borderTopColor: 'blue.400',
|
||||
borderRadius: 'full',
|
||||
})}
|
||||
style={{ animation: 'spin 1s linear infinite' }}
|
||||
/>
|
||||
<style>{`@keyframes spin { to { transform: rotate(360deg); } }`}</style>
|
||||
<span>Requesting camera...</span>
|
||||
</>
|
||||
) : (
|
||||
'No camera feed'
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
// Show loading indicator while keeping video element mounted (so stream stays attached)
|
||||
const isLoadingOpenCV = showRectifiedView && !opencvReady
|
||||
|
||||
return (
|
||||
<div
|
||||
ref={containerRef}
|
||||
data-component="vision-camera-feed"
|
||||
data-status={showRectifiedView ? 'rectified' : 'active'}
|
||||
className={css({
|
||||
position: 'relative',
|
||||
borderRadius: 'lg',
|
||||
overflow: 'hidden',
|
||||
bg: 'black',
|
||||
})}
|
||||
>
|
||||
{/* Video element - always mounted (may be hidden when showing rectified view) */}
|
||||
<video
|
||||
ref={internalVideoRef}
|
||||
autoPlay
|
||||
playsInline
|
||||
muted
|
||||
className={css({
|
||||
width: '100%',
|
||||
height: 'auto',
|
||||
display: showRectifiedView && !isLoadingOpenCV ? 'none' : 'block',
|
||||
})}
|
||||
/>
|
||||
|
||||
{/* Loading OpenCV indicator (overlays video) */}
|
||||
{isLoadingOpenCV && (
|
||||
<div
|
||||
data-element="opencv-loading-overlay"
|
||||
className={css({
|
||||
position: 'absolute',
|
||||
inset: 0,
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
gap: 2,
|
||||
bg: 'rgba(0, 0, 0, 0.7)',
|
||||
color: 'gray.400',
|
||||
fontSize: 'sm',
|
||||
})}
|
||||
>
|
||||
<div
|
||||
className={css({
|
||||
width: '24px',
|
||||
height: '24px',
|
||||
border: '2px solid',
|
||||
borderColor: 'gray.600',
|
||||
borderTopColor: 'blue.400',
|
||||
borderRadius: 'full',
|
||||
})}
|
||||
style={{ animation: 'spin 1s linear infinite' }}
|
||||
/>
|
||||
<style>{`@keyframes spin { to { transform: rotate(360deg); } }`}</style>
|
||||
<span>Loading vision processing...</span>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Rectified view canvas */}
|
||||
{showRectifiedView && !isLoadingOpenCV && (
|
||||
<canvas
|
||||
ref={rectifiedCanvasRef}
|
||||
data-element="rectified-canvas"
|
||||
className={css({
|
||||
width: '100%',
|
||||
height: 'auto',
|
||||
display: 'block',
|
||||
})}
|
||||
/>
|
||||
)}
|
||||
|
||||
{/* Calibration grid overlay (only when not in rectified mode) */}
|
||||
{showCalibrationGrid && !showRectifiedView && (
|
||||
<canvas
|
||||
ref={canvasRef}
|
||||
data-element="calibration-grid-canvas"
|
||||
className={css({
|
||||
position: 'absolute',
|
||||
inset: 0,
|
||||
pointerEvents: 'none',
|
||||
})}
|
||||
/>
|
||||
)}
|
||||
|
||||
{/* Children (e.g., CalibrationOverlay during calibration) */}
|
||||
{children}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default VisionCameraFeed
|
||||
|
|
@ -0,0 +1,192 @@
|
|||
'use client'
|
||||
|
||||
import type { ReactNode } from 'react'
|
||||
import { css } from '../../../styled-system/css'
|
||||
|
||||
export interface VisionStatusIndicatorProps {
|
||||
/** Whether calibration is complete */
|
||||
isCalibrated: boolean
|
||||
/** Whether actively detecting */
|
||||
isDetecting: boolean
|
||||
/** Detection confidence (0-1) */
|
||||
confidence: number
|
||||
/** Whether hand motion is detected */
|
||||
handDetected: boolean
|
||||
/** Current detected value (null if not stable) */
|
||||
detectedValue: number | null
|
||||
/** Number of consecutive stable frames */
|
||||
consecutiveFrames: number
|
||||
/** Minimum frames needed for stable detection */
|
||||
minFrames?: number
|
||||
}
|
||||
|
||||
/**
|
||||
* VisionStatusIndicator - Shows current detection status
|
||||
*
|
||||
* Displays:
|
||||
* - Calibration status
|
||||
* - Detection status (detecting, stable, hand blocking)
|
||||
* - Confidence level
|
||||
* - Current detected value
|
||||
*/
|
||||
export function VisionStatusIndicator({
|
||||
isCalibrated,
|
||||
isDetecting,
|
||||
confidence,
|
||||
handDetected,
|
||||
detectedValue,
|
||||
consecutiveFrames,
|
||||
minFrames = 10,
|
||||
}: VisionStatusIndicatorProps): ReactNode {
|
||||
// Determine status
|
||||
let status: 'uncalibrated' | 'detecting' | 'stable' | 'hand-blocking' = 'uncalibrated'
|
||||
let statusColor = 'gray.500'
|
||||
let statusText = 'Not calibrated'
|
||||
|
||||
if (!isCalibrated) {
|
||||
status = 'uncalibrated'
|
||||
statusColor = 'gray.500'
|
||||
statusText = 'Not calibrated'
|
||||
} else if (handDetected) {
|
||||
status = 'hand-blocking'
|
||||
statusColor = 'orange.500'
|
||||
statusText = 'Hand detected'
|
||||
} else if (detectedValue !== null && consecutiveFrames >= minFrames) {
|
||||
status = 'stable'
|
||||
statusColor = 'green.500'
|
||||
statusText = 'Stable'
|
||||
} else if (isDetecting) {
|
||||
status = 'detecting'
|
||||
statusColor = 'yellow.500'
|
||||
statusText = 'Detecting...'
|
||||
}
|
||||
|
||||
// Confidence bar width
|
||||
const confidencePercent = Math.round(confidence * 100)
|
||||
|
||||
return (
|
||||
<div
|
||||
data-component="vision-status-indicator"
|
||||
data-status={status}
|
||||
className={css({
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
gap: 1,
|
||||
p: 2,
|
||||
bg: 'rgba(0, 0, 0, 0.7)',
|
||||
borderRadius: 'lg',
|
||||
minWidth: '120px',
|
||||
})}
|
||||
>
|
||||
{/* Status indicator */}
|
||||
<div
|
||||
data-element="status-row"
|
||||
className={css({
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
gap: 2,
|
||||
})}
|
||||
>
|
||||
{/* Status dot */}
|
||||
<div
|
||||
data-element="status-dot"
|
||||
className={css({
|
||||
width: '8px',
|
||||
height: '8px',
|
||||
borderRadius: 'full',
|
||||
animation: status === 'detecting' ? 'pulse 1s infinite' : 'none',
|
||||
})}
|
||||
style={{ backgroundColor: `var(--colors-${statusColor.replace('.', '-')})` }}
|
||||
/>
|
||||
<span
|
||||
className={css({
|
||||
fontSize: 'xs',
|
||||
color: 'white',
|
||||
fontWeight: 'medium',
|
||||
})}
|
||||
>
|
||||
{statusText}
|
||||
</span>
|
||||
</div>
|
||||
|
||||
{/* Detected value */}
|
||||
{isCalibrated && detectedValue !== null && (
|
||||
<div
|
||||
data-element="detected-value"
|
||||
className={css({
|
||||
fontSize: 'xl',
|
||||
fontWeight: 'bold',
|
||||
color: status === 'stable' ? 'green.300' : 'yellow.300',
|
||||
fontFamily: 'mono',
|
||||
textAlign: 'center',
|
||||
})}
|
||||
>
|
||||
{detectedValue}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Confidence bar */}
|
||||
{isCalibrated && isDetecting && (
|
||||
<div
|
||||
data-element="confidence-bar"
|
||||
className={css({
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
gap: 0.5,
|
||||
})}
|
||||
>
|
||||
<div
|
||||
className={css({
|
||||
fontSize: 'xs',
|
||||
color: 'gray.400',
|
||||
})}
|
||||
>
|
||||
Confidence: {confidencePercent}%
|
||||
</div>
|
||||
<div
|
||||
className={css({
|
||||
height: '4px',
|
||||
bg: 'gray.700',
|
||||
borderRadius: 'full',
|
||||
overflow: 'hidden',
|
||||
})}
|
||||
>
|
||||
<div
|
||||
className={css({
|
||||
height: '100%',
|
||||
borderRadius: 'full',
|
||||
transition: 'width 0.2s',
|
||||
})}
|
||||
style={{
|
||||
width: `${confidencePercent}%`,
|
||||
backgroundColor:
|
||||
confidence > 0.8
|
||||
? 'var(--colors-green-500)'
|
||||
: confidence > 0.5
|
||||
? 'var(--colors-yellow-500)'
|
||||
: 'var(--colors-red-500)',
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Stability progress */}
|
||||
{isCalibrated && isDetecting && !handDetected && consecutiveFrames > 0 && (
|
||||
<div
|
||||
data-element="stability-progress"
|
||||
className={css({
|
||||
fontSize: 'xs',
|
||||
color: 'gray.400',
|
||||
})}
|
||||
>
|
||||
{consecutiveFrames >= minFrames
|
||||
? 'Locked'
|
||||
: `Stabilizing... ${consecutiveFrames}/${minFrames}`}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default VisionStatusIndicator
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
/**
|
||||
* Vision components for physical abacus detection
|
||||
*
|
||||
* @module components/vision
|
||||
*/
|
||||
|
||||
export { AbacusVisionBridge } from './AbacusVisionBridge'
|
||||
export type { AbacusVisionBridgeProps } from './AbacusVisionBridge'
|
||||
|
||||
export { VisionCameraFeed } from './VisionCameraFeed'
|
||||
export type { VisionCameraFeedProps } from './VisionCameraFeed'
|
||||
|
||||
export { CalibrationOverlay } from './CalibrationOverlay'
|
||||
export type { CalibrationOverlayProps } from './CalibrationOverlay'
|
||||
|
||||
export { VisionStatusIndicator } from './VisionStatusIndicator'
|
||||
export type { VisionStatusIndicatorProps } from './VisionStatusIndicator'
|
||||
|
|
@ -0,0 +1,262 @@
|
|||
'use client'
|
||||
|
||||
import { useCallback, useEffect, useRef, useState } from 'react'
|
||||
import type { CalibrationGrid, UseAbacusVisionReturn } from '@/types/vision'
|
||||
import { useCameraCalibration } from './useCameraCalibration'
|
||||
import { useDeskViewCamera } from './useDeskViewCamera'
|
||||
import { useFrameStability } from './useFrameStability'
|
||||
|
||||
export interface UseAbacusVisionOptions {
|
||||
/** Number of abacus columns to detect */
|
||||
columnCount?: number
|
||||
/** Called when a stable value is detected */
|
||||
onValueDetected?: (value: number) => void
|
||||
}
|
||||
|
||||
/**
|
||||
* useAbacusVision - Primary coordinator hook for abacus vision detection
|
||||
*
|
||||
* Combines camera management, calibration, and frame processing into
|
||||
* a single hook that outputs stable detected values.
|
||||
*
|
||||
* Usage:
|
||||
* ```tsx
|
||||
* const vision = useAbacusVision({
|
||||
* columnCount: 5,
|
||||
* onValueDetected: (value) => setDockedValue(value)
|
||||
* })
|
||||
*
|
||||
* return (
|
||||
* <VisionCameraFeed
|
||||
* videoStream={vision.videoStream}
|
||||
* calibration={vision.calibrationGrid}
|
||||
* />
|
||||
* )
|
||||
* ```
|
||||
*/
|
||||
export function useAbacusVision(options: UseAbacusVisionOptions = {}): UseAbacusVisionReturn {
|
||||
const { columnCount = 5, onValueDetected } = options
|
||||
|
||||
// State
|
||||
const [isEnabled, setIsEnabled] = useState(false)
|
||||
const [isDetecting, setIsDetecting] = useState(false)
|
||||
|
||||
// Sub-hooks
|
||||
const camera = useDeskViewCamera()
|
||||
const calibration = useCameraCalibration()
|
||||
const stability = useFrameStability()
|
||||
|
||||
// Video element ref for frame capture
|
||||
const videoRef = useRef<HTMLVideoElement | null>(null)
|
||||
const canvasRef = useRef<HTMLCanvasElement | null>(null)
|
||||
const animationFrameRef = useRef<number | null>(null)
|
||||
|
||||
// Track previous stable value to avoid duplicate callbacks
|
||||
const lastStableValueRef = useRef<number | null>(null)
|
||||
|
||||
// Sync device ID to calibration hook when camera device changes
|
||||
useEffect(() => {
|
||||
if (camera.currentDevice?.deviceId) {
|
||||
calibration.setDeviceId(camera.currentDevice.deviceId)
|
||||
}
|
||||
}, [camera.currentDevice?.deviceId, calibration])
|
||||
|
||||
/**
|
||||
* Enable vision mode - start camera and detection
|
||||
*/
|
||||
const enable = useCallback(async () => {
|
||||
setIsEnabled(true)
|
||||
await camera.requestCamera()
|
||||
}, [camera])
|
||||
|
||||
/**
|
||||
* Disable vision mode - stop camera and detection
|
||||
*/
|
||||
const disable = useCallback(() => {
|
||||
setIsEnabled(false)
|
||||
setIsDetecting(false)
|
||||
camera.stopCamera()
|
||||
stability.reset()
|
||||
|
||||
if (animationFrameRef.current) {
|
||||
cancelAnimationFrame(animationFrameRef.current)
|
||||
animationFrameRef.current = null
|
||||
}
|
||||
}, [camera, stability])
|
||||
|
||||
/**
|
||||
* Start calibration mode
|
||||
*/
|
||||
const startCalibration = useCallback(() => {
|
||||
calibration.startCalibration()
|
||||
}, [calibration])
|
||||
|
||||
/**
|
||||
* Finish calibration
|
||||
*/
|
||||
const finishCalibration = useCallback(
|
||||
(grid: CalibrationGrid) => {
|
||||
calibration.updateCalibration(grid)
|
||||
calibration.finishCalibration()
|
||||
},
|
||||
[calibration]
|
||||
)
|
||||
|
||||
/**
|
||||
* Cancel calibration
|
||||
*/
|
||||
const cancelCalibration = useCallback(() => {
|
||||
calibration.cancelCalibration()
|
||||
}, [calibration])
|
||||
|
||||
/**
|
||||
* Select specific camera
|
||||
*/
|
||||
const selectCamera = useCallback(
|
||||
(deviceId: string) => {
|
||||
camera.requestCamera(deviceId)
|
||||
},
|
||||
[camera]
|
||||
)
|
||||
|
||||
/**
|
||||
* Reset calibration
|
||||
*/
|
||||
const resetCalibration = useCallback(() => {
|
||||
calibration.resetCalibration()
|
||||
}, [calibration])
|
||||
|
||||
/**
|
||||
* Process a video frame for detection
|
||||
* (Stub - actual classification will be added when model is ready)
|
||||
*/
|
||||
const processFrame = useCallback(() => {
|
||||
const video = videoRef.current
|
||||
const canvas = canvasRef.current
|
||||
if (!video || !canvas || !calibration.isCalibrated || !calibration.calibration) {
|
||||
return
|
||||
}
|
||||
|
||||
const ctx = canvas.getContext('2d')
|
||||
if (!ctx) return
|
||||
|
||||
// Set canvas size to match video
|
||||
canvas.width = video.videoWidth
|
||||
canvas.height = video.videoHeight
|
||||
|
||||
// Draw video frame to canvas
|
||||
ctx.drawImage(video, 0, 0)
|
||||
|
||||
// Get full frame for motion detection
|
||||
const roi = calibration.calibration.roi
|
||||
const frameData = ctx.getImageData(roi.x, roi.y, roi.width, roi.height)
|
||||
stability.pushFrameData(frameData)
|
||||
|
||||
// TODO: When model is ready, slice into columns and classify
|
||||
// For now, we'll simulate detection with a placeholder
|
||||
// This will be replaced with actual TensorFlow.js inference
|
||||
|
||||
// Placeholder: Read "value" from a simple heuristic or return null
|
||||
// Real implementation will use useColumnClassifier
|
||||
}, [calibration.isCalibrated, calibration.calibration, stability])
|
||||
|
||||
/**
|
||||
* Detection loop
|
||||
*/
|
||||
const runDetectionLoop = useCallback(() => {
|
||||
if (!isEnabled || !calibration.isCalibrated || calibration.isCalibrating) {
|
||||
return
|
||||
}
|
||||
|
||||
setIsDetecting(true)
|
||||
processFrame()
|
||||
|
||||
animationFrameRef.current = requestAnimationFrame(runDetectionLoop)
|
||||
}, [isEnabled, calibration.isCalibrated, calibration.isCalibrating, processFrame])
|
||||
|
||||
// Start/stop detection loop based on state
|
||||
useEffect(() => {
|
||||
if (isEnabled && calibration.isCalibrated && !calibration.isCalibrating && camera.videoStream) {
|
||||
runDetectionLoop()
|
||||
} else {
|
||||
setIsDetecting(false)
|
||||
if (animationFrameRef.current) {
|
||||
cancelAnimationFrame(animationFrameRef.current)
|
||||
animationFrameRef.current = null
|
||||
}
|
||||
}
|
||||
|
||||
return () => {
|
||||
if (animationFrameRef.current) {
|
||||
cancelAnimationFrame(animationFrameRef.current)
|
||||
animationFrameRef.current = null
|
||||
}
|
||||
}
|
||||
}, [
|
||||
isEnabled,
|
||||
calibration.isCalibrated,
|
||||
calibration.isCalibrating,
|
||||
camera.videoStream,
|
||||
runDetectionLoop,
|
||||
])
|
||||
|
||||
// Notify when stable value changes
|
||||
useEffect(() => {
|
||||
if (stability.stableValue !== null && stability.stableValue !== lastStableValueRef.current) {
|
||||
lastStableValueRef.current = stability.stableValue
|
||||
onValueDetected?.(stability.stableValue)
|
||||
}
|
||||
}, [stability.stableValue, onValueDetected])
|
||||
|
||||
// Create hidden canvas for frame processing
|
||||
useEffect(() => {
|
||||
if (!canvasRef.current) {
|
||||
canvasRef.current = document.createElement('canvas')
|
||||
}
|
||||
}, [])
|
||||
|
||||
// Cleanup animation frame on unmount (camera cleanup handled by disable())
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
if (animationFrameRef.current) {
|
||||
cancelAnimationFrame(animationFrameRef.current)
|
||||
}
|
||||
// Note: camera.stopCamera() is called by disable() - don't duplicate
|
||||
}
|
||||
}, [])
|
||||
|
||||
return {
|
||||
// Vision state
|
||||
isEnabled,
|
||||
isCalibrated: calibration.isCalibrated,
|
||||
isDetecting,
|
||||
currentDetectedValue: stability.stableValue,
|
||||
confidence: stability.currentConfidence,
|
||||
columnConfidences: [], // TODO: per-column confidences from classifier
|
||||
|
||||
// Camera state
|
||||
isCameraLoading: camera.isLoading,
|
||||
videoStream: camera.videoStream,
|
||||
cameraError: camera.error,
|
||||
selectedDeviceId: camera.currentDevice?.deviceId ?? null,
|
||||
availableDevices: camera.availableDevices,
|
||||
isDeskViewDetected: camera.isDeskViewDetected,
|
||||
|
||||
// Calibration state
|
||||
calibrationGrid: calibration.calibration,
|
||||
isCalibrating: calibration.isCalibrating,
|
||||
|
||||
// Stability state
|
||||
isHandDetected: stability.isHandDetected,
|
||||
consecutiveFrames: stability.consecutiveFrames,
|
||||
|
||||
// Actions
|
||||
enable,
|
||||
disable,
|
||||
startCalibration,
|
||||
finishCalibration,
|
||||
cancelCalibration,
|
||||
selectCamera,
|
||||
resetCalibration,
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,205 @@
|
|||
'use client'
|
||||
|
||||
import { useCallback, useEffect, useState } from 'react'
|
||||
import type { CalibrationGrid, StoredCalibration } from '@/types/vision'
|
||||
import { CALIBRATION_STORAGE_KEY } from '@/types/vision'
|
||||
|
||||
export interface UseCameraCalibrationReturn {
|
||||
/** Whether a valid calibration exists */
|
||||
isCalibrated: boolean
|
||||
/** Current calibration grid */
|
||||
calibration: CalibrationGrid | null
|
||||
/** Whether currently in calibration mode */
|
||||
isCalibrating: boolean
|
||||
|
||||
/** Start interactive calibration mode */
|
||||
startCalibration: () => void
|
||||
/** Update calibration during drag */
|
||||
updateCalibration: (partial: Partial<CalibrationGrid>) => void
|
||||
/** Finish and save calibration */
|
||||
finishCalibration: () => void
|
||||
/** Cancel calibration without saving */
|
||||
cancelCalibration: () => void
|
||||
/** Reset/clear saved calibration */
|
||||
resetCalibration: () => void
|
||||
/** Load calibration from localStorage */
|
||||
loadCalibration: (deviceId?: string) => CalibrationGrid | null
|
||||
/** Create default calibration for given dimensions */
|
||||
createDefaultCalibration: (
|
||||
videoWidth: number,
|
||||
videoHeight: number,
|
||||
columnCount: number
|
||||
) => CalibrationGrid
|
||||
/** Set the current device ID for saving calibration */
|
||||
setDeviceId: (deviceId: string) => void
|
||||
}
|
||||
|
||||
/**
|
||||
* Hook for managing camera calibration with localStorage persistence
|
||||
*/
|
||||
export function useCameraCalibration(): UseCameraCalibrationReturn {
|
||||
const [calibration, setCalibration] = useState<CalibrationGrid | null>(null)
|
||||
const [isCalibrating, setIsCalibrating] = useState(false)
|
||||
const [currentDeviceId, setCurrentDeviceId] = useState<string | null>(null)
|
||||
|
||||
const isCalibrated = calibration !== null
|
||||
|
||||
/**
|
||||
* Create a default calibration grid centered in the video
|
||||
*/
|
||||
const createDefaultCalibration = useCallback(
|
||||
(videoWidth: number, videoHeight: number, columnCount: number): CalibrationGrid => {
|
||||
// Default to center 60% of video
|
||||
const roiWidth = videoWidth * 0.6
|
||||
const roiHeight = videoHeight * 0.7
|
||||
const roiX = (videoWidth - roiWidth) / 2
|
||||
const roiY = (videoHeight - roiHeight) / 2
|
||||
|
||||
// Create evenly-spaced column dividers
|
||||
const columnDividers: number[] = []
|
||||
for (let i = 1; i < columnCount; i++) {
|
||||
columnDividers.push(i / columnCount)
|
||||
}
|
||||
|
||||
return {
|
||||
roi: {
|
||||
x: roiX,
|
||||
y: roiY,
|
||||
width: roiWidth,
|
||||
height: roiHeight,
|
||||
},
|
||||
columnCount,
|
||||
columnDividers,
|
||||
rotation: 0,
|
||||
}
|
||||
},
|
||||
[]
|
||||
)
|
||||
|
||||
/**
|
||||
* Load calibration from localStorage for a specific device
|
||||
*/
|
||||
const loadCalibration = useCallback((deviceId?: string): CalibrationGrid | null => {
|
||||
try {
|
||||
const stored = localStorage.getItem(CALIBRATION_STORAGE_KEY)
|
||||
if (!stored) return null
|
||||
|
||||
const data = JSON.parse(stored) as StoredCalibration
|
||||
if (data.version !== 1) return null
|
||||
|
||||
// If deviceId specified, only use if it matches
|
||||
if (deviceId && data.deviceId !== deviceId) {
|
||||
return null
|
||||
}
|
||||
|
||||
return data.grid
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}, [])
|
||||
|
||||
/**
|
||||
* Save calibration to localStorage
|
||||
*/
|
||||
const saveCalibration = useCallback((grid: CalibrationGrid, deviceId: string) => {
|
||||
const stored: StoredCalibration = {
|
||||
version: 1,
|
||||
grid,
|
||||
createdAt: new Date().toISOString(),
|
||||
deviceId,
|
||||
}
|
||||
localStorage.setItem(CALIBRATION_STORAGE_KEY, JSON.stringify(stored))
|
||||
}, [])
|
||||
|
||||
/**
|
||||
* Start calibration mode
|
||||
*/
|
||||
const startCalibration = useCallback(() => {
|
||||
setIsCalibrating(true)
|
||||
}, [])
|
||||
|
||||
/**
|
||||
* Update calibration during interactive adjustment
|
||||
* Can also set a complete new calibration if none exists
|
||||
*/
|
||||
const updateCalibration = useCallback((partial: Partial<CalibrationGrid>) => {
|
||||
setCalibration((prev) => {
|
||||
// If we have a complete grid (has all required fields), use it directly
|
||||
if ('roi' in partial && 'columnCount' in partial && 'columnDividers' in partial) {
|
||||
return partial as CalibrationGrid
|
||||
}
|
||||
// Otherwise merge with existing
|
||||
if (!prev) return prev
|
||||
return { ...prev, ...partial }
|
||||
})
|
||||
}, [])
|
||||
|
||||
/**
|
||||
* Finish calibration and save
|
||||
*/
|
||||
const finishCalibration = useCallback(() => {
|
||||
if (calibration && currentDeviceId) {
|
||||
saveCalibration(calibration, currentDeviceId)
|
||||
}
|
||||
setIsCalibrating(false)
|
||||
}, [calibration, currentDeviceId, saveCalibration])
|
||||
|
||||
/**
|
||||
* Cancel calibration without saving
|
||||
*/
|
||||
const cancelCalibration = useCallback(() => {
|
||||
// Reload saved calibration
|
||||
const saved = loadCalibration(currentDeviceId ?? undefined)
|
||||
setCalibration(saved)
|
||||
setIsCalibrating(false)
|
||||
}, [currentDeviceId, loadCalibration])
|
||||
|
||||
/**
|
||||
* Reset/clear saved calibration
|
||||
*/
|
||||
const resetCalibration = useCallback(() => {
|
||||
localStorage.removeItem(CALIBRATION_STORAGE_KEY)
|
||||
setCalibration(null)
|
||||
setIsCalibrating(false)
|
||||
}, [])
|
||||
|
||||
/**
|
||||
* Initialize calibration with device ID and optionally load from storage
|
||||
*/
|
||||
const initializeCalibration = useCallback(
|
||||
(deviceId: string, videoWidth: number, videoHeight: number, columnCount: number) => {
|
||||
setCurrentDeviceId(deviceId)
|
||||
|
||||
// Try to load saved calibration
|
||||
const saved = loadCalibration(deviceId)
|
||||
if (saved) {
|
||||
setCalibration(saved)
|
||||
} else {
|
||||
// Create default calibration
|
||||
setCalibration(createDefaultCalibration(videoWidth, videoHeight, columnCount))
|
||||
}
|
||||
},
|
||||
[loadCalibration, createDefaultCalibration]
|
||||
)
|
||||
|
||||
/**
|
||||
* Set the device ID for saving calibration
|
||||
*/
|
||||
const setDeviceId = useCallback((deviceId: string) => {
|
||||
setCurrentDeviceId(deviceId)
|
||||
}, [])
|
||||
|
||||
return {
|
||||
isCalibrated,
|
||||
calibration,
|
||||
isCalibrating,
|
||||
startCalibration,
|
||||
updateCalibration,
|
||||
finishCalibration,
|
||||
cancelCalibration,
|
||||
resetCalibration,
|
||||
loadCalibration,
|
||||
createDefaultCalibration,
|
||||
setDeviceId,
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,212 @@
|
|||
'use client'
|
||||
|
||||
import { useCallback, useEffect, useRef, useState } from 'react'
|
||||
import { DESK_VIEW_PATTERNS } from '@/types/vision'
|
||||
|
||||
export interface UseDeskViewCameraReturn {
|
||||
/** Whether camera is currently loading */
|
||||
isLoading: boolean
|
||||
/** Error message if camera failed */
|
||||
error: string | null
|
||||
/** Active video stream */
|
||||
videoStream: MediaStream | null
|
||||
/** Currently selected device */
|
||||
currentDevice: MediaDeviceInfo | null
|
||||
/** All available video input devices */
|
||||
availableDevices: MediaDeviceInfo[]
|
||||
/** Whether Desk View camera was auto-detected */
|
||||
isDeskViewDetected: boolean
|
||||
|
||||
/** Request camera access, optionally specifying device ID */
|
||||
requestCamera: (deviceId?: string) => Promise<void>
|
||||
/** Stop camera stream */
|
||||
stopCamera: () => void
|
||||
/** Refresh device list */
|
||||
enumerateDevices: () => Promise<MediaDeviceInfo[]>
|
||||
}
|
||||
|
||||
/**
|
||||
* Hook for managing camera access with Desk View auto-detection
|
||||
*
|
||||
* Prioritizes finding Apple's "Desk View" camera (via Continuity Camera),
|
||||
* but falls back to manual device selection if not available.
|
||||
*/
|
||||
export function useDeskViewCamera(): UseDeskViewCameraReturn {
|
||||
const [isLoading, setIsLoading] = useState(false)
|
||||
const [error, setError] = useState<string | null>(null)
|
||||
const [videoStream, setVideoStream] = useState<MediaStream | null>(null)
|
||||
const [currentDevice, setCurrentDevice] = useState<MediaDeviceInfo | null>(null)
|
||||
const [availableDevices, setAvailableDevices] = useState<MediaDeviceInfo[]>([])
|
||||
const [isDeskViewDetected, setIsDeskViewDetected] = useState(false)
|
||||
|
||||
const streamRef = useRef<MediaStream | null>(null)
|
||||
const requestIdRef = useRef(0) // Track request ID to ignore stale completions
|
||||
|
||||
/**
|
||||
* Enumerate available video input devices
|
||||
*/
|
||||
const enumerateDevices = useCallback(async (): Promise<MediaDeviceInfo[]> => {
|
||||
try {
|
||||
const devices = await navigator.mediaDevices.enumerateDevices()
|
||||
const videoInputs = devices.filter((d) => d.kind === 'videoinput')
|
||||
setAvailableDevices(videoInputs)
|
||||
return videoInputs
|
||||
} catch (err) {
|
||||
console.error('Failed to enumerate devices:', err)
|
||||
return []
|
||||
}
|
||||
}, [])
|
||||
|
||||
/**
|
||||
* Check if a device label matches Desk View patterns
|
||||
*/
|
||||
const isDeskViewDevice = useCallback((device: MediaDeviceInfo): boolean => {
|
||||
const label = device.label.toLowerCase()
|
||||
return DESK_VIEW_PATTERNS.some((pattern) => label.includes(pattern))
|
||||
}, [])
|
||||
|
||||
/**
|
||||
* Find Desk View camera from device list
|
||||
*/
|
||||
const findDeskViewCamera = useCallback(
|
||||
(devices: MediaDeviceInfo[]): MediaDeviceInfo | null => {
|
||||
for (const device of devices) {
|
||||
if (isDeskViewDevice(device)) {
|
||||
return device
|
||||
}
|
||||
}
|
||||
return null
|
||||
},
|
||||
[isDeskViewDevice]
|
||||
)
|
||||
|
||||
/**
|
||||
* Request camera access
|
||||
*/
|
||||
const requestCamera = useCallback(
|
||||
async (deviceId?: string): Promise<void> => {
|
||||
const thisRequestId = ++requestIdRef.current
|
||||
setIsLoading(true)
|
||||
setError(null)
|
||||
|
||||
try {
|
||||
// Stop any existing stream
|
||||
if (streamRef.current) {
|
||||
for (const track of streamRef.current.getTracks()) {
|
||||
track.stop()
|
||||
}
|
||||
streamRef.current = null
|
||||
}
|
||||
|
||||
const devices = await enumerateDevices()
|
||||
|
||||
// Check if this request is still the latest
|
||||
if (thisRequestId !== requestIdRef.current) return
|
||||
|
||||
// If no deviceId specified, try to find Desk View
|
||||
let targetDeviceId = deviceId
|
||||
if (!targetDeviceId) {
|
||||
const deskViewDevice = findDeskViewCamera(devices)
|
||||
if (deskViewDevice) {
|
||||
targetDeviceId = deskViewDevice.deviceId
|
||||
setIsDeskViewDetected(true)
|
||||
} else {
|
||||
setIsDeskViewDetected(false)
|
||||
}
|
||||
}
|
||||
|
||||
const constraints: MediaStreamConstraints = {
|
||||
video: {
|
||||
width: { ideal: 1920 },
|
||||
height: { ideal: 1440 },
|
||||
...(targetDeviceId ? { deviceId: { exact: targetDeviceId } } : {}),
|
||||
},
|
||||
audio: false,
|
||||
}
|
||||
|
||||
const stream = await navigator.mediaDevices.getUserMedia(constraints)
|
||||
|
||||
// Check again if this request is still the latest
|
||||
if (thisRequestId !== requestIdRef.current) {
|
||||
for (const track of stream.getTracks()) {
|
||||
track.stop()
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
streamRef.current = stream
|
||||
setVideoStream(stream)
|
||||
|
||||
// Find which device we got
|
||||
const videoTrack = stream.getVideoTracks()[0]
|
||||
if (videoTrack) {
|
||||
const settings = videoTrack.getSettings()
|
||||
const matchingDevice = devices.find((d) => d.deviceId === settings.deviceId)
|
||||
if (matchingDevice) {
|
||||
setCurrentDevice(matchingDevice)
|
||||
setIsDeskViewDetected(isDeskViewDevice(matchingDevice))
|
||||
}
|
||||
}
|
||||
|
||||
setIsLoading(false)
|
||||
} catch (err) {
|
||||
console.error('[DeskViewCamera] Failed to access camera:', err)
|
||||
setError(err instanceof Error ? err.message : 'Failed to access camera')
|
||||
setIsLoading(false)
|
||||
}
|
||||
},
|
||||
[enumerateDevices, findDeskViewCamera, isDeskViewDevice]
|
||||
)
|
||||
|
||||
/**
|
||||
* Stop the camera stream
|
||||
*/
|
||||
const stopCamera = useCallback(() => {
|
||||
requestIdRef.current++
|
||||
|
||||
if (streamRef.current) {
|
||||
for (const track of streamRef.current.getTracks()) {
|
||||
track.stop()
|
||||
}
|
||||
streamRef.current = null
|
||||
}
|
||||
setVideoStream(null)
|
||||
setCurrentDevice(null)
|
||||
setError(null)
|
||||
}, [])
|
||||
|
||||
// Cleanup on unmount
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
if (streamRef.current) {
|
||||
for (const track of streamRef.current.getTracks()) {
|
||||
track.stop()
|
||||
}
|
||||
}
|
||||
}
|
||||
}, [])
|
||||
|
||||
// Listen for device changes (e.g., iPhone connected/disconnected)
|
||||
useEffect(() => {
|
||||
const handleDeviceChange = () => {
|
||||
enumerateDevices()
|
||||
}
|
||||
|
||||
navigator.mediaDevices.addEventListener('devicechange', handleDeviceChange)
|
||||
return () => {
|
||||
navigator.mediaDevices.removeEventListener('devicechange', handleDeviceChange)
|
||||
}
|
||||
}, [enumerateDevices])
|
||||
|
||||
return {
|
||||
isLoading,
|
||||
error,
|
||||
videoStream,
|
||||
currentDevice,
|
||||
availableDevices,
|
||||
isDeskViewDetected,
|
||||
requestCamera,
|
||||
stopCamera,
|
||||
enumerateDevices,
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,162 @@
|
|||
'use client'
|
||||
|
||||
import { useCallback, useRef, useState } from 'react'
|
||||
import type { FrameStabilityConfig } from '@/types/vision'
|
||||
import { DEFAULT_STABILITY_CONFIG } from '@/types/vision'
|
||||
|
||||
export interface UseFrameStabilityReturn {
|
||||
/** Current stable value (null if not stable) */
|
||||
stableValue: number | null
|
||||
/** How many consecutive frames have shown current value */
|
||||
consecutiveFrames: number
|
||||
/** Is currently detecting hand movement */
|
||||
isHandDetected: boolean
|
||||
/** Current unstable/in-progress value */
|
||||
currentValue: number | null
|
||||
/** Current average confidence */
|
||||
currentConfidence: number
|
||||
|
||||
/** Feed new detection result */
|
||||
pushFrame: (value: number, confidence: number) => void
|
||||
/** Feed frame data for motion detection */
|
||||
pushFrameData: (frameData: ImageData) => void
|
||||
/** Reset stability tracking */
|
||||
reset: () => void
|
||||
}
|
||||
|
||||
/**
|
||||
* Hook for debouncing frame-by-frame detection results
|
||||
*
|
||||
* Tracks consecutive frames showing the same value and only
|
||||
* outputs a "stable" value once the threshold is met.
|
||||
* Also detects large frame-to-frame changes indicating hand motion.
|
||||
*/
|
||||
export function useFrameStability(
|
||||
config: Partial<FrameStabilityConfig> = {}
|
||||
): UseFrameStabilityReturn {
|
||||
const mergedConfig = { ...DEFAULT_STABILITY_CONFIG, ...config }
|
||||
|
||||
const [stableValue, setStableValue] = useState<number | null>(null)
|
||||
const [consecutiveFrames, setConsecutiveFrames] = useState(0)
|
||||
const [isHandDetected, setIsHandDetected] = useState(false)
|
||||
const [currentValue, setCurrentValue] = useState<number | null>(null)
|
||||
const [currentConfidence, setCurrentConfidence] = useState(0)
|
||||
|
||||
// Track last detected value for comparison
|
||||
const lastValueRef = useRef<number | null>(null)
|
||||
const confidenceHistoryRef = useRef<number[]>([])
|
||||
|
||||
// Track previous frame for motion detection
|
||||
const previousFrameRef = useRef<ImageData | null>(null)
|
||||
|
||||
/**
|
||||
* Push a new detection result
|
||||
*/
|
||||
const pushFrame = useCallback(
|
||||
(value: number, confidence: number) => {
|
||||
// Skip if confidence too low
|
||||
if (confidence < mergedConfig.minConfidence) {
|
||||
return
|
||||
}
|
||||
|
||||
setCurrentValue(value)
|
||||
setCurrentConfidence(confidence)
|
||||
|
||||
// Track confidence history (last 10 frames)
|
||||
confidenceHistoryRef.current.push(confidence)
|
||||
if (confidenceHistoryRef.current.length > 10) {
|
||||
confidenceHistoryRef.current.shift()
|
||||
}
|
||||
|
||||
// Check if value matches previous
|
||||
if (value === lastValueRef.current) {
|
||||
setConsecutiveFrames((prev) => {
|
||||
const next = prev + 1
|
||||
// Check if we've reached stability threshold
|
||||
if (next >= mergedConfig.minConsecutiveFrames) {
|
||||
setStableValue(value)
|
||||
}
|
||||
return next
|
||||
})
|
||||
} else {
|
||||
// Value changed, reset counter
|
||||
lastValueRef.current = value
|
||||
setConsecutiveFrames(1)
|
||||
// Clear stable value since we're seeing a new value
|
||||
setStableValue(null)
|
||||
}
|
||||
},
|
||||
[mergedConfig.minConfidence, mergedConfig.minConsecutiveFrames]
|
||||
)
|
||||
|
||||
/**
|
||||
* Push frame data for motion detection
|
||||
*/
|
||||
const pushFrameData = useCallback(
|
||||
(frameData: ImageData) => {
|
||||
const prevFrame = previousFrameRef.current
|
||||
if (!prevFrame) {
|
||||
previousFrameRef.current = frameData
|
||||
setIsHandDetected(false)
|
||||
return
|
||||
}
|
||||
|
||||
// Compare frames for motion
|
||||
let changedPixels = 0
|
||||
const totalPixels = frameData.width * frameData.height
|
||||
const threshold = 30 // Pixel value difference threshold
|
||||
|
||||
// Sample every 4th pixel for performance
|
||||
for (let i = 0; i < frameData.data.length; i += 16) {
|
||||
const diff = Math.abs(frameData.data[i] - prevFrame.data[i])
|
||||
if (diff > threshold) {
|
||||
changedPixels++
|
||||
}
|
||||
}
|
||||
|
||||
// Adjust for sampling rate
|
||||
const sampledPixels = totalPixels / 4
|
||||
const changeRatio = changedPixels / sampledPixels
|
||||
|
||||
// Update hand detection state
|
||||
const handDetected = changeRatio > mergedConfig.handMotionThreshold
|
||||
setIsHandDetected(handDetected)
|
||||
|
||||
// If hand detected, reset stability
|
||||
if (handDetected) {
|
||||
setStableValue(null)
|
||||
setConsecutiveFrames(0)
|
||||
lastValueRef.current = null
|
||||
}
|
||||
|
||||
// Store current frame for next comparison
|
||||
previousFrameRef.current = frameData
|
||||
},
|
||||
[mergedConfig.handMotionThreshold]
|
||||
)
|
||||
|
||||
/**
|
||||
* Reset all stability tracking
|
||||
*/
|
||||
const reset = useCallback(() => {
|
||||
setStableValue(null)
|
||||
setConsecutiveFrames(0)
|
||||
setIsHandDetected(false)
|
||||
setCurrentValue(null)
|
||||
setCurrentConfidence(0)
|
||||
lastValueRef.current = null
|
||||
confidenceHistoryRef.current = []
|
||||
previousFrameRef.current = null
|
||||
}, [])
|
||||
|
||||
return {
|
||||
stableValue,
|
||||
consecutiveFrames,
|
||||
isHandDetected,
|
||||
currentValue,
|
||||
currentConfidence,
|
||||
pushFrame,
|
||||
pushFrameData,
|
||||
reset,
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,295 @@
|
|||
/**
|
||||
* Perspective Transform utilities using OpenCV.js
|
||||
*
|
||||
* Provides functions to rectify a quadrilateral region from a video frame
|
||||
* into a rectangle, correcting for camera perspective distortion.
|
||||
*/
|
||||
|
||||
import type { QuadCorners } from '@/types/vision'
|
||||
|
||||
// OpenCV.js types (minimal subset we need)
|
||||
declare global {
|
||||
interface Window {
|
||||
cv: {
|
||||
Mat: new (rows?: number, cols?: number, type?: number) => CvMat
|
||||
matFromImageData: (imageData: ImageData) => CvMat
|
||||
matFromArray: (rows: number, cols: number, type: number, data: number[]) => CvMat
|
||||
CV_8UC4: number
|
||||
CV_32FC1: number
|
||||
CV_32FC2: number
|
||||
getPerspectiveTransform: (src: CvMat, dst: CvMat) => CvMat
|
||||
warpPerspective: (
|
||||
src: CvMat,
|
||||
dst: CvMat,
|
||||
M: CvMat,
|
||||
dsize: { width: number; height: number },
|
||||
flags?: number,
|
||||
borderMode?: number,
|
||||
borderValue?: unknown
|
||||
) => void
|
||||
cvtColor: (src: CvMat, dst: CvMat, code: number) => void
|
||||
COLOR_RGBA2RGB: number
|
||||
COLOR_RGB2RGBA: number
|
||||
INTER_LINEAR: number
|
||||
BORDER_CONSTANT: number
|
||||
onRuntimeInitialized?: () => void
|
||||
}
|
||||
}
|
||||
|
||||
interface CvMat {
|
||||
rows: number
|
||||
cols: number
|
||||
data: Uint8Array
|
||||
data32F: Float32Array
|
||||
delete: () => void
|
||||
}
|
||||
}
|
||||
|
||||
let opencvLoaded = false
|
||||
let opencvLoadPromise: Promise<void> | null = null
|
||||
|
||||
/**
|
||||
* Load OpenCV.js dynamically
|
||||
*/
|
||||
export async function loadOpenCV(): Promise<void> {
|
||||
if (opencvLoaded) return
|
||||
|
||||
if (opencvLoadPromise) {
|
||||
return opencvLoadPromise
|
||||
}
|
||||
|
||||
opencvLoadPromise = new Promise<void>((resolve, reject) => {
|
||||
// Check if already loaded
|
||||
if (typeof window !== 'undefined' && window.cv && window.cv.Mat) {
|
||||
opencvLoaded = true
|
||||
resolve()
|
||||
return
|
||||
}
|
||||
|
||||
// Load the script
|
||||
const script = document.createElement('script')
|
||||
script.src = '/opencv.js'
|
||||
script.async = true
|
||||
|
||||
script.onload = () => {
|
||||
// OpenCV.js uses a callback when ready
|
||||
if (window.cv && window.cv.onRuntimeInitialized !== undefined) {
|
||||
window.cv.onRuntimeInitialized = () => {
|
||||
opencvLoaded = true
|
||||
resolve()
|
||||
}
|
||||
} else {
|
||||
// Fallback: poll for cv.Mat availability
|
||||
const checkReady = setInterval(() => {
|
||||
if (window.cv && window.cv.Mat) {
|
||||
clearInterval(checkReady)
|
||||
opencvLoaded = true
|
||||
resolve()
|
||||
}
|
||||
}, 100)
|
||||
|
||||
// Timeout after 10 seconds
|
||||
setTimeout(() => {
|
||||
clearInterval(checkReady)
|
||||
if (!opencvLoaded) {
|
||||
reject(new Error('OpenCV.js failed to initialize'))
|
||||
}
|
||||
}, 10000)
|
||||
}
|
||||
}
|
||||
|
||||
script.onerror = () => {
|
||||
reject(new Error('Failed to load OpenCV.js'))
|
||||
}
|
||||
|
||||
document.head.appendChild(script)
|
||||
})
|
||||
|
||||
return opencvLoadPromise
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if OpenCV is loaded and ready
|
||||
*/
|
||||
export function isOpenCVReady(): boolean {
|
||||
return opencvLoaded && typeof window !== 'undefined' && !!window.cv?.Mat
|
||||
}
|
||||
|
||||
export interface RectifyOptions {
|
||||
/** Output width (default: calculated from quad aspect ratio) */
|
||||
outputWidth?: number
|
||||
/** Output height (default: calculated from quad aspect ratio) */
|
||||
outputHeight?: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Rectify a quadrilateral region from video to a rectangle
|
||||
*
|
||||
* @param video - Source video element
|
||||
* @param corners - Quadrilateral corners in video coordinates
|
||||
* @param canvas - Destination canvas for rectified output
|
||||
* @param options - Output size options
|
||||
*/
|
||||
export function rectifyQuadrilateral(
|
||||
video: HTMLVideoElement,
|
||||
corners: QuadCorners,
|
||||
canvas: HTMLCanvasElement,
|
||||
options: RectifyOptions = {}
|
||||
): boolean {
|
||||
if (!isOpenCVReady()) {
|
||||
console.warn('[perspectiveTransform] OpenCV not ready')
|
||||
return false
|
||||
}
|
||||
|
||||
const cv = window.cv
|
||||
|
||||
// Calculate default output dimensions based on quad size
|
||||
const topWidth = Math.hypot(
|
||||
corners.topRight.x - corners.topLeft.x,
|
||||
corners.topRight.y - corners.topLeft.y
|
||||
)
|
||||
const bottomWidth = Math.hypot(
|
||||
corners.bottomRight.x - corners.bottomLeft.x,
|
||||
corners.bottomRight.y - corners.bottomLeft.y
|
||||
)
|
||||
const leftHeight = Math.hypot(
|
||||
corners.bottomLeft.x - corners.topLeft.x,
|
||||
corners.bottomLeft.y - corners.topLeft.y
|
||||
)
|
||||
const rightHeight = Math.hypot(
|
||||
corners.bottomRight.x - corners.topRight.x,
|
||||
corners.bottomRight.y - corners.topRight.y
|
||||
)
|
||||
|
||||
const avgWidth = (topWidth + bottomWidth) / 2
|
||||
const avgHeight = (leftHeight + rightHeight) / 2
|
||||
|
||||
const outputWidth = options.outputWidth ?? Math.round(avgWidth)
|
||||
const outputHeight = options.outputHeight ?? Math.round(avgHeight)
|
||||
|
||||
// Set canvas size
|
||||
canvas.width = outputWidth
|
||||
canvas.height = outputHeight
|
||||
|
||||
// Create temporary canvas to capture video frame
|
||||
const tempCanvas = document.createElement('canvas')
|
||||
tempCanvas.width = video.videoWidth
|
||||
tempCanvas.height = video.videoHeight
|
||||
const tempCtx = tempCanvas.getContext('2d')
|
||||
if (!tempCtx) return false
|
||||
|
||||
// Draw video frame
|
||||
tempCtx.drawImage(video, 0, 0)
|
||||
const imageData = tempCtx.getImageData(0, 0, tempCanvas.width, tempCanvas.height)
|
||||
|
||||
// Create OpenCV matrices
|
||||
let srcMat: CvMat | null = null
|
||||
let dstMat: CvMat | null = null
|
||||
let srcPoints: CvMat | null = null
|
||||
let dstPoints: CvMat | null = null
|
||||
let transformMatrix: CvMat | null = null
|
||||
|
||||
try {
|
||||
// Source image
|
||||
srcMat = cv.matFromImageData(imageData)
|
||||
|
||||
// Source points (quadrilateral corners) - order: TL, TR, BR, BL
|
||||
srcPoints = cv.matFromArray(4, 1, cv.CV_32FC2, [
|
||||
corners.topLeft.x,
|
||||
corners.topLeft.y,
|
||||
corners.topRight.x,
|
||||
corners.topRight.y,
|
||||
corners.bottomRight.x,
|
||||
corners.bottomRight.y,
|
||||
corners.bottomLeft.x,
|
||||
corners.bottomLeft.y,
|
||||
])
|
||||
|
||||
// Destination points (rectangle corners) - map to standard rectangle
|
||||
// Order: TL, TR, BR, BL (matching source points order)
|
||||
dstPoints = cv.matFromArray(4, 1, cv.CV_32FC2, [
|
||||
0,
|
||||
0, // TL → top-left of output
|
||||
outputWidth,
|
||||
0, // TR → top-right of output
|
||||
outputWidth,
|
||||
outputHeight, // BR → bottom-right of output
|
||||
0,
|
||||
outputHeight, // BL → bottom-left of output
|
||||
])
|
||||
|
||||
// Calculate perspective transform matrix
|
||||
transformMatrix = cv.getPerspectiveTransform(srcPoints, dstPoints)
|
||||
|
||||
// Create output matrix
|
||||
dstMat = new cv.Mat(outputHeight, outputWidth, cv.CV_8UC4)
|
||||
|
||||
// Apply perspective warp
|
||||
cv.warpPerspective(
|
||||
srcMat,
|
||||
dstMat,
|
||||
transformMatrix,
|
||||
{ width: outputWidth, height: outputHeight },
|
||||
cv.INTER_LINEAR,
|
||||
cv.BORDER_CONSTANT
|
||||
)
|
||||
|
||||
// Copy result to canvas
|
||||
const outputCtx = canvas.getContext('2d')
|
||||
if (!outputCtx) return false
|
||||
|
||||
const outputData = new ImageData(new Uint8ClampedArray(dstMat.data), outputWidth, outputHeight)
|
||||
outputCtx.putImageData(outputData, 0, 0)
|
||||
|
||||
return true
|
||||
} catch (err) {
|
||||
console.error('[perspectiveTransform] Error:', err)
|
||||
return false
|
||||
} finally {
|
||||
// Clean up OpenCV matrices
|
||||
srcMat?.delete()
|
||||
dstMat?.delete()
|
||||
srcPoints?.delete()
|
||||
dstPoints?.delete()
|
||||
transformMatrix?.delete()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a rectified frame processor that continuously updates a canvas
|
||||
*
|
||||
* @param video - Source video element
|
||||
* @param corners - Quadrilateral corners
|
||||
* @param canvas - Output canvas
|
||||
* @param options - Rectify options
|
||||
* @returns Stop function
|
||||
*/
|
||||
export function createRectifiedFrameLoop(
|
||||
video: HTMLVideoElement,
|
||||
corners: QuadCorners,
|
||||
canvas: HTMLCanvasElement,
|
||||
options: RectifyOptions = {}
|
||||
): () => void {
|
||||
let running = true
|
||||
let animationId: number | null = null
|
||||
|
||||
const processFrame = () => {
|
||||
if (!running) return
|
||||
|
||||
if (video.readyState >= 2) {
|
||||
// HAVE_CURRENT_DATA
|
||||
rectifyQuadrilateral(video, corners, canvas, options)
|
||||
}
|
||||
|
||||
animationId = requestAnimationFrame(processFrame)
|
||||
}
|
||||
|
||||
processFrame()
|
||||
|
||||
return () => {
|
||||
running = false
|
||||
if (animationId !== null) {
|
||||
cancelAnimationFrame(animationId)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,168 @@
|
|||
/**
|
||||
* Types for the AbacusVisionBridge feature
|
||||
*
|
||||
* Enables students to control their digital abacus using a physical soroban
|
||||
* and camera (targeting Apple's Desk View).
|
||||
*/
|
||||
|
||||
/**
|
||||
* Region of Interest (ROI) rectangle in video coordinates
|
||||
* @deprecated Use QuadCorners for perspective-aware calibration
|
||||
*/
|
||||
export interface ROI {
|
||||
x: number
|
||||
y: number
|
||||
width: number
|
||||
height: number
|
||||
}
|
||||
|
||||
/**
|
||||
* A point in 2D space (video coordinates)
|
||||
*/
|
||||
export interface Point {
|
||||
x: number
|
||||
y: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Four corners of a quadrilateral for perspective-aware ROI
|
||||
* Allows independent positioning of each corner to match camera perspective
|
||||
*/
|
||||
export interface QuadCorners {
|
||||
/** Top-left corner */
|
||||
topLeft: Point
|
||||
/** Top-right corner */
|
||||
topRight: Point
|
||||
/** Bottom-left corner */
|
||||
bottomLeft: Point
|
||||
/** Bottom-right corner */
|
||||
bottomRight: Point
|
||||
}
|
||||
|
||||
/**
|
||||
* Calibration grid defining how to slice the video feed into columns
|
||||
*/
|
||||
export interface CalibrationGrid {
|
||||
/** Bounding box in video coordinates (legacy - for backward compat) */
|
||||
roi: ROI
|
||||
/** Four corners of the quadrilateral ROI (preferred) */
|
||||
corners?: QuadCorners
|
||||
/** Number of abacus columns to detect */
|
||||
columnCount: number
|
||||
/** Column divider positions as fractions (0-1) within ROI */
|
||||
columnDividers: number[]
|
||||
/** Rotation angle in degrees (for skewed cameras) - deprecated, use corners instead */
|
||||
rotation: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Stored calibration data in localStorage
|
||||
*/
|
||||
export interface StoredCalibration {
|
||||
version: 1
|
||||
grid: CalibrationGrid
|
||||
createdAt: string
|
||||
deviceId: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Result of classifying a single column
|
||||
*/
|
||||
export interface ColumnClassificationResult {
|
||||
digit: number
|
||||
confidence: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Result of classifying all columns in a frame
|
||||
*/
|
||||
export interface FrameClassificationResult {
|
||||
digits: number[]
|
||||
confidences: number[]
|
||||
timestamp: number
|
||||
}
|
||||
|
||||
/**
|
||||
* State returned by useAbacusVision hook
|
||||
*/
|
||||
export interface AbacusVisionState {
|
||||
// Vision state
|
||||
isEnabled: boolean
|
||||
isCalibrated: boolean
|
||||
isDetecting: boolean
|
||||
currentDetectedValue: number | null
|
||||
confidence: number
|
||||
columnConfidences: number[]
|
||||
|
||||
// Camera state
|
||||
isCameraLoading: boolean
|
||||
videoStream: MediaStream | null
|
||||
cameraError: string | null
|
||||
selectedDeviceId: string | null
|
||||
availableDevices: MediaDeviceInfo[]
|
||||
isDeskViewDetected: boolean
|
||||
|
||||
// Calibration state
|
||||
calibrationGrid: CalibrationGrid | null
|
||||
isCalibrating: boolean
|
||||
|
||||
// Stability state
|
||||
isHandDetected: boolean
|
||||
consecutiveFrames: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Actions returned by useAbacusVision hook
|
||||
*/
|
||||
export interface AbacusVisionActions {
|
||||
/** Start the camera and vision processing */
|
||||
enable: () => Promise<void>
|
||||
/** Stop the camera and vision processing */
|
||||
disable: () => void
|
||||
/** Enter calibration mode */
|
||||
startCalibration: () => void
|
||||
/** Save calibration and exit calibration mode */
|
||||
finishCalibration: (grid: CalibrationGrid) => void
|
||||
/** Cancel calibration without saving */
|
||||
cancelCalibration: () => void
|
||||
/** Select a specific camera device */
|
||||
selectCamera: (deviceId: string) => void
|
||||
/** Clear saved calibration */
|
||||
resetCalibration: () => void
|
||||
}
|
||||
|
||||
/**
|
||||
* Combined state and actions from useAbacusVision
|
||||
*/
|
||||
export type UseAbacusVisionReturn = AbacusVisionState & AbacusVisionActions
|
||||
|
||||
/**
|
||||
* Configuration for frame stability detection
|
||||
*/
|
||||
export interface FrameStabilityConfig {
|
||||
/** Minimum consecutive frames showing same value to consider stable */
|
||||
minConsecutiveFrames: number
|
||||
/** Minimum confidence threshold for a valid detection */
|
||||
minConfidence: number
|
||||
/** Pixel change ratio threshold for detecting hand motion */
|
||||
handMotionThreshold: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Default stability configuration
|
||||
*/
|
||||
export const DEFAULT_STABILITY_CONFIG: FrameStabilityConfig = {
|
||||
minConsecutiveFrames: 10, // ~300ms at 30fps
|
||||
minConfidence: 0.7,
|
||||
handMotionThreshold: 0.3,
|
||||
}
|
||||
|
||||
/**
|
||||
* Patterns to detect Desk View camera by device label
|
||||
*/
|
||||
export const DESK_VIEW_PATTERNS = ['desk view', 'continuity camera', 'iphone camera']
|
||||
|
||||
/**
|
||||
* localStorage key for calibration data
|
||||
*/
|
||||
export const CALIBRATION_STORAGE_KEY = 'abacus-vision-calibration'
|
||||
Loading…
Reference in New Issue