From 5281e384ad34c9eb0fc48495db098b1fb2810536 Mon Sep 17 00:00:00 2001 From: Thomas Hallock Date: Thu, 15 Jan 2026 06:37:51 -0600 Subject: [PATCH] fix(vision): hide crop settings when phone camera not connected Don't show the crop settings section when phone camera source is selected but no phone is connected yet - the user needs to connect first before crop options are relevant. Co-Authored-By: Claude Opus 4.5 --- .../components/vision/AbacusVisionBridge.tsx | 1748 +++++++++-------- .../components/vision/CalibrationOverlay.tsx | 1071 +++++----- 2 files changed, 1469 insertions(+), 1350 deletions(-) diff --git a/apps/web/src/components/vision/AbacusVisionBridge.tsx b/apps/web/src/components/vision/AbacusVisionBridge.tsx index 6c37b452..6ad84194 100644 --- a/apps/web/src/components/vision/AbacusVisionBridge.tsx +++ b/apps/web/src/components/vision/AbacusVisionBridge.tsx @@ -1,63 +1,70 @@ -'use client' +"use client"; -import { motion, useDragControls } from 'framer-motion' -import type { ReactNode } from 'react' -import { useCallback, useEffect, useRef, useState } from 'react' -import { useAbacusVision } from '@/hooks/useAbacusVision' -import { useFrameStability } from '@/hooks/useFrameStability' -import { useRemoteCameraDesktop } from '@/hooks/useRemoteCameraDesktop' -import { analyzeColumns, analysesToDigits } from '@/lib/vision/beadDetector' -import { processImageFrame } from '@/lib/vision/frameProcessor' -import { isOpenCVReady, loadOpenCV, rectifyQuadrilateral } from '@/lib/vision/perspectiveTransform' -import type { CalibrationGrid, QuadCorners } from '@/types/vision' -import { DEFAULT_STABILITY_CONFIG } from '@/types/vision' -import { css } from '../../../styled-system/css' -import { CalibrationOverlay, type CalibrationOverlayHandle } from './CalibrationOverlay' -import { RemoteCameraQRCode } from './RemoteCameraQRCode' -import { VisionCameraFeed } from './VisionCameraFeed' -import { VisionStatusIndicator } from './VisionStatusIndicator' +import { motion, useDragControls } from "framer-motion"; +import type { ReactNode } from "react"; +import { useCallback, useEffect, useRef, useState } from "react"; +import { useAbacusVision } from "@/hooks/useAbacusVision"; +import { useFrameStability } from "@/hooks/useFrameStability"; +import { useRemoteCameraDesktop } from "@/hooks/useRemoteCameraDesktop"; +import { analyzeColumns, analysesToDigits } from "@/lib/vision/beadDetector"; +import { processImageFrame } from "@/lib/vision/frameProcessor"; +import { + isOpenCVReady, + loadOpenCV, + rectifyQuadrilateral, +} from "@/lib/vision/perspectiveTransform"; +import type { CalibrationGrid, QuadCorners } from "@/types/vision"; +import { DEFAULT_STABILITY_CONFIG } from "@/types/vision"; +import { css } from "../../../styled-system/css"; +import { + CalibrationOverlay, + type CalibrationOverlayHandle, +} from "./CalibrationOverlay"; +import { RemoteCameraQRCode } from "./RemoteCameraQRCode"; +import { VisionCameraFeed } from "./VisionCameraFeed"; +import { VisionStatusIndicator } from "./VisionStatusIndicator"; -type CameraSource = 'local' | 'phone' +type CameraSource = "local" | "phone"; /** * Configuration change payload for onConfigurationChange callback */ export interface VisionConfigurationChange { /** Camera device ID (for local camera) */ - cameraDeviceId?: string | null + cameraDeviceId?: string | null; /** Calibration grid */ - calibration?: import('@/types/vision').CalibrationGrid | null + calibration?: import("@/types/vision").CalibrationGrid | null; /** Remote camera session ID (for phone camera) */ - remoteCameraSessionId?: string | null + remoteCameraSessionId?: string | null; /** Active camera source (tracks which camera is in use) */ - activeCameraSource?: CameraSource | null + activeCameraSource?: CameraSource | null; } export interface AbacusVisionBridgeProps { /** Number of abacus columns to detect */ - columnCount: number + columnCount: number; /** Called when a stable value is detected */ - onValueDetected: (value: number) => void + onValueDetected: (value: number) => void; /** Called when vision mode is closed */ - onClose: () => void + onClose: () => void; /** Called on error */ - onError?: (error: string) => void + onError?: (error: string) => void; /** Called when configuration changes (camera, calibration, or remote session) */ - onConfigurationChange?: (config: VisionConfigurationChange) => void + onConfigurationChange?: (config: VisionConfigurationChange) => void; /** Initial camera source to show (defaults to 'local', but should be 'phone' if remote session is active) */ - initialCameraSource?: CameraSource + initialCameraSource?: CameraSource; /** Whether to show vision control buttons (enable/disable, clear settings) */ - showVisionControls?: boolean + showVisionControls?: boolean; /** Whether vision is currently enabled (for showVisionControls) */ - isVisionEnabled?: boolean + isVisionEnabled?: boolean; /** Whether vision setup is complete (for showVisionControls) */ - isVisionSetupComplete?: boolean + isVisionSetupComplete?: boolean; /** Called when user toggles vision on/off */ - onToggleVision?: () => void + onToggleVision?: () => void; /** Called when user clicks clear settings */ - onClearSettings?: () => void + onClearSettings?: () => void; /** Ref to the container element for drag constraints */ - dragConstraintRef?: React.RefObject + dragConstraintRef?: React.RefObject; } /** @@ -75,7 +82,7 @@ export function AbacusVisionBridge({ onClose, onError, onConfigurationChange, - initialCameraSource = 'local', + initialCameraSource = "local", showVisionControls = false, isVisionEnabled = false, isVisionSetupComplete = false, @@ -84,48 +91,55 @@ export function AbacusVisionBridge({ dragConstraintRef, }: AbacusVisionBridgeProps): ReactNode { const [videoDimensions, setVideoDimensions] = useState<{ - width: number - height: number - containerWidth: number - containerHeight: number - } | null>(null) + width: number; + height: number; + containerWidth: number; + containerHeight: number; + } | null>(null); - const containerRef = useRef(null) - const cameraFeedContainerRef = useRef(null) - const remoteFeedContainerRef = useRef(null) - const remoteImageRef = useRef(null) - const videoRef = useRef(null) - const previewCanvasRef = useRef(null) - const calibrationOverlayRef = useRef(null) + const containerRef = useRef(null); + const cameraFeedContainerRef = useRef(null); + const remoteFeedContainerRef = useRef(null); + const remoteImageRef = useRef(null); + const videoRef = useRef(null); + const previewCanvasRef = useRef(null); + const calibrationOverlayRef = useRef(null); // Track remote image container dimensions for calibration const [remoteContainerDimensions, setRemoteContainerDimensions] = useState<{ - width: number - height: number - } | null>(null) + width: number; + height: number; + } | null>(null); - const [calibrationCorners, setCalibrationCorners] = useState(null) - const [opencvReady, setOpencvReady] = useState(false) + const [calibrationCorners, setCalibrationCorners] = + useState(null); + const [opencvReady, setOpencvReady] = useState(false); // Camera source selection - const [cameraSource, setCameraSource] = useState(initialCameraSource) - const [remoteCameraSessionId, setRemoteCameraSessionId] = useState(null) + const [cameraSource, setCameraSource] = + useState(initialCameraSource); + const [remoteCameraSessionId, setRemoteCameraSessionId] = useState< + string | null + >(null); // Track whether we've checked localStorage for persisted session - prevents race condition // where RemoteCameraQRCode creates a new session before we read the persisted one - const [hasCheckedPersistence, setHasCheckedPersistence] = useState(false) + const [hasCheckedPersistence, setHasCheckedPersistence] = useState(false); // Remote camera state - const [remoteCalibrationMode, setRemoteCalibrationMode] = useState<'auto' | 'manual'>('auto') - const [remoteIsCalibrating, setRemoteIsCalibrating] = useState(false) - const [remoteCalibration, setRemoteCalibration] = useState(null) + const [remoteCalibrationMode, setRemoteCalibrationMode] = useState< + "auto" | "manual" + >("auto"); + const [remoteIsCalibrating, setRemoteIsCalibrating] = useState(false); + const [remoteCalibration, setRemoteCalibration] = + useState(null); // Crop settings expansion state - const [isCropSettingsExpanded, setIsCropSettingsExpanded] = useState(false) + const [isCropSettingsExpanded, setIsCropSettingsExpanded] = useState(false); const vision = useAbacusVision({ columnCount, onValueDetected, - }) + }); // Remote camera hook - destructure to get stable function references const { @@ -147,276 +161,290 @@ export function AbacusVisionBridge({ setRemoteTorch, getPersistedSessionId: remoteGetPersistedSessionId, clearSession: remoteClearSession, - } = useRemoteCameraDesktop() + } = useRemoteCameraDesktop(); // Stability tracking for remote frames - const remoteStability = useFrameStability() + const remoteStability = useFrameStability(); // Track last stable value for remote camera to avoid duplicate callbacks - const lastRemoteStableValueRef = useRef(null) + const lastRemoteStableValueRef = useRef(null); // Throttle remote frame processing - const lastRemoteInferenceTimeRef = useRef(0) - const REMOTE_INFERENCE_INTERVAL_MS = 100 // 10fps + const lastRemoteInferenceTimeRef = useRef(0); + const REMOTE_INFERENCE_INTERVAL_MS = 100; // 10fps // Track last reported configuration to avoid redundant callbacks - const lastReportedCameraRef = useRef(null) - const lastReportedCalibrationRef = useRef(null) - const lastReportedRemoteSessionRef = useRef(null) + const lastReportedCameraRef = useRef(null); + const lastReportedCalibrationRef = useRef(null); + const lastReportedRemoteSessionRef = useRef(null); // Initialize remote camera session from localStorage on mount // IMPORTANT: Must complete before rendering RemoteCameraQRCode to prevent race condition useEffect(() => { - const persistedSessionId = remoteGetPersistedSessionId() + const persistedSessionId = remoteGetPersistedSessionId(); if (persistedSessionId) { - console.log('[AbacusVisionBridge] Found persisted remote session:', persistedSessionId) - setRemoteCameraSessionId(persistedSessionId) + console.log( + "[AbacusVisionBridge] Found persisted remote session:", + persistedSessionId, + ); + setRemoteCameraSessionId(persistedSessionId); // Also notify parent about the persisted session // This ensures the parent context stays in sync with localStorage - onConfigurationChange?.({ remoteCameraSessionId: persistedSessionId }) + onConfigurationChange?.({ remoteCameraSessionId: persistedSessionId }); } // Mark that we've checked - now safe to render RemoteCameraQRCode - setHasCheckedPersistence(true) + setHasCheckedPersistence(true); // eslint-disable-next-line react-hooks/exhaustive-deps - }, [remoteGetPersistedSessionId]) // onConfigurationChange is intentionally omitted - only run on mount + }, [remoteGetPersistedSessionId]); // onConfigurationChange is intentionally omitted - only run on mount // Handle switching to phone camera const handleCameraSourceChange = useCallback( (source: CameraSource) => { - setCameraSource(source) - if (source === 'phone') { + setCameraSource(source); + if (source === "phone") { // Stop local camera - vision.disable() + vision.disable(); // Set active camera source to phone and clear local camera config // (but keep local config in storage for when we switch back) - onConfigurationChange?.({ activeCameraSource: 'phone' }) + onConfigurationChange?.({ activeCameraSource: "phone" }); // Check for persisted session and reuse it - const persistedSessionId = remoteGetPersistedSessionId() + const persistedSessionId = remoteGetPersistedSessionId(); if (persistedSessionId) { - console.log('[AbacusVisionBridge] Reusing persisted remote session:', persistedSessionId) - setRemoteCameraSessionId(persistedSessionId) + console.log( + "[AbacusVisionBridge] Reusing persisted remote session:", + persistedSessionId, + ); + setRemoteCameraSessionId(persistedSessionId); // Notify parent about the reused session onConfigurationChange?.({ remoteCameraSessionId: persistedSessionId, - }) + }); } // If no persisted session, RemoteCameraQRCode will create one } else { // Switching to local camera // Set active camera source to local // The remote session still persists in localStorage (via useRemoteCameraDesktop) for when we switch back - setRemoteCameraSessionId(null) - onConfigurationChange?.({ activeCameraSource: 'local' }) - vision.enable() + setRemoteCameraSessionId(null); + onConfigurationChange?.({ activeCameraSource: "local" }); + vision.enable(); } }, - [vision, onConfigurationChange, remoteGetPersistedSessionId] - ) + [vision, onConfigurationChange, remoteGetPersistedSessionId], + ); // Handle starting a fresh session (clear persisted and create new) const handleStartFreshSession = useCallback(() => { - remoteClearSession() - setRemoteCameraSessionId(null) - }, [remoteClearSession]) + remoteClearSession(); + setRemoteCameraSessionId(null); + }, [remoteClearSession]); // Handle session created by QR code component const handleRemoteSessionCreated = useCallback((sessionId: string) => { - setRemoteCameraSessionId(sessionId) - }, []) + setRemoteCameraSessionId(sessionId); + }, []); // Subscribe to remote camera session when sessionId changes useEffect(() => { - if (remoteCameraSessionId && cameraSource === 'phone') { - remoteSubscribe(remoteCameraSessionId) - return () => remoteUnsubscribe() + if (remoteCameraSessionId && cameraSource === "phone") { + remoteSubscribe(remoteCameraSessionId); + return () => remoteUnsubscribe(); } - }, [remoteCameraSessionId, cameraSource, remoteSubscribe, remoteUnsubscribe]) + }, [remoteCameraSessionId, cameraSource, remoteSubscribe, remoteUnsubscribe]); // Handle remote camera mode change const handleRemoteModeChange = useCallback( - (mode: 'auto' | 'manual') => { - setRemoteCalibrationMode(mode) - if (mode === 'auto') { + (mode: "auto" | "manual") => { + setRemoteCalibrationMode(mode); + if (mode === "auto") { // Tell phone to use its auto-calibration (cropped frames) - remoteSetPhoneFrameMode('cropped') - setRemoteIsCalibrating(false) + remoteSetPhoneFrameMode("cropped"); + setRemoteIsCalibrating(false); // Clear desktop calibration on phone so it goes back to auto-detection - remoteClearCalibration() - setRemoteCalibration(null) + remoteClearCalibration(); + setRemoteCalibration(null); } else { // Tell phone to send raw frames for desktop calibration - remoteSetPhoneFrameMode('raw') + remoteSetPhoneFrameMode("raw"); } }, - [remoteSetPhoneFrameMode, remoteClearCalibration] - ) + [remoteSetPhoneFrameMode, remoteClearCalibration], + ); // Start remote camera calibration const handleRemoteStartCalibration = useCallback(() => { - remoteSetPhoneFrameMode('raw') - setRemoteIsCalibrating(true) - }, [remoteSetPhoneFrameMode]) + remoteSetPhoneFrameMode("raw"); + setRemoteIsCalibrating(true); + }, [remoteSetPhoneFrameMode]); // Complete remote camera calibration const handleRemoteCalibrationComplete = useCallback( (grid: CalibrationGrid) => { - setRemoteCalibration(grid) - setRemoteIsCalibrating(false) + setRemoteCalibration(grid); + setRemoteIsCalibrating(false); // Send calibration to phone if (grid.corners) { - remoteSendCalibration(grid.corners) + remoteSendCalibration(grid.corners); } }, - [remoteSendCalibration] - ) + [remoteSendCalibration], + ); // Cancel remote camera calibration const handleRemoteCancelCalibration = useCallback(() => { - setRemoteIsCalibrating(false) + setRemoteIsCalibrating(false); // Go back to previous mode - if (remoteCalibrationMode === 'auto') { - remoteSetPhoneFrameMode('cropped') + if (remoteCalibrationMode === "auto") { + remoteSetPhoneFrameMode("cropped"); } - }, [remoteCalibrationMode, remoteSetPhoneFrameMode]) + }, [remoteCalibrationMode, remoteSetPhoneFrameMode]); // Reset remote calibration const handleRemoteResetCalibration = useCallback(() => { - setRemoteCalibration(null) - remoteSetPhoneFrameMode('cropped') - }, [remoteSetPhoneFrameMode]) + setRemoteCalibration(null); + remoteSetPhoneFrameMode("cropped"); + }, [remoteSetPhoneFrameMode]); // Start camera on mount (only for local source) useEffect(() => { - if (cameraSource === 'local') { - vision.enable() + if (cameraSource === "local") { + vision.enable(); } - return () => vision.disable() + return () => vision.disable(); // eslint-disable-next-line react-hooks/exhaustive-deps - }, []) // Only run on mount/unmount - vision functions are stable + }, []); // Only run on mount/unmount - vision functions are stable // Report errors useEffect(() => { if (vision.cameraError && onError) { - onError(vision.cameraError) + onError(vision.cameraError); } - }, [vision.cameraError, onError]) + }, [vision.cameraError, onError]); // Notify about local camera device changes and ensure activeCameraSource is set useEffect(() => { if ( - cameraSource === 'local' && + cameraSource === "local" && vision.selectedDeviceId && vision.selectedDeviceId !== lastReportedCameraRef.current ) { - lastReportedCameraRef.current = vision.selectedDeviceId + lastReportedCameraRef.current = vision.selectedDeviceId; // Set both the camera device ID and the active camera source onConfigurationChange?.({ cameraDeviceId: vision.selectedDeviceId, - activeCameraSource: 'local', - }) + activeCameraSource: "local", + }); } - }, [cameraSource, vision.selectedDeviceId, onConfigurationChange]) + }, [cameraSource, vision.selectedDeviceId, onConfigurationChange]); // Notify about local calibration changes useEffect(() => { if ( - cameraSource === 'local' && + cameraSource === "local" && vision.calibrationGrid && vision.calibrationGrid !== lastReportedCalibrationRef.current ) { - lastReportedCalibrationRef.current = vision.calibrationGrid - onConfigurationChange?.({ calibration: vision.calibrationGrid }) + lastReportedCalibrationRef.current = vision.calibrationGrid; + onConfigurationChange?.({ calibration: vision.calibrationGrid }); } - }, [cameraSource, vision.calibrationGrid, onConfigurationChange]) + }, [cameraSource, vision.calibrationGrid, onConfigurationChange]); // Notify about remote camera session changes // Reset the lastReportedRemoteSessionRef when switching away from phone camera // so that the next time we switch to phone, we'll notify the parent again useEffect(() => { - if (cameraSource !== 'phone') { + if (cameraSource !== "phone") { // When switching away from phone camera, reset the ref // This ensures we'll notify the parent again when switching back - lastReportedRemoteSessionRef.current = null + lastReportedRemoteSessionRef.current = null; } else if ( remoteCameraSessionId && remoteCameraSessionId !== lastReportedRemoteSessionRef.current ) { - lastReportedRemoteSessionRef.current = remoteCameraSessionId - onConfigurationChange?.({ remoteCameraSessionId }) + lastReportedRemoteSessionRef.current = remoteCameraSessionId; + onConfigurationChange?.({ remoteCameraSessionId }); } - }, [cameraSource, remoteCameraSessionId, onConfigurationChange]) + }, [cameraSource, remoteCameraSessionId, onConfigurationChange]); // Notify about remote calibration changes (manual mode) useEffect(() => { if ( - cameraSource === 'phone' && + cameraSource === "phone" && remoteCalibration && remoteCalibration !== lastReportedCalibrationRef.current ) { - lastReportedCalibrationRef.current = remoteCalibration - onConfigurationChange?.({ calibration: remoteCalibration }) + lastReportedCalibrationRef.current = remoteCalibration; + onConfigurationChange?.({ calibration: remoteCalibration }); } - }, [cameraSource, remoteCalibration, onConfigurationChange]) + }, [cameraSource, remoteCalibration, onConfigurationChange]); // Process remote camera frames through CV pipeline useEffect(() => { // Only process when using phone camera and connected - if (cameraSource !== 'phone' || !remoteIsPhoneConnected || !remoteLatestFrame) { - return + if ( + cameraSource !== "phone" || + !remoteIsPhoneConnected || + !remoteLatestFrame + ) { + return; } // Don't process during calibration if (remoteIsCalibrating) { - return + return; } // In manual mode, need calibration to process - if (remoteCalibrationMode === 'manual' && !remoteCalibration) { - return + if (remoteCalibrationMode === "manual" && !remoteCalibration) { + return; } // Throttle processing - const now = performance.now() - if (now - lastRemoteInferenceTimeRef.current < REMOTE_INFERENCE_INTERVAL_MS) { - return + const now = performance.now(); + if ( + now - lastRemoteInferenceTimeRef.current < + REMOTE_INFERENCE_INTERVAL_MS + ) { + return; } - lastRemoteInferenceTimeRef.current = now + lastRemoteInferenceTimeRef.current = now; // Get image element - const image = remoteImageRef.current + const image = remoteImageRef.current; if (!image || !image.complete || image.naturalWidth === 0) { - return + return; } // Determine calibration to use // In auto mode (cropped frames), no calibration needed - phone already cropped // In manual mode, use the desktop calibration - const calibration = remoteCalibrationMode === 'auto' ? null : remoteCalibration + const calibration = + remoteCalibrationMode === "auto" ? null : remoteCalibration; // Process frame through CV pipeline - const columnImages = processImageFrame(image, calibration, columnCount) - if (columnImages.length === 0) return + const columnImages = processImageFrame(image, calibration, columnCount); + if (columnImages.length === 0) return; // Run CV-based bead detection - const analyses = analyzeColumns(columnImages) - const { digits, minConfidence } = analysesToDigits(analyses) + const analyses = analyzeColumns(columnImages); + const { digits, minConfidence } = analysesToDigits(analyses); // Convert digits to number - const detectedValue = digits.reduce((acc, d) => acc * 10 + d, 0) + const detectedValue = digits.reduce((acc, d) => acc * 10 + d, 0); // Log for debugging console.log( - '[Remote CV] Bead analysis:', + "[Remote CV] Bead analysis:", analyses.map((a) => ({ digit: a.digit, conf: a.confidence.toFixed(2), - heaven: a.heavenActive ? '5' : '0', + heaven: a.heavenActive ? "5" : "0", earth: a.earthActiveCount, - })) - ) + })), + ); // Push to stability buffer - remoteStability.pushFrame(detectedValue, minConfidence) + remoteStability.pushFrame(detectedValue, minConfidence); }, [ cameraSource, remoteIsPhoneConnected, @@ -426,54 +454,54 @@ export function AbacusVisionBridge({ remoteCalibration, columnCount, remoteStability, - ]) + ]); // Notify when remote stable value changes useEffect(() => { if ( - cameraSource === 'phone' && + cameraSource === "phone" && remoteStability.stableValue !== null && remoteStability.stableValue !== lastRemoteStableValueRef.current ) { - lastRemoteStableValueRef.current = remoteStability.stableValue - onValueDetected(remoteStability.stableValue) + lastRemoteStableValueRef.current = remoteStability.stableValue; + onValueDetected(remoteStability.stableValue); } - }, [cameraSource, remoteStability.stableValue, onValueDetected]) + }, [cameraSource, remoteStability.stableValue, onValueDetected]); // Load OpenCV when calibrating (local or remote) useEffect(() => { - const isCalibrating = vision.isCalibrating || remoteIsCalibrating + const isCalibrating = vision.isCalibrating || remoteIsCalibrating; if (isCalibrating && !opencvReady) { loadOpenCV() .then(() => setOpencvReady(true)) - .catch((err) => console.error('Failed to load OpenCV:', err)) + .catch((err) => console.error("Failed to load OpenCV:", err)); } - }, [vision.isCalibrating, remoteIsCalibrating, opencvReady]) + }, [vision.isCalibrating, remoteIsCalibrating, opencvReady]); // Track remote image container dimensions for calibration overlay useEffect(() => { - const container = remoteFeedContainerRef.current - if (!container || !remoteIsCalibrating) return + const container = remoteFeedContainerRef.current; + if (!container || !remoteIsCalibrating) return; const updateDimensions = () => { - const rect = container.getBoundingClientRect() + const rect = container.getBoundingClientRect(); if (rect.width > 0 && rect.height > 0) { setRemoteContainerDimensions({ width: rect.width, height: rect.height, - }) + }); } - } + }; // Update immediately - updateDimensions() + updateDimensions(); // Also update on resize - const resizeObserver = new ResizeObserver(updateDimensions) - resizeObserver.observe(container) + const resizeObserver = new ResizeObserver(updateDimensions); + resizeObserver.observe(container); - return () => resizeObserver.disconnect() - }, [remoteIsCalibrating, remoteLatestFrame]) + return () => resizeObserver.disconnect(); + }, [remoteIsCalibrating, remoteLatestFrame]); // Render preview when calibrating useEffect(() => { @@ -483,65 +511,65 @@ export function AbacusVisionBridge({ !videoRef.current || !previewCanvasRef.current ) { - return + return; } - let running = true - const video = videoRef.current - const canvas = previewCanvasRef.current + let running = true; + const video = videoRef.current; + const canvas = previewCanvasRef.current; const drawPreview = () => { if (!running || video.readyState < 2) { - if (running) requestAnimationFrame(drawPreview) - return + if (running) requestAnimationFrame(drawPreview); + return; } if (opencvReady && isOpenCVReady()) { rectifyQuadrilateral(video, calibrationCorners, canvas, { outputWidth: 200, outputHeight: 133, - }) + }); } - requestAnimationFrame(drawPreview) - } + requestAnimationFrame(drawPreview); + }; - drawPreview() + drawPreview(); return () => { - running = false - } - }, [vision.isCalibrating, calibrationCorners, opencvReady]) + running = false; + }; + }, [vision.isCalibrating, calibrationCorners, opencvReady]); // Handle video ready - get dimensions const handleVideoReady = useCallback( (width: number, height: number) => { - const feedContainer = cameraFeedContainerRef.current - if (!feedContainer) return + const feedContainer = cameraFeedContainerRef.current; + if (!feedContainer) return; - const rect = feedContainer.getBoundingClientRect() + const rect = feedContainer.getBoundingClientRect(); setVideoDimensions({ width, height, containerWidth: rect.width, containerHeight: rect.height, - }) + }); // Initialize calibration with default grid if not already calibrated if (!vision.isCalibrated) { // Will start calibration on user action } }, - [vision.isCalibrated] - ) + [vision.isCalibrated], + ); // Update container dimensions when container resizes (e.g., when entering/exiting calibration mode) useEffect(() => { - const feedContainer = cameraFeedContainerRef.current - if (!feedContainer || !videoDimensions) return + const feedContainer = cameraFeedContainerRef.current; + if (!feedContainer || !videoDimensions) return; const resizeObserver = new ResizeObserver((entries) => { for (const entry of entries) { - const { width, height } = entry.contentRect + const { width, height } = entry.contentRect; // Only update if dimensions actually changed if ( width !== videoDimensions.containerWidth || @@ -554,37 +582,37 @@ export function AbacusVisionBridge({ containerWidth: width, containerHeight: height, } - : null - ) + : null, + ); } } - }) + }); - resizeObserver.observe(feedContainer) - return () => resizeObserver.disconnect() - }, [videoDimensions?.containerWidth, videoDimensions?.containerHeight]) + resizeObserver.observe(feedContainer); + return () => resizeObserver.disconnect(); + }, [videoDimensions?.containerWidth, videoDimensions?.containerHeight]); // Handle calibration complete const handleCalibrationComplete = useCallback( (grid: Parameters[0]) => { - vision.finishCalibration(grid) + vision.finishCalibration(grid); }, - [vision] - ) + [vision], + ); // Camera selector const handleCameraSelect = useCallback( (e: React.ChangeEvent) => { - vision.selectCamera(e.target.value) + vision.selectCamera(e.target.value); }, - [vision] - ) + [vision], + ); // Determine if any calibration is active - const isCalibrating = vision.isCalibrating || remoteIsCalibrating + const isCalibrating = vision.isCalibrating || remoteIsCalibrating; // Drag controls allow dragging from header even during calibration - const dragControls = useDragControls() + const dragControls = useDragControls(); return ( {/* Header - drag handle for repositioning modal */} @@ -617,28 +645,30 @@ export function AbacusVisionBridge({ data-element="header" onPointerDown={(e) => dragControls.start(e)} className={css({ - display: 'flex', - alignItems: 'center', - justifyContent: 'space-between', - cursor: 'grab', - touchAction: 'none', - _active: { cursor: 'grabbing' }, + display: "flex", + alignItems: "center", + justifyContent: "space-between", + cursor: "grab", + touchAction: "none", + _active: { cursor: "grabbing" }, })} > -
- {isCalibrating ? '✂️' : '📷'} - - {isCalibrating ? 'Adjust Crop Region' : 'Abacus Vision'} +
+ + {isCalibrating ? "✂️" : "📷"} + + + {isCalibrating ? "Adjust Crop Region" : "Abacus Vision"} {!isCalibrating && vision.isDeskViewDetected && ( Desk View @@ -651,18 +681,18 @@ export function AbacusVisionBridge({ type="button" onClick={onClose} className={css({ - width: '32px', - height: '32px', - display: 'flex', - alignItems: 'center', - justifyContent: 'center', - bg: 'gray.700', - color: 'white', - border: 'none', - borderRadius: 'full', - cursor: 'pointer', - fontSize: 'lg', - _hover: { bg: 'gray.600' }, + width: "32px", + height: "32px", + display: "flex", + alignItems: "center", + justifyContent: "center", + bg: "gray.700", + color: "white", + border: "none", + borderRadius: "full", + cursor: "pointer", + fontSize: "lg", + _hover: { bg: "gray.600" }, })} aria-label="Close" > @@ -675,29 +705,31 @@ export function AbacusVisionBridge({
- {cameraSource === 'local' ? ( + {cameraSource === "local" ? ( <> { - videoRef.current = el + videoRef.current = el; }} onVideoReady={handleVideoReady} > @@ -723,7 +755,7 @@ export function AbacusVisionBridge({ {vision.isEnabled && !vision.isCalibrating && (
{/* Camera selector - compact */} {vision.availableDevices.length > 0 && ( )} {/* Toolbar buttons */} -
+
{/* Flip camera */} {vision.availableDevices.length > 1 && ( @@ -820,26 +853,32 @@ export function AbacusVisionBridge({ type="button" onClick={() => vision.toggleTorch()} data-action="toggle-torch" - data-status={vision.isTorchOn ? 'on' : 'off'} + data-status={vision.isTorchOn ? "on" : "off"} className={css({ - display: 'flex', - alignItems: 'center', - justifyContent: 'center', - width: '36px', - height: '36px', - bg: vision.isTorchOn ? 'yellow.600' : 'rgba(255, 255, 255, 0.15)', - color: 'white', - border: 'none', - borderRadius: 'md', - cursor: 'pointer', - fontSize: 'md', + display: "flex", + alignItems: "center", + justifyContent: "center", + width: "36px", + height: "36px", + bg: vision.isTorchOn + ? "yellow.600" + : "rgba(255, 255, 255, 0.15)", + color: "white", + border: "none", + borderRadius: "md", + cursor: "pointer", + fontSize: "md", _hover: { - bg: vision.isTorchOn ? 'yellow.500' : 'rgba(255, 255, 255, 0.25)', + bg: vision.isTorchOn + ? "yellow.500" + : "rgba(255, 255, 255, 0.25)", }, })} - title={vision.isTorchOn ? 'Turn off flash' : 'Turn on flash'} + title={ + vision.isTorchOn ? "Turn off flash" : "Turn on flash" + } > - {vision.isTorchOn ? '🔦' : '💡'} + {vision.isTorchOn ? "🔦" : "💡"} )}
@@ -851,24 +890,24 @@ export function AbacusVisionBridge({
{!hasCheckedPersistence ? ( /* Still checking localStorage for persisted session - show loading state */
Loading... @@ -877,12 +916,12 @@ export function AbacusVisionBridge({ /* Show QR code to connect phone */

Scan with your phone to use it as a camera @@ -906,12 +945,12 @@ export function AbacusVisionBridge({ /* Waiting for phone - compact layout to fit container */

Scan with your phone to use it as a camera

- {remoteIsReconnecting ? 'Reconnecting...' : 'Waiting for phone'} - · + {remoteIsReconnecting + ? "Reconnecting..." + : "Waiting for phone"} + · )}
@@ -1125,23 +1177,23 @@ export function AbacusVisionBridge({
{/* Calibration controls area (local camera) */} - {cameraSource === 'local' && vision.isCalibrating && ( + {cameraSource === "local" && vision.isCalibrating && (
{/* Instructions and preview row */}
@@ -1150,10 +1202,10 @@ export function AbacusVisionBridge({

Drag inside to move. Drag corners to resize.

-

Yellow lines = column dividers

+

+ Yellow lines = column dividers +

@@ -1178,31 +1232,31 @@ export function AbacusVisionBridge({
{/* Rotate buttons */} -
+
{/* Cancel / Done buttons */} -
+
{/* Cancel / Done buttons */} -
+
)} - {/* Crop settings - collapsible (hidden during calibration) */} - {!isCalibrating && ( -
- {/* Collapsible header - shows summary */} - - - {/* Expanded content */} - {isCropSettingsExpanded && ( -
setIsCropSettingsExpanded(!isCropSettingsExpanded)} + data-element="crop-settings-header" className={css({ - display: 'flex', - flexDirection: 'column', - gap: 2, - px: 2, - pb: 2, - borderTop: '1px solid', - borderColor: 'gray.700', + width: "100%", + display: "flex", + alignItems: "center", + justifyContent: "space-between", + p: 2, + bg: "transparent", + border: "none", + cursor: "pointer", + color: "white", + _hover: { bg: "gray.750" }, })} > - {/* Manual crop indicator (if set) */} - {((cameraSource === 'local' && vision.isCalibrated) || - (cameraSource === 'phone' && remoteCalibration)) && ( -
+ -
+ + Crop + + + · + + {/* Status summary */} + {(cameraSource === "local" && vision.isCalibrated) || + (cameraSource === "phone" && remoteCalibration) ? ( + + Manual + + ) : ( + - - - Using manual crop region - -
- -
- )} + {cameraSource === "local" + ? vision.markerDetection.allMarkersFound + ? "Auto" + : `${vision.markerDetection.markersFound}/4 markers` + : remoteFrameMode === "cropped" + ? "Auto" + : "Detecting..."} + + )} +
+ - {/* Auto crop status (when no manual crop) */} - {!( - (cameraSource === 'local' && vision.isCalibrated) || - (cameraSource === 'phone' && remoteCalibration) - ) && ( - <> + {/* Expanded content */} + {isCropSettingsExpanded && ( +
+ {/* Manual crop indicator (if set) */} + {((cameraSource === "local" && vision.isCalibrated) || + (cameraSource === "phone" && remoteCalibration)) && (
- - {cameraSource === 'local' - ? vision.markerDetection.allMarkersFound - ? 'Auto-crop using markers' - : `Looking for markers (${vision.markerDetection.markersFound}/4 found)` - : remoteFrameMode === 'cropped' - ? 'Phone auto-cropping' - : 'Looking for markers...'} + + Using manual crop region
-
- - {/* Get markers link */} - - - {/* Manual crop button */} - {!(cameraSource === 'local' ? vision.isCalibrating : remoteIsCalibrating) && ( - )} - - )} +
+ )} - {/* Troubleshooting - clear all settings */} - {showVisionControls && isVisionSetupComplete && ( -
- + )} + + )} + + {/* Troubleshooting - clear all settings */} + {showVisionControls && isVisionSetupComplete && ( +
- Clear all settings - -
- )} -
- )} -
- )} + +
+ )} +
+ )} +
+ )} {/* Error display */} - {cameraSource === 'local' && vision.cameraError && ( + {cameraSource === "local" && vision.cameraError && (
{vision.cameraError} @@ -1780,12 +1857,12 @@ export function AbacusVisionBridge({
{/* Always show button, but disabled with explanation if setup not complete */} @@ -1796,24 +1873,28 @@ export function AbacusVisionBridge({ className={css({ px: 4, py: 2.5, - bg: !isVisionSetupComplete ? 'gray.600' : isVisionEnabled ? 'red.600' : 'green.600', - color: 'white', - borderRadius: 'lg', - fontWeight: 'semibold', - fontSize: 'sm', - border: 'none', - cursor: isVisionSetupComplete ? 'pointer' : 'not-allowed', - transition: 'all 0.2s', + bg: !isVisionSetupComplete + ? "gray.600" + : isVisionEnabled + ? "red.600" + : "green.600", + color: "white", + borderRadius: "lg", + fontWeight: "semibold", + fontSize: "sm", + border: "none", + cursor: isVisionSetupComplete ? "pointer" : "not-allowed", + transition: "all 0.2s", opacity: isVisionSetupComplete ? 1 : 0.7, _hover: isVisionSetupComplete ? { - bg: isVisionEnabled ? 'red.700' : 'green.700', - transform: 'scale(1.02)', + bg: isVisionEnabled ? "red.700" : "green.700", + transform: "scale(1.02)", } : {}, })} > - {isVisionEnabled ? 'Disable Vision' : 'Enable Vision'} + {isVisionEnabled ? "Disable Vision" : "Enable Vision"} {/* Explanation when setup is not complete */} @@ -1821,16 +1902,16 @@ export function AbacusVisionBridge({

- {cameraSource === 'local' - ? 'Waiting for camera access...' + {cameraSource === "local" + ? "Waiting for camera access..." : !remoteIsPhoneConnected - ? 'Connect your phone to enable vision' - : 'Setting up camera...'} + ? "Connect your phone to enable vision" + : "Setting up camera..."}

)} @@ -1840,21 +1921,22 @@ export function AbacusVisionBridge({ data-element="training-data-disclaimer" className={css({ p: 2, - bg: 'blue.900/50', - borderRadius: 'md', - fontSize: '10px', - color: 'blue.200', + bg: "blue.900/50", + borderRadius: "md", + fontSize: "10px", + color: "blue.200", lineHeight: 1.3, })} > - Training Data: Column images may be saved on correct answers to - improve bead detection. No personal data is collected. + Training Data: Column images may be saved on + correct answers to improve bead detection. No personal data is + collected.
)}
)} - ) + ); } -export default AbacusVisionBridge +export default AbacusVisionBridge; diff --git a/apps/web/src/components/vision/CalibrationOverlay.tsx b/apps/web/src/components/vision/CalibrationOverlay.tsx index 2e8f7846..9713b135 100644 --- a/apps/web/src/components/vision/CalibrationOverlay.tsx +++ b/apps/web/src/components/vision/CalibrationOverlay.tsx @@ -1,56 +1,63 @@ -'use client' +"use client"; -import type { ReactNode } from 'react' -import { forwardRef, useCallback, useEffect, useImperativeHandle, useRef, useState } from 'react' -import { css } from '../../../styled-system/css' -import type { CalibrationGrid, Point, QuadCorners, ROI } from '@/types/vision' +import type { ReactNode } from "react"; +import { + forwardRef, + useCallback, + useEffect, + useImperativeHandle, + useRef, + useState, +} from "react"; +import { css } from "../../../styled-system/css"; +import type { CalibrationGrid, Point, QuadCorners, ROI } from "@/types/vision"; export interface CalibrationOverlayProps { /** Number of columns to configure */ - columnCount: number + columnCount: number; /** Video dimensions */ - videoWidth: number - videoHeight: number + videoWidth: number; + videoHeight: number; /** Container dimensions (displayed size) */ - containerWidth: number - containerHeight: number + containerWidth: number; + containerHeight: number; /** Current calibration (if any) */ - initialCalibration?: CalibrationGrid | null + initialCalibration?: CalibrationGrid | null; /** Called when calibration is completed */ - onComplete: (grid: CalibrationGrid) => void + onComplete: (grid: CalibrationGrid) => void; /** Called when calibration is cancelled */ - onCancel: () => void + onCancel: () => void; /** Video element for live preview */ - videoElement?: HTMLVideoElement | null + videoElement?: HTMLVideoElement | null; /** Called when corners change (for external preview) */ - onCornersChange?: (corners: QuadCorners) => void + onCornersChange?: (corners: QuadCorners) => void; } /** Imperative handle for controlling CalibrationOverlay from parent */ export interface CalibrationOverlayHandle { /** Rotate the calibration quad 90° in the given direction */ - rotate: (direction: 'left' | 'right') => void + rotate: (direction: "left" | "right") => void; /** Complete calibration with current settings */ - complete: () => void + complete: () => void; } -type CornerKey = 'topLeft' | 'topRight' | 'bottomLeft' | 'bottomRight' -type DragTarget = CornerKey | 'quad' | `divider-${number}` | null +type CornerKey = "topLeft" | "topRight" | "bottomLeft" | "bottomRight"; +type DragTarget = CornerKey | "quad" | `divider-${number}` | null; /** * Convert QuadCorners to legacy ROI format (bounding box) */ function cornersToROI(corners: QuadCorners): ROI { - const minX = Math.min(corners.topLeft.x, corners.bottomLeft.x) - const maxX = Math.max(corners.topRight.x, corners.bottomRight.x) - const minY = Math.min(corners.topLeft.y, corners.topRight.y) - const maxY = Math.max(corners.bottomLeft.y, corners.bottomRight.y) + const minX = Math.min(corners.topLeft.x, corners.bottomLeft.x); + const maxX = Math.max(corners.topRight.x, corners.bottomRight.x); + const minY = Math.min(corners.topLeft.y, corners.topRight.y); + const maxY = Math.max(corners.bottomLeft.y, corners.bottomRight.y); return { x: minX, y: minY, width: maxX - minX, height: maxY - minY, - } + }; } /** @@ -59,17 +66,24 @@ function cornersToROI(corners: QuadCorners): ROI { * @param corners - The quadrilateral corners * @returns Top and bottom points for a vertical line at position t */ -function getColumnLine(t: number, corners: QuadCorners): { top: Point; bottom: Point } { +function getColumnLine( + t: number, + corners: QuadCorners, +): { top: Point; bottom: Point } { return { top: { x: corners.topLeft.x + t * (corners.topRight.x - corners.topLeft.x), y: corners.topLeft.y + t * (corners.topRight.y - corners.topLeft.y), }, bottom: { - x: corners.bottomLeft.x + t * (corners.bottomRight.x - corners.bottomLeft.x), - y: corners.bottomLeft.y + t * (corners.bottomRight.y - corners.bottomLeft.y), + x: + corners.bottomLeft.x + + t * (corners.bottomRight.x - corners.bottomLeft.x), + y: + corners.bottomLeft.y + + t * (corners.bottomRight.y - corners.bottomLeft.y), }, - } + }; } /** @@ -83,516 +97,539 @@ function getColumnLine(t: number, corners: QuadCorners): { top: Point; bottom: P * Control buttons (Cancel, Done, Rotate) are NOT rendered by this component. * Use the imperative handle to trigger rotate/complete from parent controls. */ -export const CalibrationOverlay = forwardRef( - function CalibrationOverlay( - { - columnCount, - videoWidth, - videoHeight, - containerWidth, - containerHeight, - initialCalibration, - onComplete, - onCancel, - videoElement, - onCornersChange, - }, - ref - ): ReactNode { - // Calculate actual visible video bounds (accounting for object-fit: contain letterboxing) - const videoAspect = videoWidth / videoHeight - const containerAspect = containerWidth / containerHeight +export const CalibrationOverlay = forwardRef< + CalibrationOverlayHandle, + CalibrationOverlayProps +>(function CalibrationOverlay( + { + columnCount, + videoWidth, + videoHeight, + containerWidth, + containerHeight, + initialCalibration, + onComplete, + onCancel, + videoElement, + onCornersChange, + }, + ref, +): ReactNode { + // Calculate actual visible video bounds (accounting for object-fit: contain letterboxing) + const videoAspect = videoWidth / videoHeight; + const containerAspect = containerWidth / containerHeight; - let displayedVideoWidth: number - let displayedVideoHeight: number - let videoOffsetX: number - let videoOffsetY: number + let displayedVideoWidth: number; + let displayedVideoHeight: number; + let videoOffsetX: number; + let videoOffsetY: number; - if (videoAspect > containerAspect) { - // Video is wider than container - letterbox top/bottom - displayedVideoWidth = containerWidth - displayedVideoHeight = containerWidth / videoAspect - videoOffsetX = 0 - videoOffsetY = (containerHeight - displayedVideoHeight) / 2 - } else { - // Video is taller than container - letterbox left/right - displayedVideoHeight = containerHeight - displayedVideoWidth = containerHeight * videoAspect - videoOffsetX = (containerWidth - displayedVideoWidth) / 2 - videoOffsetY = 0 - } + if (videoAspect > containerAspect) { + // Video is wider than container - letterbox top/bottom + displayedVideoWidth = containerWidth; + displayedVideoHeight = containerWidth / videoAspect; + videoOffsetX = 0; + videoOffsetY = (containerHeight - displayedVideoHeight) / 2; + } else { + // Video is taller than container - letterbox left/right + displayedVideoHeight = containerHeight; + displayedVideoWidth = containerHeight * videoAspect; + videoOffsetX = (containerWidth - displayedVideoWidth) / 2; + videoOffsetY = 0; + } - // Uniform scale factor (maintains aspect ratio) - const scale = displayedVideoWidth / videoWidth + // Uniform scale factor (maintains aspect ratio) + const scale = displayedVideoWidth / videoWidth; - // Initialize corners state - const getDefaultCorners = (): QuadCorners => { - // Default to a slightly trapezoidal shape (wider at bottom for typical desk perspective) - // Use larger margins to ensure all corners are visible and draggable - const topMargin = 0.15 - const bottomMargin = 0.2 // Larger margin at bottom to keep handles visible - const sideMargin = 0.15 - const topInset = 0.03 // Make top slightly narrower than bottom for perspective - return { - topLeft: { - x: videoWidth * (sideMargin + topInset), - y: videoHeight * topMargin, - }, - topRight: { - x: videoWidth * (1 - sideMargin - topInset), - y: videoHeight * topMargin, - }, - bottomLeft: { - x: videoWidth * sideMargin, - y: videoHeight * (1 - bottomMargin), - }, - bottomRight: { - x: videoWidth * (1 - sideMargin), - y: videoHeight * (1 - bottomMargin), - }, - } - } - - const [corners, setCorners] = useState(() => { - if (initialCalibration?.corners) { - return initialCalibration.corners - } - // Convert from legacy ROI if available - if (initialCalibration?.roi) { - const roi = initialCalibration.roi - return { - topLeft: { x: roi.x, y: roi.y }, - topRight: { x: roi.x + roi.width, y: roi.y }, - bottomLeft: { x: roi.x, y: roi.y + roi.height }, - bottomRight: { x: roi.x + roi.width, y: roi.y + roi.height }, - } - } - return getDefaultCorners() - }) - - // Initialize column dividers (evenly spaced) - const getDefaultDividers = (): number[] => { - const dividers: number[] = [] - for (let i = 1; i < columnCount; i++) { - dividers.push(i / columnCount) - } - return dividers - } - - const [dividers, setDividers] = useState( - initialCalibration?.columnDividers ?? getDefaultDividers() - ) - - // Drag state - const [dragTarget, setDragTarget] = useState(null) - const dragStartRef = useRef<{ - x: number - y: number - corners: QuadCorners - dividers: number[] - } | null>(null) - - // Rotation animation state - const [rotationAnimation, setRotationAnimation] = useState<{ - startCorners: QuadCorners - endCorners: QuadCorners - startTime: number - } | null>(null) - const ROTATION_DURATION_MS = 300 - - // Easing function: easeOutCubic for smooth deceleration - const easeOutCubic = (t: number): number => { - return 1 - Math.pow(1 - t, 3) - } - - // Interpolate between two points - const lerpPoint = (a: Point, b: Point, t: number): Point => ({ - x: a.x + (b.x - a.x) * t, - y: a.y + (b.y - a.y) * t, - }) - - // Interpolate between two sets of corners - const lerpCorners = (a: QuadCorners, b: QuadCorners, t: number): QuadCorners => ({ - topLeft: lerpPoint(a.topLeft, b.topLeft, t), - topRight: lerpPoint(a.topRight, b.topRight, t), - bottomLeft: lerpPoint(a.bottomLeft, b.bottomLeft, t), - bottomRight: lerpPoint(a.bottomRight, b.bottomRight, t), - }) - - // Animate rotation - useEffect(() => { - if (!rotationAnimation) return - - let animationId: number - - const animate = () => { - const elapsed = performance.now() - rotationAnimation.startTime - const progress = Math.min(elapsed / ROTATION_DURATION_MS, 1) - const easedProgress = easeOutCubic(progress) - - const interpolatedCorners = lerpCorners( - rotationAnimation.startCorners, - rotationAnimation.endCorners, - easedProgress - ) - - setCorners(interpolatedCorners) - - if (progress < 1) { - animationId = requestAnimationFrame(animate) - } else { - // Animation complete - ensure we're exactly at the end position - setCorners(rotationAnimation.endCorners) - setRotationAnimation(null) - } - } - - animationId = requestAnimationFrame(animate) - - return () => { - if (animationId) { - cancelAnimationFrame(animationId) - } - } - }, [rotationAnimation]) - - // Notify parent when corners change - useEffect(() => { - onCornersChange?.(corners) - }, [corners, onCornersChange]) - - // Handle pointer down on corners or dividers - const handlePointerDown = useCallback( - (e: React.PointerEvent, target: DragTarget) => { - e.preventDefault() - e.stopPropagation() - setDragTarget(target) - dragStartRef.current = { - x: e.clientX, - y: e.clientY, - corners: { ...corners }, - dividers: [...dividers], - } - ;(e.target as HTMLElement).setPointerCapture(e.pointerId) - }, - [corners, dividers] - ) - - // Handle pointer move during drag - const handlePointerMove = useCallback( - (e: React.PointerEvent) => { - if (!dragTarget || !dragStartRef.current) return - - const dx = (e.clientX - dragStartRef.current.x) / scale - const dy = (e.clientY - dragStartRef.current.y) / scale - const startCorners = dragStartRef.current.corners - - if (dragTarget === 'quad') { - // Move entire quadrilateral - // Calculate bounds to keep all corners within video - const minX = Math.min(startCorners.topLeft.x, startCorners.bottomLeft.x) - const maxX = Math.max(startCorners.topRight.x, startCorners.bottomRight.x) - const minY = Math.min(startCorners.topLeft.y, startCorners.topRight.y) - const maxY = Math.max(startCorners.bottomLeft.y, startCorners.bottomRight.y) - - // Clamp movement to keep quad within video bounds - const clampedDx = Math.max(-minX, Math.min(videoWidth - maxX, dx)) - const clampedDy = Math.max(-minY, Math.min(videoHeight - maxY, dy)) - - setCorners({ - topLeft: { - x: startCorners.topLeft.x + clampedDx, - y: startCorners.topLeft.y + clampedDy, - }, - topRight: { - x: startCorners.topRight.x + clampedDx, - y: startCorners.topRight.y + clampedDy, - }, - bottomLeft: { - x: startCorners.bottomLeft.x + clampedDx, - y: startCorners.bottomLeft.y + clampedDy, - }, - bottomRight: { - x: startCorners.bottomRight.x + clampedDx, - y: startCorners.bottomRight.y + clampedDy, - }, - }) - } else if ( - dragTarget === 'topLeft' || - dragTarget === 'topRight' || - dragTarget === 'bottomLeft' || - dragTarget === 'bottomRight' - ) { - // Move single corner - const startPoint = startCorners[dragTarget] - const newPoint: Point = { - x: Math.max(0, Math.min(videoWidth, startPoint.x + dx)), - y: Math.max(0, Math.min(videoHeight, startPoint.y + dy)), - } - setCorners((prev) => ({ - ...prev, - [dragTarget]: newPoint, - })) - } else if (dragTarget.startsWith('divider-')) { - // Move divider - const index = Number.parseInt(dragTarget.split('-')[1], 10) - const startDividers = dragStartRef.current.dividers - - // Calculate dx as fraction of quad width (average of top and bottom widths) - const topWidth = startCorners.topRight.x - startCorners.topLeft.x - const bottomWidth = startCorners.bottomRight.x - startCorners.bottomLeft.x - const avgWidth = (topWidth + bottomWidth) / 2 - const dxFraction = dx / avgWidth - - const newDividers = [...startDividers] - const minPos = index === 0 ? 0.05 : startDividers[index - 1] + 0.05 - const maxPos = index === startDividers.length - 1 ? 0.95 : startDividers[index + 1] - 0.05 - newDividers[index] = Math.max(minPos, Math.min(maxPos, startDividers[index] + dxFraction)) - setDividers(newDividers) - } - }, - [dragTarget, scale, videoWidth, videoHeight] - ) - - // Handle pointer up - const handlePointerUp = useCallback(() => { - setDragTarget(null) - dragStartRef.current = null - }, []) - - /** - * Rotate corners 90° clockwise or counter-clockwise around the quad center - * This reassigns corner labels, not their positions - * Animates the transition smoothly - */ - const handleRotate = useCallback( - (direction: 'left' | 'right') => { - // Don't start a new rotation if one is already in progress - if (rotationAnimation) return - - const currentCorners = corners - - const newCorners = - direction === 'right' - ? // Rotate 90° clockwise: TL→TR, TR→BR, BR→BL, BL→TL - { - topLeft: currentCorners.bottomLeft, - topRight: currentCorners.topLeft, - bottomRight: currentCorners.topRight, - bottomLeft: currentCorners.bottomRight, - } - : // Rotate 90° counter-clockwise: TL→BL, BL→BR, BR→TR, TR→TL - { - topLeft: currentCorners.topRight, - topRight: currentCorners.bottomRight, - bottomRight: currentCorners.bottomLeft, - bottomLeft: currentCorners.topLeft, - } - - // Start animation - setRotationAnimation({ - startCorners: currentCorners, - endCorners: newCorners, - startTime: performance.now(), - }) - }, - [corners, rotationAnimation] - ) - - // Handle complete - const handleComplete = useCallback(() => { - const grid: CalibrationGrid = { - roi: cornersToROI(corners), - corners, - columnCount, - columnDividers: dividers, - rotation: 0, // Deprecated - perspective handled by corners - } - onComplete(grid) - }, [corners, columnCount, dividers, onComplete]) - - // Expose imperative methods to parent - useImperativeHandle( - ref, - () => ({ - rotate: handleRotate, - complete: handleComplete, - }), - [handleRotate, handleComplete] - ) - - // Convert corners to display coordinates (accounting for letterbox offset) - const displayCorners: QuadCorners = { + // Initialize corners state + const getDefaultCorners = (): QuadCorners => { + // Default to a slightly trapezoidal shape (wider at bottom for typical desk perspective) + // Use larger margins to ensure all corners are visible and draggable + const topMargin = 0.15; + const bottomMargin = 0.2; // Larger margin at bottom to keep handles visible + const sideMargin = 0.15; + const topInset = 0.03; // Make top slightly narrower than bottom for perspective + return { topLeft: { - x: corners.topLeft.x * scale + videoOffsetX, - y: corners.topLeft.y * scale + videoOffsetY, + x: videoWidth * (sideMargin + topInset), + y: videoHeight * topMargin, }, topRight: { - x: corners.topRight.x * scale + videoOffsetX, - y: corners.topRight.y * scale + videoOffsetY, + x: videoWidth * (1 - sideMargin - topInset), + y: videoHeight * topMargin, }, bottomLeft: { - x: corners.bottomLeft.x * scale + videoOffsetX, - y: corners.bottomLeft.y * scale + videoOffsetY, + x: videoWidth * sideMargin, + y: videoHeight * (1 - bottomMargin), }, bottomRight: { - x: corners.bottomRight.x * scale + videoOffsetX, - y: corners.bottomRight.y * scale + videoOffsetY, + x: videoWidth * (1 - sideMargin), + y: videoHeight * (1 - bottomMargin), }, - } + }; + }; - // Create SVG path for the quadrilateral - const quadPath = `M ${displayCorners.topLeft.x} ${displayCorners.topLeft.y} + const [corners, setCorners] = useState(() => { + if (initialCalibration?.corners) { + return initialCalibration.corners; + } + // Convert from legacy ROI if available + if (initialCalibration?.roi) { + const roi = initialCalibration.roi; + return { + topLeft: { x: roi.x, y: roi.y }, + topRight: { x: roi.x + roi.width, y: roi.y }, + bottomLeft: { x: roi.x, y: roi.y + roi.height }, + bottomRight: { x: roi.x + roi.width, y: roi.y + roi.height }, + }; + } + return getDefaultCorners(); + }); + + // Initialize column dividers (evenly spaced) + const getDefaultDividers = (): number[] => { + const dividers: number[] = []; + for (let i = 1; i < columnCount; i++) { + dividers.push(i / columnCount); + } + return dividers; + }; + + const [dividers, setDividers] = useState( + initialCalibration?.columnDividers ?? getDefaultDividers(), + ); + + // Drag state + const [dragTarget, setDragTarget] = useState(null); + const dragStartRef = useRef<{ + x: number; + y: number; + corners: QuadCorners; + dividers: number[]; + } | null>(null); + + // Rotation animation state + const [rotationAnimation, setRotationAnimation] = useState<{ + startCorners: QuadCorners; + endCorners: QuadCorners; + startTime: number; + } | null>(null); + const ROTATION_DURATION_MS = 300; + + // Easing function: easeOutCubic for smooth deceleration + const easeOutCubic = (t: number): number => { + return 1 - Math.pow(1 - t, 3); + }; + + // Interpolate between two points + const lerpPoint = (a: Point, b: Point, t: number): Point => ({ + x: a.x + (b.x - a.x) * t, + y: a.y + (b.y - a.y) * t, + }); + + // Interpolate between two sets of corners + const lerpCorners = ( + a: QuadCorners, + b: QuadCorners, + t: number, + ): QuadCorners => ({ + topLeft: lerpPoint(a.topLeft, b.topLeft, t), + topRight: lerpPoint(a.topRight, b.topRight, t), + bottomLeft: lerpPoint(a.bottomLeft, b.bottomLeft, t), + bottomRight: lerpPoint(a.bottomRight, b.bottomRight, t), + }); + + // Animate rotation + useEffect(() => { + if (!rotationAnimation) return; + + let animationId: number; + + const animate = () => { + const elapsed = performance.now() - rotationAnimation.startTime; + const progress = Math.min(elapsed / ROTATION_DURATION_MS, 1); + const easedProgress = easeOutCubic(progress); + + const interpolatedCorners = lerpCorners( + rotationAnimation.startCorners, + rotationAnimation.endCorners, + easedProgress, + ); + + setCorners(interpolatedCorners); + + if (progress < 1) { + animationId = requestAnimationFrame(animate); + } else { + // Animation complete - ensure we're exactly at the end position + setCorners(rotationAnimation.endCorners); + setRotationAnimation(null); + } + }; + + animationId = requestAnimationFrame(animate); + + return () => { + if (animationId) { + cancelAnimationFrame(animationId); + } + }; + }, [rotationAnimation]); + + // Notify parent when corners change + useEffect(() => { + onCornersChange?.(corners); + }, [corners, onCornersChange]); + + // Handle pointer down on corners or dividers + const handlePointerDown = useCallback( + (e: React.PointerEvent, target: DragTarget) => { + e.preventDefault(); + e.stopPropagation(); + setDragTarget(target); + dragStartRef.current = { + x: e.clientX, + y: e.clientY, + corners: { ...corners }, + dividers: [...dividers], + }; + (e.target as HTMLElement).setPointerCapture(e.pointerId); + }, + [corners, dividers], + ); + + // Handle pointer move during drag + const handlePointerMove = useCallback( + (e: React.PointerEvent) => { + if (!dragTarget || !dragStartRef.current) return; + + const dx = (e.clientX - dragStartRef.current.x) / scale; + const dy = (e.clientY - dragStartRef.current.y) / scale; + const startCorners = dragStartRef.current.corners; + + if (dragTarget === "quad") { + // Move entire quadrilateral + // Calculate bounds to keep all corners within video + const minX = Math.min( + startCorners.topLeft.x, + startCorners.bottomLeft.x, + ); + const maxX = Math.max( + startCorners.topRight.x, + startCorners.bottomRight.x, + ); + const minY = Math.min(startCorners.topLeft.y, startCorners.topRight.y); + const maxY = Math.max( + startCorners.bottomLeft.y, + startCorners.bottomRight.y, + ); + + // Clamp movement to keep quad within video bounds + const clampedDx = Math.max(-minX, Math.min(videoWidth - maxX, dx)); + const clampedDy = Math.max(-minY, Math.min(videoHeight - maxY, dy)); + + setCorners({ + topLeft: { + x: startCorners.topLeft.x + clampedDx, + y: startCorners.topLeft.y + clampedDy, + }, + topRight: { + x: startCorners.topRight.x + clampedDx, + y: startCorners.topRight.y + clampedDy, + }, + bottomLeft: { + x: startCorners.bottomLeft.x + clampedDx, + y: startCorners.bottomLeft.y + clampedDy, + }, + bottomRight: { + x: startCorners.bottomRight.x + clampedDx, + y: startCorners.bottomRight.y + clampedDy, + }, + }); + } else if ( + dragTarget === "topLeft" || + dragTarget === "topRight" || + dragTarget === "bottomLeft" || + dragTarget === "bottomRight" + ) { + // Move single corner + const startPoint = startCorners[dragTarget]; + const newPoint: Point = { + x: Math.max(0, Math.min(videoWidth, startPoint.x + dx)), + y: Math.max(0, Math.min(videoHeight, startPoint.y + dy)), + }; + setCorners((prev) => ({ + ...prev, + [dragTarget]: newPoint, + })); + } else if (dragTarget.startsWith("divider-")) { + // Move divider + const index = Number.parseInt(dragTarget.split("-")[1], 10); + const startDividers = dragStartRef.current.dividers; + + // Calculate dx as fraction of quad width (average of top and bottom widths) + const topWidth = startCorners.topRight.x - startCorners.topLeft.x; + const bottomWidth = + startCorners.bottomRight.x - startCorners.bottomLeft.x; + const avgWidth = (topWidth + bottomWidth) / 2; + const dxFraction = dx / avgWidth; + + const newDividers = [...startDividers]; + const minPos = index === 0 ? 0.05 : startDividers[index - 1] + 0.05; + const maxPos = + index === startDividers.length - 1 + ? 0.95 + : startDividers[index + 1] - 0.05; + newDividers[index] = Math.max( + minPos, + Math.min(maxPos, startDividers[index] + dxFraction), + ); + setDividers(newDividers); + } + }, + [dragTarget, scale, videoWidth, videoHeight], + ); + + // Handle pointer up + const handlePointerUp = useCallback(() => { + setDragTarget(null); + dragStartRef.current = null; + }, []); + + /** + * Rotate corners 90° clockwise or counter-clockwise around the quad center + * This reassigns corner labels, not their positions + * Animates the transition smoothly + */ + const handleRotate = useCallback( + (direction: "left" | "right") => { + // Don't start a new rotation if one is already in progress + if (rotationAnimation) return; + + const currentCorners = corners; + + const newCorners = + direction === "right" + ? // Rotate 90° clockwise: TL→TR, TR→BR, BR→BL, BL→TL + { + topLeft: currentCorners.bottomLeft, + topRight: currentCorners.topLeft, + bottomRight: currentCorners.topRight, + bottomLeft: currentCorners.bottomRight, + } + : // Rotate 90° counter-clockwise: TL→BL, BL→BR, BR→TR, TR→TL + { + topLeft: currentCorners.topRight, + topRight: currentCorners.bottomRight, + bottomRight: currentCorners.bottomLeft, + bottomLeft: currentCorners.topLeft, + }; + + // Start animation + setRotationAnimation({ + startCorners: currentCorners, + endCorners: newCorners, + startTime: performance.now(), + }); + }, + [corners, rotationAnimation], + ); + + // Handle complete + const handleComplete = useCallback(() => { + const grid: CalibrationGrid = { + roi: cornersToROI(corners), + corners, + columnCount, + columnDividers: dividers, + rotation: 0, // Deprecated - perspective handled by corners + }; + onComplete(grid); + }, [corners, columnCount, dividers, onComplete]); + + // Expose imperative methods to parent + useImperativeHandle( + ref, + () => ({ + rotate: handleRotate, + complete: handleComplete, + }), + [handleRotate, handleComplete], + ); + + // Convert corners to display coordinates (accounting for letterbox offset) + const displayCorners: QuadCorners = { + topLeft: { + x: corners.topLeft.x * scale + videoOffsetX, + y: corners.topLeft.y * scale + videoOffsetY, + }, + topRight: { + x: corners.topRight.x * scale + videoOffsetX, + y: corners.topRight.y * scale + videoOffsetY, + }, + bottomLeft: { + x: corners.bottomLeft.x * scale + videoOffsetX, + y: corners.bottomLeft.y * scale + videoOffsetY, + }, + bottomRight: { + x: corners.bottomRight.x * scale + videoOffsetX, + y: corners.bottomRight.y * scale + videoOffsetY, + }, + }; + + // Create SVG path for the quadrilateral + const quadPath = `M ${displayCorners.topLeft.x} ${displayCorners.topLeft.y} L ${displayCorners.topRight.x} ${displayCorners.topRight.y} L ${displayCorners.bottomRight.x} ${displayCorners.bottomRight.y} - L ${displayCorners.bottomLeft.x} ${displayCorners.bottomLeft.y} Z` + L ${displayCorners.bottomLeft.x} ${displayCorners.bottomLeft.y} Z`; - const handleSize = 16 + const handleSize = 16; - // Corner positions for handles - const cornerPositions: { key: CornerKey; point: Point }[] = [ - { key: 'topLeft', point: displayCorners.topLeft }, - { key: 'topRight', point: displayCorners.topRight }, - { key: 'bottomLeft', point: displayCorners.bottomLeft }, - { key: 'bottomRight', point: displayCorners.bottomRight }, - ] + // Corner positions for handles + const cornerPositions: { key: CornerKey; point: Point }[] = [ + { key: "topLeft", point: displayCorners.topLeft }, + { key: "topRight", point: displayCorners.topRight }, + { key: "bottomLeft", point: displayCorners.bottomLeft }, + { key: "bottomRight", point: displayCorners.bottomRight }, + ]; - return ( -
+ {/* Semi-transparent overlay outside quadrilateral */} + - {/* Semi-transparent overlay outside quadrilateral */} - - {/* Darkened area outside quadrilateral */} - - - - - - - + {/* Darkened area outside quadrilateral */} + + + + + + + - {/* Clickable fill area inside quadrilateral - for moving the entire quad */} - handlePointerDown(e, 'quad')} - /> + {/* Clickable fill area inside quadrilateral - for moving the entire quad */} + handlePointerDown(e, "quad")} + /> - {/* Quadrilateral border */} - + {/* Quadrilateral border */} + - {/* Column divider lines */} - {dividers.map((divider, i) => { - const line = getColumnLine(divider, displayCorners) - return ( - handlePointerDown(e, `divider-${i}`)} - /> - ) + {/* Column divider lines */} + {dividers.map((divider, i) => { + const line = getColumnLine(divider, displayCorners); + return ( + handlePointerDown(e, `divider-${i}`)} + /> + ); + })} + + {/* Beam indicator line (20% from top, interpolated) */} + {(() => { + const beamT = 0.2; + const leftPoint: Point = { + x: + displayCorners.topLeft.x + + beamT * (displayCorners.bottomLeft.x - displayCorners.topLeft.x), + y: + displayCorners.topLeft.y + + beamT * (displayCorners.bottomLeft.y - displayCorners.topLeft.y), + }; + const rightPoint: Point = { + x: + displayCorners.topRight.x + + beamT * + (displayCorners.bottomRight.x - displayCorners.topRight.x), + y: + displayCorners.topRight.y + + beamT * + (displayCorners.bottomRight.y - displayCorners.topRight.y), + }; + return ( + + ); + })()} + + + {/* Corner drag handles */} + {cornerPositions.map(({ key, point }) => ( +
handlePointerDown(e, key)} + /> + ))} +
+ ); +}); - {/* Beam indicator line (20% from top, interpolated) */} - {(() => { - const beamT = 0.2 - const leftPoint: Point = { - x: - displayCorners.topLeft.x + - beamT * (displayCorners.bottomLeft.x - displayCorners.topLeft.x), - y: - displayCorners.topLeft.y + - beamT * (displayCorners.bottomLeft.y - displayCorners.topLeft.y), - } - const rightPoint: Point = { - x: - displayCorners.topRight.x + - beamT * (displayCorners.bottomRight.x - displayCorners.topRight.x), - y: - displayCorners.topRight.y + - beamT * (displayCorners.bottomRight.y - displayCorners.topRight.y), - } - return ( - - ) - })()} - - - {/* Corner drag handles */} - {cornerPositions.map(({ key, point }) => ( -
handlePointerDown(e, key)} - /> - ))} -
- ) - } -) - -export default CalibrationOverlay +export default CalibrationOverlay;