feat(vision): add remote phone camera support for abacus detection
Enable using a phone as a remote camera source for abacus vision detection. The phone handles calibration and perspective correction locally, then streams cropped frames to the desktop via Socket.IO. Features: - QR code generation for easy phone connection - Auto-calibration with ArUco markers on phone - Manual calibration option - Proper OpenCV perspective transform (not bounding box crop) - Real-time frame streaming with frame rate display - LAN address support via NEXT_PUBLIC_LAN_HOST env var 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Thomas Hallock
parent
9e9a06f2e4
commit
8e4975d395
|
|
@ -0,0 +1,62 @@
|
|||
import { NextResponse } from 'next/server'
|
||||
import { createRemoteCameraSession, getRemoteCameraSession } from '@/lib/remote-camera/session-manager'
|
||||
|
||||
/**
|
||||
* POST /api/remote-camera
|
||||
* Create a new remote camera session
|
||||
*/
|
||||
export async function POST() {
|
||||
try {
|
||||
const session = createRemoteCameraSession()
|
||||
|
||||
return NextResponse.json({
|
||||
sessionId: session.id,
|
||||
expiresAt: session.expiresAt.toISOString(),
|
||||
})
|
||||
} catch (error) {
|
||||
console.error('Failed to create remote camera session:', error)
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to create session' },
|
||||
{ status: 500 }
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* GET /api/remote-camera?sessionId=xxx
|
||||
* Check if a session is valid and get its status
|
||||
*/
|
||||
export async function GET(request: Request) {
|
||||
try {
|
||||
const url = new URL(request.url)
|
||||
const sessionId = url.searchParams.get('sessionId')
|
||||
|
||||
if (!sessionId) {
|
||||
return NextResponse.json(
|
||||
{ error: 'Session ID required' },
|
||||
{ status: 400 }
|
||||
)
|
||||
}
|
||||
|
||||
const session = getRemoteCameraSession(sessionId)
|
||||
|
||||
if (!session) {
|
||||
return NextResponse.json(
|
||||
{ error: 'Session not found or expired' },
|
||||
{ status: 404 }
|
||||
)
|
||||
}
|
||||
|
||||
return NextResponse.json({
|
||||
sessionId: session.id,
|
||||
phoneConnected: session.phoneConnected,
|
||||
expiresAt: session.expiresAt.toISOString(),
|
||||
})
|
||||
} catch (error) {
|
||||
console.error('Failed to get remote camera session:', error)
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to get session' },
|
||||
{ status: 500 }
|
||||
)
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,514 @@
|
|||
'use client'
|
||||
|
||||
import { useParams } from 'next/navigation'
|
||||
import { useCallback, useEffect, useRef, useState } from 'react'
|
||||
import { VisionCameraFeed } from '@/components/vision/VisionCameraFeed'
|
||||
import { CalibrationOverlay } from '@/components/vision/CalibrationOverlay'
|
||||
import { useDeskViewCamera } from '@/hooks/useDeskViewCamera'
|
||||
import { useRemoteCameraPhone } from '@/hooks/useRemoteCameraPhone'
|
||||
import {
|
||||
detectMarkers,
|
||||
initArucoDetector,
|
||||
loadAruco,
|
||||
MARKER_IDS,
|
||||
} from '@/lib/vision/arucoDetection'
|
||||
import type { CalibrationGrid, QuadCorners } from '@/types/vision'
|
||||
import { css } from '../../../../styled-system/css'
|
||||
|
||||
type CalibrationMode = 'auto' | 'manual'
|
||||
type ConnectionStatus = 'connecting' | 'connected' | 'error' | 'expired'
|
||||
|
||||
/**
|
||||
* Phone Camera Page
|
||||
*
|
||||
* Accessed by scanning QR code on desktop.
|
||||
* Captures video, performs calibration (auto or manual),
|
||||
* and sends cropped abacus frames to the desktop.
|
||||
*/
|
||||
export default function RemoteCameraPage() {
|
||||
const params = useParams<{ sessionId: string }>()
|
||||
const sessionId = params.sessionId
|
||||
|
||||
// Session validation state
|
||||
const [sessionStatus, setSessionStatus] = useState<ConnectionStatus>('connecting')
|
||||
const [sessionError, setSessionError] = useState<string | null>(null)
|
||||
|
||||
// Camera state
|
||||
const {
|
||||
isLoading: isCameraLoading,
|
||||
error: cameraError,
|
||||
videoStream,
|
||||
currentDevice,
|
||||
requestCamera,
|
||||
stopCamera,
|
||||
} = useDeskViewCamera()
|
||||
|
||||
// Remote camera connection
|
||||
const {
|
||||
isConnected,
|
||||
isSending,
|
||||
error: connectionError,
|
||||
connect,
|
||||
disconnect,
|
||||
startSending,
|
||||
stopSending,
|
||||
updateCalibration,
|
||||
} = useRemoteCameraPhone()
|
||||
|
||||
// Calibration state
|
||||
const [calibrationMode, setCalibrationMode] = useState<CalibrationMode>('auto')
|
||||
const [calibration, setCalibration] = useState<CalibrationGrid | null>(null)
|
||||
const [isCalibrating, setIsCalibrating] = useState(false)
|
||||
const [markersDetected, setMarkersDetected] = useState(0)
|
||||
const [arucoReady, setArucoReady] = useState(false)
|
||||
|
||||
// Video element ref
|
||||
const videoRef = useRef<HTMLVideoElement | null>(null)
|
||||
const [videoDimensions, setVideoDimensions] = useState({ width: 0, height: 0 })
|
||||
const containerRef = useRef<HTMLDivElement>(null)
|
||||
const [containerDimensions, setContainerDimensions] = useState({ width: 0, height: 0 })
|
||||
|
||||
// Validate session on mount
|
||||
useEffect(() => {
|
||||
async function validateSession() {
|
||||
try {
|
||||
const response = await fetch(`/api/remote-camera?sessionId=${sessionId}`)
|
||||
if (response.ok) {
|
||||
setSessionStatus('connected')
|
||||
} else if (response.status === 404) {
|
||||
setSessionStatus('expired')
|
||||
setSessionError('Session not found or expired')
|
||||
} else {
|
||||
setSessionStatus('error')
|
||||
const data = await response.json()
|
||||
setSessionError(data.error || 'Failed to validate session')
|
||||
}
|
||||
} catch (err) {
|
||||
setSessionStatus('error')
|
||||
setSessionError('Network error')
|
||||
}
|
||||
}
|
||||
|
||||
validateSession()
|
||||
}, [sessionId])
|
||||
|
||||
// Load ArUco library
|
||||
useEffect(() => {
|
||||
loadAruco()
|
||||
.then(() => {
|
||||
initArucoDetector()
|
||||
setArucoReady(true)
|
||||
})
|
||||
.catch((err) => {
|
||||
console.error('Failed to load ArUco:', err)
|
||||
})
|
||||
}, [])
|
||||
|
||||
// Connect to session when validated
|
||||
useEffect(() => {
|
||||
if (sessionStatus === 'connected' && !isConnected) {
|
||||
connect(sessionId)
|
||||
}
|
||||
}, [sessionStatus, isConnected, sessionId, connect])
|
||||
|
||||
// Request camera when connected
|
||||
useEffect(() => {
|
||||
if (isConnected && !videoStream && !isCameraLoading) {
|
||||
requestCamera()
|
||||
}
|
||||
}, [isConnected, videoStream, isCameraLoading, requestCamera])
|
||||
|
||||
// Update container dimensions
|
||||
useEffect(() => {
|
||||
const container = containerRef.current
|
||||
if (!container) return
|
||||
|
||||
const updateDimensions = () => {
|
||||
const rect = container.getBoundingClientRect()
|
||||
setContainerDimensions({ width: rect.width, height: rect.height })
|
||||
}
|
||||
|
||||
updateDimensions()
|
||||
const resizeObserver = new ResizeObserver(updateDimensions)
|
||||
resizeObserver.observe(container)
|
||||
|
||||
return () => resizeObserver.disconnect()
|
||||
}, [])
|
||||
|
||||
// Auto-detect markers
|
||||
useEffect(() => {
|
||||
if (calibrationMode !== 'auto' || !videoStream || !arucoReady || !videoRef.current) return
|
||||
|
||||
const video = videoRef.current
|
||||
let animationId: number
|
||||
|
||||
const detectLoop = () => {
|
||||
if (video.readyState >= 2) {
|
||||
const result = detectMarkers(video)
|
||||
setMarkersDetected(result.markersFound)
|
||||
|
||||
if (result.allMarkersFound && result.quadCorners) {
|
||||
// Auto-calibration successful!
|
||||
const grid: CalibrationGrid = {
|
||||
roi: {
|
||||
x: Math.min(result.quadCorners.topLeft.x, result.quadCorners.bottomLeft.x),
|
||||
y: Math.min(result.quadCorners.topLeft.y, result.quadCorners.topRight.y),
|
||||
width:
|
||||
Math.max(result.quadCorners.topRight.x, result.quadCorners.bottomRight.x) -
|
||||
Math.min(result.quadCorners.topLeft.x, result.quadCorners.bottomLeft.x),
|
||||
height:
|
||||
Math.max(result.quadCorners.bottomLeft.y, result.quadCorners.bottomRight.y) -
|
||||
Math.min(result.quadCorners.topLeft.y, result.quadCorners.topRight.y),
|
||||
},
|
||||
corners: result.quadCorners,
|
||||
columnCount: 13, // Default column count
|
||||
columnDividers: Array.from({ length: 12 }, (_, i) => (i + 1) / 13),
|
||||
rotation: 0,
|
||||
}
|
||||
setCalibration(grid)
|
||||
}
|
||||
}
|
||||
|
||||
animationId = requestAnimationFrame(detectLoop)
|
||||
}
|
||||
|
||||
detectLoop()
|
||||
|
||||
return () => {
|
||||
if (animationId) cancelAnimationFrame(animationId)
|
||||
}
|
||||
}, [calibrationMode, videoStream, arucoReady])
|
||||
|
||||
// Start/stop sending based on calibration
|
||||
useEffect(() => {
|
||||
if (calibration && isConnected && videoRef.current && calibration.corners) {
|
||||
startSending(videoRef.current, calibration.corners)
|
||||
}
|
||||
}, [calibration, isConnected, startSending])
|
||||
|
||||
// Update calibration when corners change
|
||||
const handleCornersChange = useCallback(
|
||||
(corners: QuadCorners) => {
|
||||
if (isSending) {
|
||||
updateCalibration(corners)
|
||||
}
|
||||
},
|
||||
[isSending, updateCalibration]
|
||||
)
|
||||
|
||||
// Handle manual calibration complete
|
||||
const handleCalibrationComplete = useCallback((grid: CalibrationGrid) => {
|
||||
setCalibration(grid)
|
||||
setIsCalibrating(false)
|
||||
}, [])
|
||||
|
||||
// Handle calibration cancel
|
||||
const handleCalibrationCancel = useCallback(() => {
|
||||
setIsCalibrating(false)
|
||||
}, [])
|
||||
|
||||
// Handle video ready
|
||||
const handleVideoReady = useCallback((width: number, height: number) => {
|
||||
setVideoDimensions({ width, height })
|
||||
}, [])
|
||||
|
||||
// Cleanup on unmount
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
stopSending()
|
||||
disconnect()
|
||||
stopCamera()
|
||||
}
|
||||
}, [stopSending, disconnect, stopCamera])
|
||||
|
||||
// Render based on session status
|
||||
if (sessionStatus === 'connecting') {
|
||||
return (
|
||||
<div
|
||||
className={css({
|
||||
minHeight: '100vh',
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
bg: 'gray.900',
|
||||
color: 'white',
|
||||
})}
|
||||
>
|
||||
<div className={css({ textAlign: 'center' })}>
|
||||
<div
|
||||
className={css({
|
||||
width: 10,
|
||||
height: 10,
|
||||
border: '3px solid',
|
||||
borderColor: 'gray.600',
|
||||
borderTopColor: 'blue.400',
|
||||
borderRadius: 'full',
|
||||
mx: 'auto',
|
||||
mb: 4,
|
||||
})}
|
||||
style={{ animation: 'spin 1s linear infinite' }}
|
||||
/>
|
||||
<style>{`@keyframes spin { to { transform: rotate(360deg); } }`}</style>
|
||||
<p>Connecting to session...</p>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
if (sessionStatus === 'expired' || sessionStatus === 'error') {
|
||||
return (
|
||||
<div
|
||||
className={css({
|
||||
minHeight: '100vh',
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
bg: 'gray.900',
|
||||
color: 'white',
|
||||
px: 4,
|
||||
})}
|
||||
>
|
||||
<div className={css({ textAlign: 'center', maxWidth: '400px' })}>
|
||||
<div className={css({ fontSize: '4xl', mb: 4 })}>
|
||||
{sessionStatus === 'expired' ? '⏰' : '❌'}
|
||||
</div>
|
||||
<h1 className={css({ fontSize: 'xl', fontWeight: 'bold', mb: 2 })}>
|
||||
{sessionStatus === 'expired' ? 'Session Expired' : 'Connection Error'}
|
||||
</h1>
|
||||
<p className={css({ color: 'gray.400', mb: 4 })}>
|
||||
{sessionError || 'Please scan the QR code again on your desktop.'}
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
return (
|
||||
<div
|
||||
className={css({
|
||||
minHeight: '100vh',
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
bg: 'gray.900',
|
||||
color: 'white',
|
||||
})}
|
||||
data-component="remote-camera-page"
|
||||
>
|
||||
{/* Header */}
|
||||
<header
|
||||
className={css({
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'space-between',
|
||||
px: 4,
|
||||
py: 3,
|
||||
borderBottom: '1px solid',
|
||||
borderColor: 'gray.800',
|
||||
})}
|
||||
>
|
||||
<h1 className={css({ fontSize: 'lg', fontWeight: 'semibold' })}>Remote Camera</h1>
|
||||
<div
|
||||
className={css({
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
gap: 2,
|
||||
fontSize: 'sm',
|
||||
})}
|
||||
>
|
||||
<span
|
||||
className={css({
|
||||
width: 2,
|
||||
height: 2,
|
||||
borderRadius: 'full',
|
||||
bg: isConnected ? 'green.500' : 'yellow.500',
|
||||
})}
|
||||
/>
|
||||
<span className={css({ color: 'gray.400' })}>
|
||||
{isConnected ? (isSending ? 'Streaming' : 'Connected') : 'Connecting...'}
|
||||
</span>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
{/* Camera Feed */}
|
||||
<div
|
||||
ref={containerRef}
|
||||
className={css({
|
||||
flex: 1,
|
||||
position: 'relative',
|
||||
minHeight: '300px',
|
||||
})}
|
||||
>
|
||||
<VisionCameraFeed
|
||||
videoStream={videoStream}
|
||||
isLoading={isCameraLoading}
|
||||
calibration={calibration}
|
||||
showCalibrationGrid={!!calibration && !isCalibrating}
|
||||
videoRef={(el) => {
|
||||
videoRef.current = el
|
||||
}}
|
||||
onVideoReady={handleVideoReady}
|
||||
>
|
||||
{/* Manual calibration overlay */}
|
||||
{isCalibrating &&
|
||||
videoDimensions.width > 0 &&
|
||||
containerDimensions.width > 0 && (
|
||||
<CalibrationOverlay
|
||||
columnCount={13}
|
||||
videoWidth={videoDimensions.width}
|
||||
videoHeight={videoDimensions.height}
|
||||
containerWidth={containerDimensions.width}
|
||||
containerHeight={containerDimensions.height}
|
||||
initialCalibration={calibration}
|
||||
onComplete={handleCalibrationComplete}
|
||||
onCancel={handleCalibrationCancel}
|
||||
videoElement={videoRef.current}
|
||||
onCornersChange={handleCornersChange}
|
||||
/>
|
||||
)}
|
||||
</VisionCameraFeed>
|
||||
|
||||
{/* Camera error overlay */}
|
||||
{cameraError && (
|
||||
<div
|
||||
className={css({
|
||||
position: 'absolute',
|
||||
inset: 0,
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
bg: 'rgba(0, 0, 0, 0.8)',
|
||||
p: 4,
|
||||
})}
|
||||
>
|
||||
<div className={css({ textAlign: 'center' })}>
|
||||
<p className={css({ color: 'red.400', mb: 4 })}>{cameraError}</p>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => requestCamera()}
|
||||
className={css({
|
||||
px: 4,
|
||||
py: 2,
|
||||
bg: 'blue.600',
|
||||
color: 'white',
|
||||
borderRadius: 'lg',
|
||||
fontWeight: 'medium',
|
||||
border: 'none',
|
||||
cursor: 'pointer',
|
||||
})}
|
||||
>
|
||||
Retry Camera
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Controls */}
|
||||
<div
|
||||
className={css({
|
||||
px: 4,
|
||||
py: 3,
|
||||
borderTop: '1px solid',
|
||||
borderColor: 'gray.800',
|
||||
})}
|
||||
>
|
||||
{/* Mode selector */}
|
||||
<div className={css({ display: 'flex', gap: 2, mb: 3 })}>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setCalibrationMode('auto')}
|
||||
className={css({
|
||||
flex: 1,
|
||||
px: 3,
|
||||
py: 2,
|
||||
borderRadius: 'lg',
|
||||
fontWeight: 'medium',
|
||||
border: 'none',
|
||||
cursor: 'pointer',
|
||||
bg: calibrationMode === 'auto' ? 'blue.600' : 'gray.700',
|
||||
color: 'white',
|
||||
})}
|
||||
>
|
||||
Auto (Markers)
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setCalibrationMode('manual')}
|
||||
className={css({
|
||||
flex: 1,
|
||||
px: 3,
|
||||
py: 2,
|
||||
borderRadius: 'lg',
|
||||
fontWeight: 'medium',
|
||||
border: 'none',
|
||||
cursor: 'pointer',
|
||||
bg: calibrationMode === 'manual' ? 'blue.600' : 'gray.700',
|
||||
color: 'white',
|
||||
})}
|
||||
>
|
||||
Manual
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{/* Status */}
|
||||
{calibrationMode === 'auto' && (
|
||||
<div
|
||||
className={css({
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
gap: 2,
|
||||
mb: 3,
|
||||
p: 3,
|
||||
bg: 'gray.800',
|
||||
borderRadius: 'lg',
|
||||
})}
|
||||
>
|
||||
<span className={css({ fontSize: 'lg' })}>
|
||||
{markersDetected === 4 ? '✅' : '🔍'}
|
||||
</span>
|
||||
<div>
|
||||
<p className={css({ fontWeight: 'medium' })}>
|
||||
{markersDetected}/4 Markers Detected
|
||||
</p>
|
||||
<p className={css({ fontSize: 'sm', color: 'gray.400' })}>
|
||||
{calibration
|
||||
? 'Calibrated - Streaming'
|
||||
: 'Point camera at abacus markers'}
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{calibrationMode === 'manual' && !isCalibrating && (
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setIsCalibrating(true)}
|
||||
disabled={!videoStream}
|
||||
className={css({
|
||||
width: '100%',
|
||||
px: 4,
|
||||
py: 3,
|
||||
bg: 'green.600',
|
||||
color: 'white',
|
||||
borderRadius: 'lg',
|
||||
fontWeight: 'medium',
|
||||
fontSize: 'lg',
|
||||
border: 'none',
|
||||
cursor: 'pointer',
|
||||
_disabled: { opacity: 0.5, cursor: 'not-allowed' },
|
||||
})}
|
||||
>
|
||||
{calibration ? 'Recalibrate' : 'Start Calibration'}
|
||||
</button>
|
||||
)}
|
||||
|
||||
{/* Connection error */}
|
||||
{connectionError && (
|
||||
<p className={css({ color: 'red.400', fontSize: 'sm', mt: 2 })}>
|
||||
{connectionError}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
|
@ -8,9 +8,12 @@ import type { QuadCorners } from '@/types/vision'
|
|||
import { DEFAULT_STABILITY_CONFIG } from '@/types/vision'
|
||||
import { css } from '../../../styled-system/css'
|
||||
import { CalibrationOverlay } from './CalibrationOverlay'
|
||||
import { RemoteCameraReceiver } from './RemoteCameraReceiver'
|
||||
import { VisionCameraFeed } from './VisionCameraFeed'
|
||||
import { VisionStatusIndicator } from './VisionStatusIndicator'
|
||||
|
||||
type CameraSource = 'local' | 'phone'
|
||||
|
||||
export interface AbacusVisionBridgeProps {
|
||||
/** Number of abacus columns to detect */
|
||||
columnCount: number
|
||||
|
|
@ -52,14 +55,51 @@ export function AbacusVisionBridge({
|
|||
const [calibrationCorners, setCalibrationCorners] = useState<QuadCorners | null>(null)
|
||||
const [opencvReady, setOpencvReady] = useState(false)
|
||||
|
||||
// Camera source selection
|
||||
const [cameraSource, setCameraSource] = useState<CameraSource>('local')
|
||||
const [remoteCameraSessionId, setRemoteCameraSessionId] = useState<string | null>(null)
|
||||
|
||||
const vision = useAbacusVision({
|
||||
columnCount,
|
||||
onValueDetected,
|
||||
})
|
||||
|
||||
// Start camera on mount
|
||||
// Handle switching to phone camera
|
||||
const handleCameraSourceChange = useCallback(
|
||||
(source: CameraSource) => {
|
||||
setCameraSource(source)
|
||||
if (source === 'phone') {
|
||||
// Stop local camera - session will be created by RemoteCameraQRCode
|
||||
vision.disable()
|
||||
} else {
|
||||
// Stop remote session and start local camera
|
||||
setRemoteCameraSessionId(null)
|
||||
vision.enable()
|
||||
}
|
||||
},
|
||||
[vision]
|
||||
)
|
||||
|
||||
// Handle session created by QR code component
|
||||
const handleRemoteSessionCreated = useCallback((sessionId: string) => {
|
||||
setRemoteCameraSessionId(sessionId)
|
||||
}, [])
|
||||
|
||||
// Handle receiving frame from phone (for future processing)
|
||||
const handleRemoteFrame = useCallback(
|
||||
(imageData: string, timestamp: number) => {
|
||||
// TODO: Process the frame for bead detection
|
||||
// For now, the phone does the calibration and just sends cropped frames
|
||||
console.log('[VisionBridge] Received frame from phone', timestamp)
|
||||
},
|
||||
[]
|
||||
)
|
||||
|
||||
// Start camera on mount (only for local source)
|
||||
useEffect(() => {
|
||||
vision.enable()
|
||||
if (cameraSource === 'local') {
|
||||
vision.enable()
|
||||
}
|
||||
return () => vision.disable()
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, []) // Only run on mount/unmount - vision functions are stable
|
||||
|
|
@ -220,8 +260,57 @@ export function AbacusVisionBridge({
|
|||
</button>
|
||||
</div>
|
||||
|
||||
{/* Camera selector (if multiple cameras) */}
|
||||
{vision.availableDevices.length > 1 && (
|
||||
{/* Camera source selector */}
|
||||
<div
|
||||
data-element="camera-source"
|
||||
className={css({
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
gap: 2,
|
||||
p: 2,
|
||||
bg: 'gray.800',
|
||||
borderRadius: 'md',
|
||||
})}
|
||||
>
|
||||
<span className={css({ color: 'gray.400', fontSize: 'sm' })}>Source:</span>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => handleCameraSourceChange('local')}
|
||||
className={css({
|
||||
px: 3,
|
||||
py: 1,
|
||||
fontSize: 'sm',
|
||||
border: 'none',
|
||||
borderRadius: 'md',
|
||||
cursor: 'pointer',
|
||||
bg: cameraSource === 'local' ? 'blue.600' : 'gray.700',
|
||||
color: 'white',
|
||||
_hover: { bg: cameraSource === 'local' ? 'blue.500' : 'gray.600' },
|
||||
})}
|
||||
>
|
||||
Local Camera
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => handleCameraSourceChange('phone')}
|
||||
className={css({
|
||||
px: 3,
|
||||
py: 1,
|
||||
fontSize: 'sm',
|
||||
border: 'none',
|
||||
borderRadius: 'md',
|
||||
cursor: 'pointer',
|
||||
bg: cameraSource === 'phone' ? 'blue.600' : 'gray.700',
|
||||
color: 'white',
|
||||
_hover: { bg: cameraSource === 'phone' ? 'blue.500' : 'gray.600' },
|
||||
})}
|
||||
>
|
||||
Phone Camera
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{/* Camera selector (if multiple cameras and using local) */}
|
||||
{cameraSource === 'local' && vision.availableDevices.length > 1 && (
|
||||
<select
|
||||
data-element="camera-selector"
|
||||
value={vision.selectedDeviceId ?? ''}
|
||||
|
|
@ -244,57 +333,59 @@ export function AbacusVisionBridge({
|
|||
</select>
|
||||
)}
|
||||
|
||||
{/* Calibration mode toggle */}
|
||||
<div
|
||||
data-element="calibration-mode"
|
||||
className={css({
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
gap: 2,
|
||||
p: 2,
|
||||
bg: 'gray.800',
|
||||
borderRadius: 'md',
|
||||
})}
|
||||
>
|
||||
<span className={css({ color: 'gray.400', fontSize: 'sm' })}>Mode:</span>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => vision.setCalibrationMode('auto')}
|
||||
{/* Calibration mode toggle (local camera only) */}
|
||||
{cameraSource === 'local' && (
|
||||
<div
|
||||
data-element="calibration-mode"
|
||||
className={css({
|
||||
px: 3,
|
||||
py: 1,
|
||||
fontSize: 'sm',
|
||||
border: 'none',
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
gap: 2,
|
||||
p: 2,
|
||||
bg: 'gray.800',
|
||||
borderRadius: 'md',
|
||||
cursor: 'pointer',
|
||||
bg: vision.calibrationMode === 'auto' ? 'blue.600' : 'gray.700',
|
||||
color: 'white',
|
||||
_hover: { bg: vision.calibrationMode === 'auto' ? 'blue.500' : 'gray.600' },
|
||||
})}
|
||||
>
|
||||
Auto (Markers)
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => vision.setCalibrationMode('manual')}
|
||||
className={css({
|
||||
px: 3,
|
||||
py: 1,
|
||||
fontSize: 'sm',
|
||||
border: 'none',
|
||||
borderRadius: 'md',
|
||||
cursor: 'pointer',
|
||||
bg: vision.calibrationMode === 'manual' ? 'blue.600' : 'gray.700',
|
||||
color: 'white',
|
||||
_hover: { bg: vision.calibrationMode === 'manual' ? 'blue.500' : 'gray.600' },
|
||||
})}
|
||||
>
|
||||
Manual
|
||||
</button>
|
||||
</div>
|
||||
<span className={css({ color: 'gray.400', fontSize: 'sm' })}>Mode:</span>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => vision.setCalibrationMode('auto')}
|
||||
className={css({
|
||||
px: 3,
|
||||
py: 1,
|
||||
fontSize: 'sm',
|
||||
border: 'none',
|
||||
borderRadius: 'md',
|
||||
cursor: 'pointer',
|
||||
bg: vision.calibrationMode === 'auto' ? 'blue.600' : 'gray.700',
|
||||
color: 'white',
|
||||
_hover: { bg: vision.calibrationMode === 'auto' ? 'blue.500' : 'gray.600' },
|
||||
})}
|
||||
>
|
||||
Auto (Markers)
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => vision.setCalibrationMode('manual')}
|
||||
className={css({
|
||||
px: 3,
|
||||
py: 1,
|
||||
fontSize: 'sm',
|
||||
border: 'none',
|
||||
borderRadius: 'md',
|
||||
cursor: 'pointer',
|
||||
bg: vision.calibrationMode === 'manual' ? 'blue.600' : 'gray.700',
|
||||
color: 'white',
|
||||
_hover: { bg: vision.calibrationMode === 'manual' ? 'blue.500' : 'gray.600' },
|
||||
})}
|
||||
>
|
||||
Manual
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Marker detection status (in auto mode) */}
|
||||
{vision.calibrationMode === 'auto' && (
|
||||
{/* Marker detection status (in auto mode, local camera only) */}
|
||||
{cameraSource === 'local' && vision.calibrationMode === 'auto' && (
|
||||
<div
|
||||
data-element="marker-status"
|
||||
className={css({
|
||||
|
|
@ -340,58 +431,69 @@ export function AbacusVisionBridge({
|
|||
|
||||
{/* Camera feed */}
|
||||
<div ref={cameraFeedContainerRef} className={css({ position: 'relative' })}>
|
||||
<VisionCameraFeed
|
||||
videoStream={vision.videoStream}
|
||||
isLoading={vision.isCameraLoading}
|
||||
calibration={vision.calibrationGrid}
|
||||
showCalibrationGrid={!vision.isCalibrating && !vision.isCalibrated}
|
||||
showRectifiedView={vision.isCalibrated && !vision.isCalibrating}
|
||||
videoRef={(el) => {
|
||||
videoRef.current = el
|
||||
}}
|
||||
onVideoReady={handleVideoReady}
|
||||
>
|
||||
{/* Calibration overlay when calibrating */}
|
||||
{vision.isCalibrating && videoDimensions && (
|
||||
<CalibrationOverlay
|
||||
columnCount={columnCount}
|
||||
videoWidth={videoDimensions.width}
|
||||
videoHeight={videoDimensions.height}
|
||||
containerWidth={videoDimensions.containerWidth}
|
||||
containerHeight={videoDimensions.containerHeight}
|
||||
initialCalibration={vision.calibrationGrid}
|
||||
onComplete={handleCalibrationComplete}
|
||||
onCancel={vision.cancelCalibration}
|
||||
videoElement={videoRef.current}
|
||||
onCornersChange={setCalibrationCorners}
|
||||
/>
|
||||
)}
|
||||
</VisionCameraFeed>
|
||||
{cameraSource === 'local' ? (
|
||||
<>
|
||||
<VisionCameraFeed
|
||||
videoStream={vision.videoStream}
|
||||
isLoading={vision.isCameraLoading}
|
||||
calibration={vision.calibrationGrid}
|
||||
showCalibrationGrid={!vision.isCalibrating && !vision.isCalibrated}
|
||||
showRectifiedView={vision.isCalibrated && !vision.isCalibrating}
|
||||
videoRef={(el) => {
|
||||
videoRef.current = el
|
||||
}}
|
||||
onVideoReady={handleVideoReady}
|
||||
>
|
||||
{/* Calibration overlay when calibrating */}
|
||||
{vision.isCalibrating && videoDimensions && (
|
||||
<CalibrationOverlay
|
||||
columnCount={columnCount}
|
||||
videoWidth={videoDimensions.width}
|
||||
videoHeight={videoDimensions.height}
|
||||
containerWidth={videoDimensions.containerWidth}
|
||||
containerHeight={videoDimensions.containerHeight}
|
||||
initialCalibration={vision.calibrationGrid}
|
||||
onComplete={handleCalibrationComplete}
|
||||
onCancel={vision.cancelCalibration}
|
||||
videoElement={videoRef.current}
|
||||
onCornersChange={setCalibrationCorners}
|
||||
/>
|
||||
)}
|
||||
</VisionCameraFeed>
|
||||
|
||||
{/* Status indicator */}
|
||||
{vision.isEnabled && !vision.isCalibrating && (
|
||||
<div
|
||||
className={css({
|
||||
position: 'absolute',
|
||||
top: 2,
|
||||
left: 2,
|
||||
})}
|
||||
>
|
||||
<VisionStatusIndicator
|
||||
isCalibrated={vision.isCalibrated}
|
||||
isDetecting={vision.isDetecting}
|
||||
confidence={vision.confidence}
|
||||
handDetected={vision.isHandDetected}
|
||||
detectedValue={vision.currentDetectedValue}
|
||||
consecutiveFrames={vision.consecutiveFrames}
|
||||
minFrames={DEFAULT_STABILITY_CONFIG.minConsecutiveFrames}
|
||||
/>
|
||||
</div>
|
||||
{/* Status indicator */}
|
||||
{vision.isEnabled && !vision.isCalibrating && (
|
||||
<div
|
||||
className={css({
|
||||
position: 'absolute',
|
||||
top: 2,
|
||||
left: 2,
|
||||
})}
|
||||
>
|
||||
<VisionStatusIndicator
|
||||
isCalibrated={vision.isCalibrated}
|
||||
isDetecting={vision.isDetecting}
|
||||
confidence={vision.confidence}
|
||||
handDetected={vision.isHandDetected}
|
||||
detectedValue={vision.currentDetectedValue}
|
||||
consecutiveFrames={vision.consecutiveFrames}
|
||||
minFrames={DEFAULT_STABILITY_CONFIG.minConsecutiveFrames}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
</>
|
||||
) : (
|
||||
/* Phone camera - show QR code and receive frames */
|
||||
<RemoteCameraReceiver
|
||||
sessionId={remoteCameraSessionId}
|
||||
onFrame={handleRemoteFrame}
|
||||
onSessionCreated={handleRemoteSessionCreated}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Rectified preview during calibration */}
|
||||
{vision.isCalibrating && (
|
||||
{/* Rectified preview during calibration (local camera only) */}
|
||||
{cameraSource === 'local' && vision.isCalibrating && (
|
||||
<div
|
||||
data-element="calibration-preview"
|
||||
className={css({
|
||||
|
|
@ -416,8 +518,8 @@ export function AbacusVisionBridge({
|
|||
</div>
|
||||
)}
|
||||
|
||||
{/* Actions (manual mode only) */}
|
||||
{vision.calibrationMode === 'manual' && (
|
||||
{/* Actions (manual mode, local camera only) */}
|
||||
{cameraSource === 'local' && vision.calibrationMode === 'manual' && (
|
||||
<div
|
||||
data-element="actions"
|
||||
className={css({
|
||||
|
|
@ -485,7 +587,7 @@ export function AbacusVisionBridge({
|
|||
)}
|
||||
|
||||
{/* Instructions */}
|
||||
{!vision.isCalibrated && !vision.isCalibrating && (
|
||||
{cameraSource === 'local' && !vision.isCalibrated && !vision.isCalibrating && (
|
||||
<p
|
||||
className={css({
|
||||
color: 'gray.400',
|
||||
|
|
@ -499,8 +601,20 @@ export function AbacusVisionBridge({
|
|||
</p>
|
||||
)}
|
||||
|
||||
{cameraSource === 'phone' && !remoteCameraSessionId && (
|
||||
<p
|
||||
className={css({
|
||||
color: 'gray.400',
|
||||
fontSize: 'sm',
|
||||
textAlign: 'center',
|
||||
})}
|
||||
>
|
||||
Scan the QR code with your phone to use it as a remote camera
|
||||
</p>
|
||||
)}
|
||||
|
||||
{/* Error display */}
|
||||
{vision.cameraError && (
|
||||
{cameraSource === 'local' && vision.cameraError && (
|
||||
<div
|
||||
className={css({
|
||||
p: 3,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,173 @@
|
|||
'use client'
|
||||
|
||||
import { useEffect } from 'react'
|
||||
import { AbacusQRCode } from '@/components/common/AbacusQRCode'
|
||||
import { useRemoteCameraSession } from '@/hooks/useRemoteCameraSession'
|
||||
import { css } from '../../../styled-system/css'
|
||||
|
||||
export interface RemoteCameraQRCodeProps {
|
||||
/** Called when a session is created with the session ID */
|
||||
onSessionCreated?: (sessionId: string) => void
|
||||
/** Size of the QR code in pixels */
|
||||
size?: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Displays a QR code for phone camera connection
|
||||
*
|
||||
* Automatically creates a remote camera session and shows a QR code
|
||||
* that phones can scan to connect as a remote camera source.
|
||||
*/
|
||||
export function RemoteCameraQRCode({ onSessionCreated, size = 200 }: RemoteCameraQRCodeProps) {
|
||||
const { session, isCreating, error, createSession, getPhoneUrl } = useRemoteCameraSession()
|
||||
|
||||
// Create session on mount
|
||||
useEffect(() => {
|
||||
if (!session && !isCreating) {
|
||||
createSession().then((newSession) => {
|
||||
if (newSession && onSessionCreated) {
|
||||
onSessionCreated(newSession.sessionId)
|
||||
}
|
||||
})
|
||||
}
|
||||
}, [session, isCreating, createSession, onSessionCreated])
|
||||
|
||||
const phoneUrl = getPhoneUrl()
|
||||
|
||||
if (isCreating) {
|
||||
return (
|
||||
<div
|
||||
className={css({
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
gap: 3,
|
||||
p: 4,
|
||||
})}
|
||||
data-component="remote-camera-qr-loading"
|
||||
>
|
||||
<div
|
||||
className={css({
|
||||
width: `${size}px`,
|
||||
height: `${size}px`,
|
||||
bg: 'gray.100',
|
||||
borderRadius: 'lg',
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
})}
|
||||
>
|
||||
<span className={css({ color: 'gray.500', fontSize: 'sm' })}>Creating session...</span>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
if (error) {
|
||||
return (
|
||||
<div
|
||||
className={css({
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
gap: 3,
|
||||
p: 4,
|
||||
})}
|
||||
data-component="remote-camera-qr-error"
|
||||
>
|
||||
<div
|
||||
className={css({
|
||||
width: `${size}px`,
|
||||
height: `${size}px`,
|
||||
bg: 'red.50',
|
||||
borderRadius: 'lg',
|
||||
border: '1px solid',
|
||||
borderColor: 'red.200',
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
p: 4,
|
||||
textAlign: 'center',
|
||||
})}
|
||||
>
|
||||
<span className={css({ color: 'red.600', fontSize: 'sm' })}>{error}</span>
|
||||
</div>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => createSession()}
|
||||
className={css({
|
||||
px: 4,
|
||||
py: 2,
|
||||
bg: 'blue.600',
|
||||
color: 'white',
|
||||
borderRadius: 'lg',
|
||||
fontWeight: 'medium',
|
||||
cursor: 'pointer',
|
||||
border: 'none',
|
||||
_hover: { bg: 'blue.700' },
|
||||
})}
|
||||
>
|
||||
Retry
|
||||
</button>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
if (!session || !phoneUrl) {
|
||||
return null
|
||||
}
|
||||
|
||||
return (
|
||||
<div
|
||||
className={css({
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
alignItems: 'center',
|
||||
gap: 4,
|
||||
})}
|
||||
data-component="remote-camera-qr"
|
||||
>
|
||||
{/* QR Code */}
|
||||
<div
|
||||
className={css({
|
||||
bg: 'white',
|
||||
p: 4,
|
||||
borderRadius: 'xl',
|
||||
shadow: 'md',
|
||||
})}
|
||||
>
|
||||
<AbacusQRCode value={phoneUrl} size={size} />
|
||||
</div>
|
||||
|
||||
{/* Instructions */}
|
||||
<div className={css({ textAlign: 'center' })}>
|
||||
<p className={css({ fontSize: 'sm', color: 'gray.600', mb: 1 })}>
|
||||
Scan with your phone to use it as a camera
|
||||
</p>
|
||||
<p className={css({ fontSize: 'xs', color: 'gray.400' })}>
|
||||
Session expires in 10 minutes
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{/* URL for manual entry */}
|
||||
<div
|
||||
className={css({
|
||||
fontSize: 'xs',
|
||||
color: 'gray.500',
|
||||
bg: 'gray.100',
|
||||
px: 3,
|
||||
py: 2,
|
||||
borderRadius: 'md',
|
||||
fontFamily: 'mono',
|
||||
wordBreak: 'break-all',
|
||||
maxWidth: '280px',
|
||||
textAlign: 'center',
|
||||
})}
|
||||
>
|
||||
{phoneUrl}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
|
@ -0,0 +1,207 @@
|
|||
'use client'
|
||||
|
||||
import { useEffect, useMemo } from 'react'
|
||||
import { useRemoteCameraDesktop } from '@/hooks/useRemoteCameraDesktop'
|
||||
import { RemoteCameraQRCode } from './RemoteCameraQRCode'
|
||||
import { css } from '../../../styled-system/css'
|
||||
|
||||
export interface RemoteCameraReceiverProps {
|
||||
/** Session ID to receive frames for */
|
||||
sessionId: string | null
|
||||
/** Called when a frame is received (for processing) */
|
||||
onFrame?: (imageData: string, timestamp: number) => void
|
||||
/** Called when session ID is available (from QR code creation) */
|
||||
onSessionCreated?: (sessionId: string) => void
|
||||
/** Size of the QR code when showing (default 200) */
|
||||
qrCodeSize?: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Desktop component for receiving remote camera frames
|
||||
*
|
||||
* Shows a QR code when no phone is connected, then displays
|
||||
* the cropped abacus frames sent from the phone.
|
||||
*/
|
||||
export function RemoteCameraReceiver({
|
||||
sessionId,
|
||||
onFrame,
|
||||
onSessionCreated,
|
||||
qrCodeSize = 200,
|
||||
}: RemoteCameraReceiverProps) {
|
||||
const { isPhoneConnected, latestFrame, frameRate, error, subscribe, unsubscribe } =
|
||||
useRemoteCameraDesktop()
|
||||
|
||||
// Subscribe when sessionId changes
|
||||
useEffect(() => {
|
||||
if (sessionId) {
|
||||
subscribe(sessionId)
|
||||
return () => {
|
||||
unsubscribe()
|
||||
}
|
||||
}
|
||||
}, [sessionId, subscribe, unsubscribe])
|
||||
|
||||
// Notify parent when frame received
|
||||
useEffect(() => {
|
||||
if (latestFrame && onFrame) {
|
||||
onFrame(latestFrame.imageData, latestFrame.timestamp)
|
||||
}
|
||||
}, [latestFrame, onFrame])
|
||||
|
||||
// Create image src from base64 data
|
||||
const imageSrc = useMemo(() => {
|
||||
if (!latestFrame) return null
|
||||
return `data:image/jpeg;base64,${latestFrame.imageData}`
|
||||
}, [latestFrame])
|
||||
|
||||
// Show QR code if no session yet
|
||||
if (!sessionId) {
|
||||
return (
|
||||
<div
|
||||
className={css({
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
p: 6,
|
||||
})}
|
||||
data-component="remote-camera-receiver-qr"
|
||||
>
|
||||
<h3 className={css({ fontSize: 'lg', fontWeight: 'semibold', mb: 4 })}>
|
||||
Connect Phone Camera
|
||||
</h3>
|
||||
<RemoteCameraQRCode onSessionCreated={onSessionCreated} size={qrCodeSize} />
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
// Show error if any
|
||||
if (error) {
|
||||
return (
|
||||
<div
|
||||
className={css({
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
p: 6,
|
||||
bg: 'red.50',
|
||||
borderRadius: 'lg',
|
||||
border: '1px solid',
|
||||
borderColor: 'red.200',
|
||||
})}
|
||||
data-component="remote-camera-receiver-error"
|
||||
>
|
||||
<span className={css({ color: 'red.600', fontSize: 'sm' })}>{error}</span>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
// Show waiting for phone
|
||||
if (!isPhoneConnected) {
|
||||
return (
|
||||
<div
|
||||
className={css({
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
p: 6,
|
||||
})}
|
||||
data-component="remote-camera-receiver-waiting"
|
||||
>
|
||||
<h3 className={css({ fontSize: 'lg', fontWeight: 'semibold', mb: 4 })}>
|
||||
Waiting for Phone...
|
||||
</h3>
|
||||
<p className={css({ color: 'gray.500', fontSize: 'sm', mb: 4, textAlign: 'center' })}>
|
||||
Scan the QR code with your phone to connect
|
||||
</p>
|
||||
<RemoteCameraQRCode onSessionCreated={onSessionCreated} size={qrCodeSize} />
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
// Show received frame
|
||||
return (
|
||||
<div
|
||||
className={css({
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
alignItems: 'center',
|
||||
position: 'relative',
|
||||
})}
|
||||
data-component="remote-camera-receiver-connected"
|
||||
>
|
||||
{/* Frame display */}
|
||||
<div
|
||||
className={css({
|
||||
position: 'relative',
|
||||
width: '100%',
|
||||
bg: 'gray.900',
|
||||
borderRadius: 'lg',
|
||||
overflow: 'hidden',
|
||||
})}
|
||||
>
|
||||
{imageSrc ? (
|
||||
<img
|
||||
src={imageSrc}
|
||||
alt="Remote camera view"
|
||||
className={css({
|
||||
width: '100%',
|
||||
height: 'auto',
|
||||
display: 'block',
|
||||
})}
|
||||
/>
|
||||
) : (
|
||||
<div
|
||||
className={css({
|
||||
width: '100%',
|
||||
aspectRatio: '2/1',
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
color: 'gray.400',
|
||||
})}
|
||||
>
|
||||
Waiting for frames...
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Status bar */}
|
||||
<div
|
||||
className={css({
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
gap: 4,
|
||||
mt: 2,
|
||||
px: 2,
|
||||
py: 1,
|
||||
bg: 'gray.100',
|
||||
borderRadius: 'md',
|
||||
fontSize: 'xs',
|
||||
color: 'gray.600',
|
||||
})}
|
||||
>
|
||||
<span
|
||||
className={css({
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
gap: 1,
|
||||
})}
|
||||
>
|
||||
<span
|
||||
className={css({
|
||||
width: 2,
|
||||
height: 2,
|
||||
borderRadius: 'full',
|
||||
bg: isPhoneConnected ? 'green.500' : 'red.500',
|
||||
})}
|
||||
/>
|
||||
{isPhoneConnected ? 'Connected' : 'Disconnected'}
|
||||
</span>
|
||||
<span>{frameRate} fps</span>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
|
@ -0,0 +1,165 @@
|
|||
'use client'
|
||||
|
||||
import { useCallback, useEffect, useRef, useState } from 'react'
|
||||
import { io, type Socket } from 'socket.io-client'
|
||||
|
||||
interface RemoteCameraFrame {
|
||||
imageData: string // Base64 JPEG
|
||||
timestamp: number
|
||||
}
|
||||
|
||||
interface UseRemoteCameraDesktopReturn {
|
||||
/** Whether the phone is connected */
|
||||
isPhoneConnected: boolean
|
||||
/** Latest frame from the phone */
|
||||
latestFrame: RemoteCameraFrame | null
|
||||
/** Frame rate (frames per second) */
|
||||
frameRate: number
|
||||
/** Error message if connection failed */
|
||||
error: string | null
|
||||
/** Subscribe to receive frames for a session */
|
||||
subscribe: (sessionId: string) => void
|
||||
/** Unsubscribe from the session */
|
||||
unsubscribe: () => void
|
||||
}
|
||||
|
||||
/**
|
||||
* Hook for receiving remote camera frames on the desktop
|
||||
*
|
||||
* Subscribes to a remote camera session via Socket.IO and receives
|
||||
* cropped abacus images from the connected phone.
|
||||
*/
|
||||
export function useRemoteCameraDesktop(): UseRemoteCameraDesktopReturn {
|
||||
const [socket, setSocket] = useState<Socket | null>(null)
|
||||
const [isConnected, setIsConnected] = useState(false)
|
||||
const [isPhoneConnected, setIsPhoneConnected] = useState(false)
|
||||
const [latestFrame, setLatestFrame] = useState<RemoteCameraFrame | null>(null)
|
||||
const [frameRate, setFrameRate] = useState(0)
|
||||
const [error, setError] = useState<string | null>(null)
|
||||
const currentSessionId = useRef<string | null>(null)
|
||||
|
||||
// Frame rate calculation
|
||||
const frameTimestamps = useRef<number[]>([])
|
||||
|
||||
// Initialize socket connection
|
||||
useEffect(() => {
|
||||
const socketInstance = io({
|
||||
path: '/api/socket',
|
||||
autoConnect: true,
|
||||
})
|
||||
|
||||
socketInstance.on('connect', () => {
|
||||
setIsConnected(true)
|
||||
})
|
||||
|
||||
socketInstance.on('disconnect', () => {
|
||||
setIsConnected(false)
|
||||
})
|
||||
|
||||
setSocket(socketInstance)
|
||||
|
||||
return () => {
|
||||
socketInstance.disconnect()
|
||||
}
|
||||
}, [])
|
||||
|
||||
const calculateFrameRate = useCallback(() => {
|
||||
const now = Date.now()
|
||||
// Keep only frames from last second
|
||||
frameTimestamps.current = frameTimestamps.current.filter((t) => now - t < 1000)
|
||||
setFrameRate(frameTimestamps.current.length)
|
||||
}, [])
|
||||
|
||||
// Set up Socket.IO event listeners
|
||||
useEffect(() => {
|
||||
if (!socket) return
|
||||
|
||||
const handleConnected = ({ phoneConnected }: { phoneConnected: boolean }) => {
|
||||
setIsPhoneConnected(phoneConnected)
|
||||
setError(null)
|
||||
}
|
||||
|
||||
const handleDisconnected = ({ phoneConnected }: { phoneConnected: boolean }) => {
|
||||
setIsPhoneConnected(phoneConnected)
|
||||
setLatestFrame(null)
|
||||
setFrameRate(0)
|
||||
}
|
||||
|
||||
const handleStatus = ({ phoneConnected }: { phoneConnected: boolean }) => {
|
||||
setIsPhoneConnected(phoneConnected)
|
||||
}
|
||||
|
||||
const handleFrame = (frame: RemoteCameraFrame) => {
|
||||
setLatestFrame(frame)
|
||||
frameTimestamps.current.push(Date.now())
|
||||
calculateFrameRate()
|
||||
}
|
||||
|
||||
const handleError = ({ error: errorMsg }: { error: string }) => {
|
||||
setError(errorMsg)
|
||||
}
|
||||
|
||||
socket.on('remote-camera:connected', handleConnected)
|
||||
socket.on('remote-camera:disconnected', handleDisconnected)
|
||||
socket.on('remote-camera:status', handleStatus)
|
||||
socket.on('remote-camera:frame', handleFrame)
|
||||
socket.on('remote-camera:error', handleError)
|
||||
|
||||
return () => {
|
||||
socket.off('remote-camera:connected', handleConnected)
|
||||
socket.off('remote-camera:disconnected', handleDisconnected)
|
||||
socket.off('remote-camera:status', handleStatus)
|
||||
socket.off('remote-camera:frame', handleFrame)
|
||||
socket.off('remote-camera:error', handleError)
|
||||
}
|
||||
}, [socket, calculateFrameRate])
|
||||
|
||||
// Frame rate update interval
|
||||
useEffect(() => {
|
||||
const interval = setInterval(calculateFrameRate, 500)
|
||||
return () => clearInterval(interval)
|
||||
}, [calculateFrameRate])
|
||||
|
||||
const subscribe = useCallback(
|
||||
(sessionId: string) => {
|
||||
if (!socket || !isConnected) {
|
||||
setError('Socket not connected')
|
||||
return
|
||||
}
|
||||
|
||||
currentSessionId.current = sessionId
|
||||
setError(null)
|
||||
socket.emit('remote-camera:subscribe', { sessionId })
|
||||
},
|
||||
[socket, isConnected]
|
||||
)
|
||||
|
||||
const unsubscribe = useCallback(() => {
|
||||
if (!socket || !currentSessionId.current) return
|
||||
|
||||
socket.emit('remote-camera:leave', { sessionId: currentSessionId.current })
|
||||
currentSessionId.current = null
|
||||
setIsPhoneConnected(false)
|
||||
setLatestFrame(null)
|
||||
setFrameRate(0)
|
||||
setError(null)
|
||||
}, [socket])
|
||||
|
||||
// Cleanup on unmount
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
if (socket && currentSessionId.current) {
|
||||
socket.emit('remote-camera:leave', { sessionId: currentSessionId.current })
|
||||
}
|
||||
}
|
||||
}, [socket])
|
||||
|
||||
return {
|
||||
isPhoneConnected,
|
||||
latestFrame,
|
||||
frameRate,
|
||||
error,
|
||||
subscribe,
|
||||
unsubscribe,
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,278 @@
|
|||
'use client'
|
||||
|
||||
import { useCallback, useEffect, useRef, useState } from 'react'
|
||||
import { io, type Socket } from 'socket.io-client'
|
||||
import {
|
||||
isOpenCVReady,
|
||||
loadOpenCV,
|
||||
rectifyQuadrilateralToBase64,
|
||||
} from '@/lib/vision/perspectiveTransform'
|
||||
import type { QuadCorners } from '@/types/vision'
|
||||
|
||||
interface UseRemoteCameraPhoneOptions {
|
||||
/** Target frame rate (default 10) */
|
||||
targetFps?: number
|
||||
/** JPEG quality (0-1, default 0.8) */
|
||||
jpegQuality?: number
|
||||
/** Target width for cropped image (default 400) */
|
||||
targetWidth?: number
|
||||
}
|
||||
|
||||
interface UseRemoteCameraPhoneReturn {
|
||||
/** Whether connected to the session */
|
||||
isConnected: boolean
|
||||
/** Whether currently sending frames */
|
||||
isSending: boolean
|
||||
/** Error message if any */
|
||||
error: string | null
|
||||
/** Connect to a session */
|
||||
connect: (sessionId: string) => void
|
||||
/** Disconnect from session */
|
||||
disconnect: () => void
|
||||
/** Start sending frames with current calibration */
|
||||
startSending: (video: HTMLVideoElement, calibration: QuadCorners) => void
|
||||
/** Stop sending frames */
|
||||
stopSending: () => void
|
||||
/** Update calibration while sending */
|
||||
updateCalibration: (calibration: QuadCorners) => void
|
||||
}
|
||||
|
||||
/**
|
||||
* Hook for sending remote camera frames from phone
|
||||
*
|
||||
* Handles connecting to a session, capturing video frames,
|
||||
* applying perspective crop, and sending to the desktop.
|
||||
*/
|
||||
export function useRemoteCameraPhone(
|
||||
options: UseRemoteCameraPhoneOptions = {}
|
||||
): UseRemoteCameraPhoneReturn {
|
||||
const { targetFps = 10, jpegQuality = 0.8, targetWidth = 400 } = options
|
||||
|
||||
const [isSocketConnected, setIsSocketConnected] = useState(false)
|
||||
const [isConnected, setIsConnected] = useState(false)
|
||||
const [isSending, setIsSending] = useState(false)
|
||||
const [error, setError] = useState<string | null>(null)
|
||||
const [opencvReady, setOpencvReady] = useState(false)
|
||||
|
||||
// Use refs for values that need to be accessed in animation loop
|
||||
// to avoid stale closure issues
|
||||
const socketRef = useRef<Socket | null>(null)
|
||||
const isConnectedRef = useRef(false)
|
||||
const opencvReadyRef = useRef(false)
|
||||
const sessionIdRef = useRef<string | null>(null)
|
||||
const videoRef = useRef<HTMLVideoElement | null>(null)
|
||||
const calibrationRef = useRef<QuadCorners | null>(null)
|
||||
const animationFrameRef = useRef<number | null>(null)
|
||||
const lastFrameTimeRef = useRef(0)
|
||||
|
||||
// Keep refs in sync with state
|
||||
useEffect(() => {
|
||||
isConnectedRef.current = isConnected
|
||||
}, [isConnected])
|
||||
|
||||
useEffect(() => {
|
||||
opencvReadyRef.current = opencvReady
|
||||
}, [opencvReady])
|
||||
|
||||
// Initialize socket connection
|
||||
useEffect(() => {
|
||||
const socketInstance = io({
|
||||
path: '/api/socket',
|
||||
autoConnect: true,
|
||||
})
|
||||
|
||||
socketInstance.on('connect', () => {
|
||||
setIsSocketConnected(true)
|
||||
})
|
||||
|
||||
socketInstance.on('disconnect', () => {
|
||||
setIsSocketConnected(false)
|
||||
setIsConnected(false)
|
||||
isConnectedRef.current = false
|
||||
})
|
||||
|
||||
socketRef.current = socketInstance
|
||||
|
||||
return () => {
|
||||
socketInstance.disconnect()
|
||||
}
|
||||
}, [])
|
||||
|
||||
// Load OpenCV on mount
|
||||
useEffect(() => {
|
||||
loadOpenCV()
|
||||
.then(() => {
|
||||
setOpencvReady(true)
|
||||
})
|
||||
.catch((err) => {
|
||||
console.error('[RemoteCameraPhone] Failed to load OpenCV:', err)
|
||||
setError('Failed to load image processing library')
|
||||
})
|
||||
}, [])
|
||||
|
||||
// Set up Socket.IO event listeners
|
||||
useEffect(() => {
|
||||
const socket = socketRef.current
|
||||
if (!socket) return
|
||||
|
||||
const handleError = ({ error: errorMsg }: { error: string }) => {
|
||||
setError(errorMsg)
|
||||
setIsConnected(false)
|
||||
isConnectedRef.current = false
|
||||
}
|
||||
|
||||
socket.on('remote-camera:error', handleError)
|
||||
|
||||
return () => {
|
||||
socket.off('remote-camera:error', handleError)
|
||||
}
|
||||
}, [isSocketConnected]) // Re-run when socket connects
|
||||
|
||||
/**
|
||||
* Apply perspective transform and extract the quadrilateral region
|
||||
* Uses OpenCV for proper perspective correction
|
||||
*/
|
||||
const cropToQuad = useCallback(
|
||||
(video: HTMLVideoElement, quad: QuadCorners): string | null => {
|
||||
if (!isOpenCVReady()) {
|
||||
console.warn('[RemoteCameraPhone] OpenCV not ready for perspective transform')
|
||||
return null
|
||||
}
|
||||
|
||||
return rectifyQuadrilateralToBase64(video, quad, {
|
||||
outputWidth: targetWidth,
|
||||
jpegQuality,
|
||||
})
|
||||
},
|
||||
[targetWidth, jpegQuality]
|
||||
)
|
||||
|
||||
/**
|
||||
* Frame capture loop - uses refs to avoid stale closure issues
|
||||
*/
|
||||
const captureFrame = useCallback(() => {
|
||||
const video = videoRef.current
|
||||
const calibration = calibrationRef.current
|
||||
const sessionId = sessionIdRef.current
|
||||
const socket = socketRef.current
|
||||
const isConnected = isConnectedRef.current
|
||||
const cvReady = opencvReadyRef.current
|
||||
|
||||
if (!video || !calibration || !sessionId || !socket || !isConnected || !cvReady) {
|
||||
animationFrameRef.current = requestAnimationFrame(captureFrame)
|
||||
return
|
||||
}
|
||||
|
||||
const now = performance.now()
|
||||
const frameInterval = 1000 / targetFps
|
||||
|
||||
if (now - lastFrameTimeRef.current < frameInterval) {
|
||||
animationFrameRef.current = requestAnimationFrame(captureFrame)
|
||||
return
|
||||
}
|
||||
|
||||
lastFrameTimeRef.current = now
|
||||
|
||||
// Crop and encode frame
|
||||
const imageData = cropToQuad(video, calibration)
|
||||
if (imageData) {
|
||||
socket.emit('remote-camera:frame', {
|
||||
sessionId,
|
||||
imageData,
|
||||
timestamp: Date.now(),
|
||||
})
|
||||
}
|
||||
|
||||
animationFrameRef.current = requestAnimationFrame(captureFrame)
|
||||
}, [targetFps, cropToQuad])
|
||||
|
||||
const connect = useCallback(
|
||||
(sessionId: string) => {
|
||||
const socket = socketRef.current
|
||||
if (!socket || !isSocketConnected) {
|
||||
setError('Socket not connected')
|
||||
return
|
||||
}
|
||||
|
||||
sessionIdRef.current = sessionId
|
||||
setError(null)
|
||||
|
||||
socket.emit('remote-camera:join', { sessionId })
|
||||
setIsConnected(true)
|
||||
isConnectedRef.current = true
|
||||
},
|
||||
[isSocketConnected]
|
||||
)
|
||||
|
||||
const disconnect = useCallback(() => {
|
||||
const socket = socketRef.current
|
||||
if (!socket || !sessionIdRef.current) return
|
||||
|
||||
// Stop sending first
|
||||
if (animationFrameRef.current) {
|
||||
cancelAnimationFrame(animationFrameRef.current)
|
||||
animationFrameRef.current = null
|
||||
}
|
||||
setIsSending(false)
|
||||
|
||||
socket.emit('remote-camera:leave', { sessionId: sessionIdRef.current })
|
||||
sessionIdRef.current = null
|
||||
setIsConnected(false)
|
||||
isConnectedRef.current = false
|
||||
videoRef.current = null
|
||||
calibrationRef.current = null
|
||||
}, [])
|
||||
|
||||
const startSending = useCallback(
|
||||
(video: HTMLVideoElement, calibration: QuadCorners) => {
|
||||
if (!isConnected) {
|
||||
setError('Not connected to session')
|
||||
return
|
||||
}
|
||||
|
||||
videoRef.current = video
|
||||
calibrationRef.current = calibration
|
||||
setIsSending(true)
|
||||
|
||||
// Start capture loop
|
||||
animationFrameRef.current = requestAnimationFrame(captureFrame)
|
||||
},
|
||||
[isConnected, captureFrame]
|
||||
)
|
||||
|
||||
const stopSending = useCallback(() => {
|
||||
if (animationFrameRef.current) {
|
||||
cancelAnimationFrame(animationFrameRef.current)
|
||||
animationFrameRef.current = null
|
||||
}
|
||||
setIsSending(false)
|
||||
}, [])
|
||||
|
||||
const updateCalibration = useCallback((calibration: QuadCorners) => {
|
||||
calibrationRef.current = calibration
|
||||
}, [])
|
||||
|
||||
// Cleanup on unmount
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
if (animationFrameRef.current) {
|
||||
cancelAnimationFrame(animationFrameRef.current)
|
||||
}
|
||||
const socket = socketRef.current
|
||||
if (socket && sessionIdRef.current) {
|
||||
socket.emit('remote-camera:leave', { sessionId: sessionIdRef.current })
|
||||
}
|
||||
}
|
||||
}, [])
|
||||
|
||||
return {
|
||||
isConnected,
|
||||
isSending,
|
||||
error,
|
||||
connect,
|
||||
disconnect,
|
||||
startSending,
|
||||
stopSending,
|
||||
updateCalibration,
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,118 @@
|
|||
'use client'
|
||||
|
||||
import { useCallback, useState } from 'react'
|
||||
|
||||
interface RemoteCameraSession {
|
||||
sessionId: string
|
||||
expiresAt: string
|
||||
phoneConnected?: boolean
|
||||
}
|
||||
|
||||
interface UseRemoteCameraSessionReturn {
|
||||
/** Current session data */
|
||||
session: RemoteCameraSession | null
|
||||
/** Whether a session is being created */
|
||||
isCreating: boolean
|
||||
/** Error message if session creation failed */
|
||||
error: string | null
|
||||
/** Create a new remote camera session */
|
||||
createSession: () => Promise<RemoteCameraSession | null>
|
||||
/** Clear the current session */
|
||||
clearSession: () => void
|
||||
/** Get the URL for the phone to scan */
|
||||
getPhoneUrl: () => string | null
|
||||
}
|
||||
|
||||
/**
|
||||
* Hook for managing remote camera sessions
|
||||
*
|
||||
* Used by the desktop to create sessions and generate QR codes
|
||||
* for phones to scan.
|
||||
*/
|
||||
export function useRemoteCameraSession(): UseRemoteCameraSessionReturn {
|
||||
const [session, setSession] = useState<RemoteCameraSession | null>(null)
|
||||
const [isCreating, setIsCreating] = useState(false)
|
||||
const [error, setError] = useState<string | null>(null)
|
||||
|
||||
const createSession = useCallback(async (): Promise<RemoteCameraSession | null> => {
|
||||
setIsCreating(true)
|
||||
setError(null)
|
||||
|
||||
try {
|
||||
const response = await fetch('/api/remote-camera', {
|
||||
method: 'POST',
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const data = await response.json()
|
||||
throw new Error(data.error || 'Failed to create session')
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
const newSession: RemoteCameraSession = {
|
||||
sessionId: data.sessionId,
|
||||
expiresAt: data.expiresAt,
|
||||
phoneConnected: false,
|
||||
}
|
||||
|
||||
setSession(newSession)
|
||||
return newSession
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : 'Failed to create session'
|
||||
setError(message)
|
||||
console.error('Failed to create remote camera session:', err)
|
||||
return null
|
||||
} finally {
|
||||
setIsCreating(false)
|
||||
}
|
||||
}, [])
|
||||
|
||||
const clearSession = useCallback(() => {
|
||||
setSession(null)
|
||||
setError(null)
|
||||
}, [])
|
||||
|
||||
const getPhoneUrl = useCallback((): string | null => {
|
||||
if (!session) return null
|
||||
|
||||
// Get the base URL - prefer LAN host for phone access
|
||||
if (typeof window === 'undefined') return null
|
||||
|
||||
// Use NEXT_PUBLIC_LAN_HOST if set, otherwise construct from current location
|
||||
// This allows phones on the same network to reach the server
|
||||
let baseUrl: string
|
||||
|
||||
const lanHost = process.env.NEXT_PUBLIC_LAN_HOST
|
||||
if (lanHost) {
|
||||
// If just hostname/IP provided, use current protocol and port
|
||||
if (lanHost.includes('://')) {
|
||||
baseUrl = lanHost
|
||||
} else {
|
||||
const port = window.location.port ? `:${window.location.port}` : ''
|
||||
baseUrl = `${window.location.protocol}//${lanHost}${port}`
|
||||
}
|
||||
} else {
|
||||
// Fallback: replace localhost with local IP if possible
|
||||
const hostname = window.location.hostname
|
||||
if (hostname === 'localhost' || hostname === '127.0.0.1') {
|
||||
// Try to use the page's own hostname as-is (might already be LAN address)
|
||||
// For true localhost, user needs to set NEXT_PUBLIC_LAN_HOST
|
||||
baseUrl = window.location.origin
|
||||
} else {
|
||||
// Already using a non-localhost address (e.g., LAN IP, domain)
|
||||
baseUrl = window.location.origin
|
||||
}
|
||||
}
|
||||
|
||||
return `${baseUrl}/remote-camera/${session.sessionId}`
|
||||
}, [session])
|
||||
|
||||
return {
|
||||
session,
|
||||
isCreating,
|
||||
error,
|
||||
createSession,
|
||||
clearSession,
|
||||
getPhoneUrl,
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,127 @@
|
|||
/**
|
||||
* Remote Camera Session Manager
|
||||
*
|
||||
* Manages in-memory sessions for phone-to-desktop camera streaming.
|
||||
* Sessions are short-lived (10 minute TTL) and stored in memory.
|
||||
*/
|
||||
|
||||
import { createId } from '@paralleldrive/cuid2'
|
||||
|
||||
export interface RemoteCameraSession {
|
||||
id: string
|
||||
createdAt: Date
|
||||
expiresAt: Date
|
||||
phoneConnected: boolean
|
||||
}
|
||||
|
||||
// In-memory session storage
|
||||
// Using globalThis to persist across hot reloads in development
|
||||
declare global {
|
||||
// eslint-disable-next-line no-var
|
||||
var __remoteCameraSessions: Map<string, RemoteCameraSession> | undefined
|
||||
}
|
||||
|
||||
const SESSION_TTL_MS = 10 * 60 * 1000 // 10 minutes
|
||||
const CLEANUP_INTERVAL_MS = 60 * 1000 // 1 minute
|
||||
|
||||
function getSessions(): Map<string, RemoteCameraSession> {
|
||||
if (!globalThis.__remoteCameraSessions) {
|
||||
globalThis.__remoteCameraSessions = new Map()
|
||||
// Start cleanup interval
|
||||
setInterval(cleanupExpiredSessions, CLEANUP_INTERVAL_MS)
|
||||
}
|
||||
return globalThis.__remoteCameraSessions
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new remote camera session
|
||||
*/
|
||||
export function createRemoteCameraSession(): RemoteCameraSession {
|
||||
const sessions = getSessions()
|
||||
const now = new Date()
|
||||
|
||||
const session: RemoteCameraSession = {
|
||||
id: createId(),
|
||||
createdAt: now,
|
||||
expiresAt: new Date(now.getTime() + SESSION_TTL_MS),
|
||||
phoneConnected: false,
|
||||
}
|
||||
|
||||
sessions.set(session.id, session)
|
||||
return session
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a session by ID
|
||||
*/
|
||||
export function getRemoteCameraSession(sessionId: string): RemoteCameraSession | null {
|
||||
const sessions = getSessions()
|
||||
const session = sessions.get(sessionId)
|
||||
|
||||
if (!session) return null
|
||||
|
||||
// Check if expired
|
||||
if (new Date() > session.expiresAt) {
|
||||
sessions.delete(sessionId)
|
||||
return null
|
||||
}
|
||||
|
||||
return session
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark a session as having a connected phone
|
||||
*/
|
||||
export function markPhoneConnected(sessionId: string): boolean {
|
||||
const sessions = getSessions()
|
||||
const session = sessions.get(sessionId)
|
||||
|
||||
if (!session) return false
|
||||
|
||||
session.phoneConnected = true
|
||||
// Extend TTL when phone connects
|
||||
session.expiresAt = new Date(Date.now() + SESSION_TTL_MS)
|
||||
return true
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark a session as having the phone disconnected
|
||||
*/
|
||||
export function markPhoneDisconnected(sessionId: string): boolean {
|
||||
const sessions = getSessions()
|
||||
const session = sessions.get(sessionId)
|
||||
|
||||
if (!session) return false
|
||||
|
||||
session.phoneConnected = false
|
||||
return true
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a session
|
||||
*/
|
||||
export function deleteRemoteCameraSession(sessionId: string): boolean {
|
||||
const sessions = getSessions()
|
||||
return sessions.delete(sessionId)
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up expired sessions
|
||||
*/
|
||||
function cleanupExpiredSessions(): void {
|
||||
const sessions = getSessions()
|
||||
const now = new Date()
|
||||
|
||||
for (const [id, session] of sessions) {
|
||||
if (now > session.expiresAt) {
|
||||
sessions.delete(id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get session count (for debugging)
|
||||
*/
|
||||
export function getSessionCount(): number {
|
||||
return getSessions().size
|
||||
}
|
||||
|
|
@ -257,6 +257,43 @@ export function rectifyQuadrilateral(
|
|||
}
|
||||
}
|
||||
|
||||
export interface RectifyToBase64Options extends RectifyOptions {
|
||||
/** JPEG quality (0-1, default 0.8) */
|
||||
jpegQuality?: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Rectify a quadrilateral region from video to a base64-encoded JPEG
|
||||
*
|
||||
* This is the function to use for remote camera streaming - it returns
|
||||
* a base64 string ready to send over the network.
|
||||
*
|
||||
* @param video - Source video element
|
||||
* @param corners - Quadrilateral corners in video coordinates
|
||||
* @param options - Output size and quality options
|
||||
* @returns Base64-encoded JPEG (without data URL prefix), or null on failure
|
||||
*/
|
||||
export function rectifyQuadrilateralToBase64(
|
||||
video: HTMLVideoElement,
|
||||
corners: QuadCorners,
|
||||
options: RectifyToBase64Options = {}
|
||||
): string | null {
|
||||
const { jpegQuality = 0.8, ...rectifyOptions } = options
|
||||
|
||||
// Create a temporary canvas
|
||||
const canvas = document.createElement('canvas')
|
||||
|
||||
// Use the existing rectifyQuadrilateral function
|
||||
const success = rectifyQuadrilateral(video, corners, canvas, rectifyOptions)
|
||||
if (!success) {
|
||||
return null
|
||||
}
|
||||
|
||||
// Convert to JPEG and return base64 (without data URL prefix)
|
||||
const dataUrl = canvas.toDataURL('image/jpeg', jpegQuality)
|
||||
return dataUrl.split(',')[1]
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a rectified frame processor that continuously updates a canvas
|
||||
*
|
||||
|
|
|
|||
|
|
@ -19,6 +19,11 @@ import type { GameMove } from './lib/arcade/validation/types'
|
|||
import { getGameConfig } from './lib/arcade/game-config-helpers'
|
||||
import { canPerformAction, isParentOf } from './lib/classroom'
|
||||
import { incrementShareViewCount, validateSessionShare } from './lib/session-share'
|
||||
import {
|
||||
getRemoteCameraSession,
|
||||
markPhoneConnected,
|
||||
markPhoneDisconnected,
|
||||
} from './lib/remote-camera/session-manager'
|
||||
|
||||
// Yjs server-side imports
|
||||
import * as Y from 'yjs'
|
||||
|
|
@ -1073,7 +1078,130 @@ export function initializeSocketServer(httpServer: HTTPServer) {
|
|||
}
|
||||
)
|
||||
|
||||
// Remote Camera: Phone joins a remote camera session
|
||||
socket.on('remote-camera:join', async ({ sessionId }: { sessionId: string }) => {
|
||||
try {
|
||||
const session = getRemoteCameraSession(sessionId)
|
||||
if (!session) {
|
||||
socket.emit('remote-camera:error', { error: 'Invalid or expired session' })
|
||||
return
|
||||
}
|
||||
|
||||
// Mark phone as connected
|
||||
markPhoneConnected(sessionId)
|
||||
|
||||
// Join the session room
|
||||
await socket.join(`remote-camera:${sessionId}`)
|
||||
|
||||
// Store session ID on socket for cleanup on disconnect
|
||||
socket.data.remoteCameraSessionId = sessionId
|
||||
|
||||
console.log(`📱 Phone connected to remote camera session: ${sessionId}`)
|
||||
|
||||
// Notify desktop that phone is connected
|
||||
socket.to(`remote-camera:${sessionId}`).emit('remote-camera:connected', {
|
||||
phoneConnected: true,
|
||||
})
|
||||
} catch (error) {
|
||||
console.error('Error joining remote camera session:', error)
|
||||
socket.emit('remote-camera:error', { error: 'Failed to join session' })
|
||||
}
|
||||
})
|
||||
|
||||
// Remote Camera: Desktop subscribes to receive frames
|
||||
socket.on('remote-camera:subscribe', async ({ sessionId }: { sessionId: string }) => {
|
||||
try {
|
||||
const session = getRemoteCameraSession(sessionId)
|
||||
if (!session) {
|
||||
socket.emit('remote-camera:error', { error: 'Invalid or expired session' })
|
||||
return
|
||||
}
|
||||
|
||||
await socket.join(`remote-camera:${sessionId}`)
|
||||
console.log(`🖥️ Desktop subscribed to remote camera session: ${sessionId}`)
|
||||
|
||||
// Send current connection status
|
||||
socket.emit('remote-camera:status', {
|
||||
phoneConnected: session.phoneConnected,
|
||||
})
|
||||
} catch (error) {
|
||||
console.error('Error subscribing to remote camera session:', error)
|
||||
socket.emit('remote-camera:error', { error: 'Failed to subscribe' })
|
||||
}
|
||||
})
|
||||
|
||||
// Remote Camera: Phone sends cropped frame to desktop
|
||||
socket.on(
|
||||
'remote-camera:frame',
|
||||
({
|
||||
sessionId,
|
||||
imageData,
|
||||
timestamp,
|
||||
}: {
|
||||
sessionId: string
|
||||
imageData: string // Base64 JPEG
|
||||
timestamp: number
|
||||
}) => {
|
||||
// Forward frame to desktop (all other sockets in the room)
|
||||
socket.to(`remote-camera:${sessionId}`).emit('remote-camera:frame', {
|
||||
imageData,
|
||||
timestamp,
|
||||
})
|
||||
}
|
||||
)
|
||||
|
||||
// Remote Camera: Phone sends calibration data to desktop
|
||||
socket.on(
|
||||
'remote-camera:calibration',
|
||||
({
|
||||
sessionId,
|
||||
corners,
|
||||
columnCount,
|
||||
}: {
|
||||
sessionId: string
|
||||
corners: { topLeft: { x: number; y: number }; topRight: { x: number; y: number }; bottomRight: { x: number; y: number }; bottomLeft: { x: number; y: number } }
|
||||
columnCount: number
|
||||
}) => {
|
||||
// Forward calibration data to desktop
|
||||
socket.to(`remote-camera:${sessionId}`).emit('remote-camera:calibration', {
|
||||
corners,
|
||||
columnCount,
|
||||
})
|
||||
}
|
||||
)
|
||||
|
||||
// Remote Camera: Leave session
|
||||
socket.on('remote-camera:leave', async ({ sessionId }: { sessionId: string }) => {
|
||||
try {
|
||||
await socket.leave(`remote-camera:${sessionId}`)
|
||||
|
||||
// If this was the phone, mark as disconnected
|
||||
if (socket.data.remoteCameraSessionId === sessionId) {
|
||||
markPhoneDisconnected(sessionId)
|
||||
socket.data.remoteCameraSessionId = undefined
|
||||
|
||||
console.log(`📱 Phone left remote camera session: ${sessionId}`)
|
||||
|
||||
// Notify desktop
|
||||
socket.to(`remote-camera:${sessionId}`).emit('remote-camera:disconnected', {
|
||||
phoneConnected: false,
|
||||
})
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error leaving remote camera session:', error)
|
||||
}
|
||||
})
|
||||
|
||||
socket.on('disconnect', () => {
|
||||
// Handle remote camera cleanup on disconnect
|
||||
const remoteCameraSessionId = socket.data.remoteCameraSessionId as string | undefined
|
||||
if (remoteCameraSessionId) {
|
||||
markPhoneDisconnected(remoteCameraSessionId)
|
||||
io!.to(`remote-camera:${remoteCameraSessionId}`).emit('remote-camera:disconnected', {
|
||||
phoneConnected: false,
|
||||
})
|
||||
console.log(`📱 Phone disconnected from remote camera session: ${remoteCameraSessionId}`)
|
||||
}
|
||||
// Don't delete session on disconnect - it persists across devices
|
||||
})
|
||||
})
|
||||
|
|
|
|||
Loading…
Reference in New Issue