diff --git a/apps/web/src/app/api/vision-training/config.ts b/apps/web/src/app/api/vision-training/config.ts index f51e80d6..d1c83a8f 100644 --- a/apps/web/src/app/api/vision-training/config.ts +++ b/apps/web/src/app/api/vision-training/config.ts @@ -92,6 +92,57 @@ interface SetupResult { hasGpu: boolean } +/** + * Required Python modules for training + */ +const REQUIRED_MODULES = [ + { name: 'tensorflow', importName: 'tensorflow', pipName: 'tensorflow' }, + { name: 'PIL (Pillow)', importName: 'PIL', pipName: 'Pillow' }, + { name: 'sklearn (scikit-learn)', importName: 'sklearn', pipName: 'scikit-learn' }, + { name: 'numpy', importName: 'numpy', pipName: 'numpy' }, +] + +export interface DependencyCheckResult { + allInstalled: boolean + missing: { name: string; pipName: string }[] + installed: { name: string; pipName: string }[] + error?: string +} + +/** + * Check if all required Python dependencies are installed + */ +export async function checkDependencies(): Promise { + if (!fs.existsSync(TRAINING_PYTHON)) { + return { + allInstalled: false, + missing: REQUIRED_MODULES.map((m) => ({ name: m.name, pipName: m.pipName })), + installed: [], + error: 'Python virtual environment not found', + } + } + + const missing: { name: string; pipName: string }[] = [] + const installed: { name: string; pipName: string }[] = [] + + for (const mod of REQUIRED_MODULES) { + try { + await execAsync(`"${TRAINING_PYTHON}" -c "import ${mod.importName}"`, { + timeout: 10000, + }) + installed.push({ name: mod.name, pipName: mod.pipName }) + } catch { + missing.push({ name: mod.name, pipName: mod.pipName }) + } + } + + return { + allInstalled: missing.length === 0, + missing, + installed, + } +} + /** * Find the best Python to use for the venv. * On Apple Silicon, prefer Homebrew ARM Python for Metal GPU support. @@ -145,6 +196,11 @@ async function isVenvReady(): Promise { } } +/** + * Path to the requirements.txt file + */ +const REQUIREMENTS_FILE = path.join(TRAINING_SCRIPTS_DIR, 'requirements.txt') + /** * Create the venv and install dependencies */ @@ -178,6 +234,29 @@ async function createVenv(): Promise { }) } + // Install all other requirements from requirements.txt + if (fs.existsSync(REQUIREMENTS_FILE)) { + console.log('[vision-training] Installing additional requirements from requirements.txt...') + await execAsync(`"${TRAINING_PYTHON}" -m pip install -r "${REQUIREMENTS_FILE}"`, { + timeout: 600000, + }) + } else { + // Fallback: install known required packages individually + console.log( + '[vision-training] requirements.txt not found, installing packages individually...' + ) + const packages = ['Pillow', 'scikit-learn', 'numpy', 'tensorflowjs'] + for (const pkg of packages) { + try { + await execAsync(`"${TRAINING_PYTHON}" -m pip install "${pkg}"`, { + timeout: 300000, + }) + } catch (e) { + console.warn(`[vision-training] Failed to install ${pkg}: ${e}`) + } + } + } + // Verify installation const { stdout } = await execAsync( `"${TRAINING_PYTHON}" -c "import tensorflow as tf; print(len(tf.config.list_physical_devices('GPU')))"`, diff --git a/apps/web/src/app/api/vision-training/preflight/route.ts b/apps/web/src/app/api/vision-training/preflight/route.ts new file mode 100644 index 00000000..fedbf07e --- /dev/null +++ b/apps/web/src/app/api/vision-training/preflight/route.ts @@ -0,0 +1,96 @@ +import { NextResponse } from 'next/server' +import { checkDependencies, ensureVenvReady, isPlatformSupported, TRAINING_PYTHON } from '../config' + +export interface PreflightResult { + ready: boolean + platform: { + supported: boolean + reason?: string + } + venv: { + exists: boolean + python: string + isAppleSilicon: boolean + hasGpu: boolean + error?: string + } + dependencies: { + allInstalled: boolean + installed: { name: string; pipName: string }[] + missing: { name: string; pipName: string }[] + error?: string + } +} + +/** + * GET /api/vision-training/preflight + * + * Performs a complete preflight check for training readiness: + * 1. Platform support (TensorFlow availability) + * 2. Venv existence and setup + * 3. All required Python dependencies installed + * + * Returns a structured result indicating if training can proceed. + */ +export async function GET(): Promise> { + // Check platform support first + const platformCheck = isPlatformSupported() + if (!platformCheck.supported) { + return NextResponse.json({ + ready: false, + platform: platformCheck, + venv: { + exists: false, + python: TRAINING_PYTHON, + isAppleSilicon: false, + hasGpu: false, + }, + dependencies: { + allInstalled: false, + installed: [], + missing: [], + error: 'Platform not supported', + }, + }) + } + + // Check/setup venv + const venvResult = await ensureVenvReady() + + if (!venvResult.success) { + return NextResponse.json({ + ready: false, + platform: { supported: true }, + venv: { + exists: false, + python: venvResult.python, + isAppleSilicon: venvResult.isAppleSilicon, + hasGpu: venvResult.hasGpu, + error: venvResult.error, + }, + dependencies: { + allInstalled: false, + installed: [], + missing: [], + error: 'Venv setup failed', + }, + }) + } + + // Check all dependencies + const depResult = await checkDependencies() + + const ready = depResult.allInstalled + + return NextResponse.json({ + ready, + platform: { supported: true }, + venv: { + exists: true, + python: venvResult.python, + isAppleSilicon: venvResult.isAppleSilicon, + hasGpu: venvResult.hasGpu, + }, + dependencies: depResult, + }) +} diff --git a/apps/web/src/app/vision-training/train/components/TrainingDataCapture.tsx b/apps/web/src/app/vision-training/train/components/TrainingDataCapture.tsx new file mode 100644 index 00000000..d52253e3 --- /dev/null +++ b/apps/web/src/app/vision-training/train/components/TrainingDataCapture.tsx @@ -0,0 +1,281 @@ +'use client' + +import { useCallback, useRef, useState } from 'react' +import { css } from '../../../../../styled-system/css' +import { CameraCapture, type CameraSource } from '@/components/vision/CameraCapture' + +interface TrainingDataCaptureProps { + /** Called when samples are saved successfully */ + onSamplesCollected: () => void + /** Number of physical abacus columns (default 4) */ + columnCount?: number +} + +/** + * Inline training data capture component for the training wizard + * + * Uses the reusable CameraCapture component which supports both + * local device camera and phone camera via QR code. + * + * Captures abacus images and saves them as training data for the column classifier. + */ +export function TrainingDataCapture({ + onSamplesCollected, + columnCount = 4, +}: TrainingDataCaptureProps) { + // Capture state + const [inputValue, setInputValue] = useState('') + const [isCapturing, setIsCapturing] = useState(false) + const [lastCaptureStatus, setLastCaptureStatus] = useState<{ + success: boolean + message: string + } | null>(null) + const [captureCount, setCaptureCount] = useState(0) + const [isPhoneConnected, setIsPhoneConnected] = useState(false) + const [cameraSource, setCameraSource] = useState('local') + + const inputRef = useRef(null) + const captureElementRef = useRef(null) + + // Handle capture from camera + const handleCapture = useCallback((element: HTMLImageElement | HTMLVideoElement) => { + captureElementRef.current = element + }, []) + + // Capture training data + const captureTrainingData = useCallback(async () => { + const value = parseInt(inputValue, 10) + if (Number.isNaN(value) || value < 0) { + setLastCaptureStatus({ success: false, message: 'Enter a valid non-negative number' }) + return + } + + // Get the current capture element + const element = captureElementRef.current + if (!element) { + setLastCaptureStatus({ success: false, message: 'No camera frame available' }) + return + } + + // For video, check if it's playing + if (element instanceof HTMLVideoElement) { + if (element.readyState < 2) { + setLastCaptureStatus({ success: false, message: 'Camera not ready' }) + return + } + } + + // For image, check if it's loaded + if (element instanceof HTMLImageElement) { + if (!element.complete || element.naturalWidth === 0) { + setLastCaptureStatus({ success: false, message: 'Camera frame not ready' }) + return + } + } + + setIsCapturing(true) + setLastCaptureStatus(null) + + try { + // Import frame processor dynamically + const { processImageFrame } = await import('@/lib/vision/frameProcessor') + const { imageDataToBase64Png } = await import('@/lib/vision/trainingData') + + // For video elements, we need to draw to a temp image first + let imageElement: HTMLImageElement + if (element instanceof HTMLVideoElement) { + // Create a canvas to capture the video frame + const canvas = document.createElement('canvas') + canvas.width = element.videoWidth + canvas.height = element.videoHeight + const ctx = canvas.getContext('2d') + if (!ctx) { + throw new Error('Failed to create canvas context') + } + ctx.drawImage(element, 0, 0) + + // Convert canvas to image + imageElement = new Image() + imageElement.src = canvas.toDataURL('image/jpeg') + await new Promise((resolve, reject) => { + imageElement.onload = resolve + imageElement.onerror = reject + }) + } else { + imageElement = element + } + + // Slice the image into columns + const columnImages = processImageFrame(imageElement, null, columnCount) + + if (columnImages.length === 0) { + throw new Error('Failed to slice image into columns') + } + + // Convert to base64 and prepare request + const columns = columnImages.map((imgData: ImageData, index: number) => ({ + columnIndex: index, + imageData: imageDataToBase64Png(imgData), + })) + + // Send to collect API + const response = await fetch('/api/vision-training/collect', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + columns, + correctAnswer: value, + playerId: 'training-wizard', + sessionId: 'manual-capture', + }), + }) + + const result = await response.json() + + if (result.success) { + setCaptureCount((c) => c + 1) + setLastCaptureStatus({ + success: true, + message: `Saved ${result.savedCount} columns for "${value}"`, + }) + setInputValue('') + // Focus input for next capture + inputRef.current?.focus() + onSamplesCollected() + } else { + throw new Error(result.error || 'Failed to save') + } + } catch (error) { + console.error('[TrainingDataCapture] Error:', error) + setLastCaptureStatus({ + success: false, + message: error instanceof Error ? error.message : 'Failed to capture', + }) + } finally { + setIsCapturing(false) + } + }, [inputValue, columnCount, onSamplesCollected]) + + // Handle keyboard shortcut (Enter to capture) + const handleKeyDown = useCallback( + (e: React.KeyboardEvent) => { + if (e.key === 'Enter' && !isCapturing && inputValue) { + captureTrainingData() + } + }, + [captureTrainingData, isCapturing, inputValue] + ) + + // Determine if capture is possible + const canCapture = cameraSource === 'local' || isPhoneConnected + + return ( +
+ {/* Header */} +
+ ๐Ÿ“ธ + + Capture Training Data + + {captureCount > 0 && ( + + +{captureCount} this session + + )} +
+ + {/* Camera capture component */} + + + {/* Capture controls - show when camera is ready */} + {canCapture && ( +
+
+ setInputValue(e.target.value)} + onKeyDown={handleKeyDown} + disabled={isCapturing} + className={css({ + flex: 1, + px: 3, + py: 2, + bg: 'gray.700', + border: '1px solid', + borderColor: 'gray.600', + borderRadius: 'md', + color: 'gray.100', + fontSize: 'md', + fontFamily: 'mono', + _placeholder: { color: 'gray.500' }, + _focus: { outline: 'none', borderColor: 'blue.500' }, + _disabled: { opacity: 0.5 }, + })} + /> + +
+ + {/* Status message */} + {lastCaptureStatus && ( +
+ {lastCaptureStatus.success ? 'โœ“' : 'โœ—'} {lastCaptureStatus.message} +
+ )} +
+ )} + + {/* Instructions */} +
+

1. Point camera at your abacus

+

2. Set beads to show a number

+

3. Type that number and press Capture (or Enter)

+
+
+ ) +} diff --git a/apps/web/src/app/vision-training/train/components/wizard/CardCarousel.tsx b/apps/web/src/app/vision-training/train/components/wizard/CardCarousel.tsx index 4c5c22bf..3d61d1f4 100644 --- a/apps/web/src/app/vision-training/train/components/wizard/CardCarousel.tsx +++ b/apps/web/src/app/vision-training/train/components/wizard/CardCarousel.tsx @@ -8,6 +8,7 @@ import { type CardId, type SamplesData, type HardwareInfo, + type PreflightInfo, type TrainingConfig, type ServerPhase, type EpochData, @@ -24,6 +25,9 @@ interface CardCarouselProps { hardwareInfo: HardwareInfo | null hardwareLoading: boolean fetchHardware: () => void + preflightInfo: PreflightInfo | null + preflightLoading: boolean + fetchPreflight: () => void config: TrainingConfig setConfig: (config: TrainingConfig | ((prev: TrainingConfig) => TrainingConfig)) => void isGpu: boolean @@ -54,6 +58,9 @@ export function CardCarousel({ hardwareInfo, hardwareLoading, fetchHardware, + preflightInfo, + preflightLoading, + fetchPreflight, config, setConfig, isGpu, @@ -107,6 +114,21 @@ export function CardCarousel({ } return 'Detect HW' + case 'dependencies': + if (preflightInfo?.ready) { + return { + primary: 'Ready', + secondary: `${preflightInfo.dependencies.installed.length} packages`, + } + } + if (preflightInfo?.dependencies.missing.length) { + return { + primary: 'Missing', + secondary: `${preflightInfo.dependencies.missing.length} packages`, + } + } + return 'Check deps' + case 'config': return { primary: `${config.epochs} epochs`, @@ -175,6 +197,9 @@ export function CardCarousel({ hardwareInfo={hardwareInfo} hardwareLoading={hardwareLoading} fetchHardware={fetchHardware} + preflightInfo={preflightInfo} + preflightLoading={preflightLoading} + fetchPreflight={fetchPreflight} config={config} setConfig={setConfig} isGpu={isGpu} diff --git a/apps/web/src/app/vision-training/train/components/wizard/ExpandedCard.tsx b/apps/web/src/app/vision-training/train/components/wizard/ExpandedCard.tsx index 16b56378..1a000c03 100644 --- a/apps/web/src/app/vision-training/train/components/wizard/ExpandedCard.tsx +++ b/apps/web/src/app/vision-training/train/components/wizard/ExpandedCard.tsx @@ -3,6 +3,7 @@ import { css } from '../../../../../../styled-system/css' import { DataCard } from './cards/DataCard' import { HardwareCard } from './cards/HardwareCard' +import { DependencyCard } from './cards/DependencyCard' import { ConfigCard } from './cards/ConfigCard' import { SetupCard } from './cards/SetupCard' import { LoadingCard } from './cards/LoadingCard' @@ -14,6 +15,7 @@ import { type CardId, type SamplesData, type HardwareInfo, + type PreflightInfo, type TrainingConfig, type ServerPhase, type EpochData, @@ -29,6 +31,9 @@ interface ExpandedCardProps { hardwareInfo: HardwareInfo | null hardwareLoading: boolean fetchHardware: () => void + preflightInfo: PreflightInfo | null + preflightLoading: boolean + fetchPreflight: () => void config: TrainingConfig setConfig: (config: TrainingConfig | ((prev: TrainingConfig) => TrainingConfig)) => void isGpu: boolean @@ -56,6 +61,9 @@ export function ExpandedCard({ hardwareInfo, hardwareLoading, fetchHardware, + preflightInfo, + preflightLoading, + fetchPreflight, config, setConfig, isGpu, @@ -95,6 +103,15 @@ export function ExpandedCard({ onProgress={onProgress} /> ) + case 'dependencies': + return ( + + ) case 'config': return ( void + preflightInfo: PreflightInfo | null + preflightLoading: boolean + fetchPreflight: () => void config: TrainingConfig setConfig: (config: TrainingConfig | ((prev: TrainingConfig) => TrainingConfig)) => void isGpu: boolean @@ -63,6 +67,9 @@ export function PhaseSection({ hardwareInfo, hardwareLoading, fetchHardware, + preflightInfo, + preflightLoading, + fetchPreflight, config, setConfig, isGpu, @@ -165,6 +172,9 @@ export function PhaseSection({ hardwareInfo={hardwareInfo} hardwareLoading={hardwareLoading} fetchHardware={fetchHardware} + preflightInfo={preflightInfo} + preflightLoading={preflightLoading} + fetchPreflight={fetchPreflight} config={config} setConfig={setConfig} isGpu={isGpu} diff --git a/apps/web/src/app/vision-training/train/components/wizard/TrainingWizard.tsx b/apps/web/src/app/vision-training/train/components/wizard/TrainingWizard.tsx index 8ad85079..6824abef 100644 --- a/apps/web/src/app/vision-training/train/components/wizard/TrainingWizard.tsx +++ b/apps/web/src/app/vision-training/train/components/wizard/TrainingWizard.tsx @@ -8,6 +8,7 @@ import { serverPhaseToWizardPosition, type SamplesData, type HardwareInfo, + type PreflightInfo, type TrainingConfig, type ServerPhase, type EpochData, @@ -24,6 +25,10 @@ interface TrainingWizardProps { hardwareInfo: HardwareInfo | null hardwareLoading: boolean fetchHardware: () => void + // Preflight state + preflightInfo: PreflightInfo | null + preflightLoading: boolean + fetchPreflight: () => void // Config state config: TrainingConfig setConfig: (config: TrainingConfig | ((prev: TrainingConfig) => TrainingConfig)) => void @@ -48,6 +53,9 @@ export function TrainingWizard({ hardwareInfo, hardwareLoading, fetchHardware, + preflightInfo, + preflightLoading, + fetchPreflight, config, setConfig, serverPhase, @@ -131,6 +139,12 @@ export function TrainingWizard({ label: hardwareInfo.deviceType === 'gpu' ? 'GPU' : 'CPU', value: hardwareInfo.deviceName.split(' ').slice(0, 2).join(' '), } + case 'dependencies': + if (!preflightInfo?.ready) return null + return { + label: 'Packages', + value: `${preflightInfo.dependencies.installed.length}`, + } case 'config': return { label: 'Epochs', value: `${config.epochs}` } case 'setup': @@ -171,6 +185,9 @@ export function TrainingWizard({ hardwareInfo={hardwareInfo} hardwareLoading={hardwareLoading} fetchHardware={fetchHardware} + preflightInfo={preflightInfo} + preflightLoading={preflightLoading} + fetchPreflight={fetchPreflight} config={config} setConfig={setConfig} isGpu={isGpu} @@ -190,8 +207,10 @@ export function TrainingWizard({ onCancel={onCancel} onTrainAgain={handleTrainAgain} onSyncComplete={onSyncComplete} - // Validation - canStartTraining={!!hasEnoughData && !hardwareLoading && !hardwareInfo?.error} + // Validation - require data, hardware, and dependencies all ready + canStartTraining={ + !!hasEnoughData && !hardwareLoading && !hardwareInfo?.error && !!preflightInfo?.ready + } /> ))} diff --git a/apps/web/src/app/vision-training/train/components/wizard/cards/DataCard.tsx b/apps/web/src/app/vision-training/train/components/wizard/cards/DataCard.tsx index 7415ebb0..d4f3c97a 100644 --- a/apps/web/src/app/vision-training/train/components/wizard/cards/DataCard.tsx +++ b/apps/web/src/app/vision-training/train/components/wizard/cards/DataCard.tsx @@ -1,9 +1,9 @@ 'use client' import { useCallback, useEffect, useRef, useState } from 'react' -import Link from 'next/link' import { css } from '../../../../../../../styled-system/css' import type { SamplesData } from '../types' +import { TrainingDataCapture } from '../../TrainingDataCapture' interface DataCardProps { samples: SamplesData | null @@ -12,6 +12,14 @@ interface DataCardProps { onSyncComplete?: () => void // Callback to refresh samples after sync } +// Training data requirements +const REQUIREMENTS = { + minTotal: 50, // Minimum total images + minPerDigit: 3, // Minimum per digit + goodTotal: 200, // Good total + goodPerDigit: 10, // Good per digit +} + interface SyncStatus { available: boolean remote?: { host: string; totalImages: number } @@ -32,22 +40,79 @@ const QUALITY_CONFIG: Record< { color: string; label: string; barWidth: string } > = { none: { color: 'gray.500', label: 'No Data', barWidth: '0%' }, - insufficient: { color: 'red.400', label: 'Need More', barWidth: '20%' }, + insufficient: { color: 'red.400', label: 'Insufficient', barWidth: '20%' }, minimal: { color: 'yellow.400', label: 'Minimal', barWidth: '50%' }, good: { color: 'green.400', label: 'Good', barWidth: '80%' }, excellent: { color: 'green.300', label: 'Excellent', barWidth: '100%' }, } +/** + * Compute what's missing from the training data + */ +function computeRequirements(samples: SamplesData | null): { + needsMore: boolean + totalNeeded: number + missingDigits: number[] + message: string +} { + if (!samples || !samples.hasData) { + return { + needsMore: true, + totalNeeded: REQUIREMENTS.minTotal, + missingDigits: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], + message: `Need ${REQUIREMENTS.minTotal}+ images total, ${REQUIREMENTS.minPerDigit}+ per digit`, + } + } + + const digitCounts = Object.entries(samples.digits).map(([digit, data]) => ({ + digit: parseInt(digit, 10), + count: data.count, + })) + + const missingDigits = digitCounts + .filter((d) => d.count < REQUIREMENTS.minPerDigit) + .map((d) => d.digit) + + const totalNeeded = Math.max(0, REQUIREMENTS.minTotal - samples.totalImages) + const needsMore = samples.totalImages < REQUIREMENTS.minTotal || missingDigits.length > 0 + + let message = '' + if (needsMore) { + const parts = [] + if (totalNeeded > 0) { + parts.push(`${totalNeeded} more images`) + } + if (missingDigits.length > 0) { + parts.push(`digits ${missingDigits.join(', ')} need ${REQUIREMENTS.minPerDigit}+ each`) + } + message = `Need: ${parts.join('; ')}` + } + + return { needsMore, totalNeeded, missingDigits, message } +} + export function DataCard({ samples, samplesLoading, onProgress, onSyncComplete }: DataCardProps) { const [syncStatus, setSyncStatus] = useState(null) const [syncProgress, setSyncProgress] = useState({ phase: 'idle', message: '' }) const [syncChecking, setSyncChecking] = useState(true) + const [showCapture, setShowCapture] = useState(false) + const [showContinueWarning, setShowContinueWarning] = useState(false) const abortRef = useRef(null) + const requirements = computeRequirements(samples) const isReady = samples?.hasData && samples.dataQuality !== 'none' && samples.dataQuality !== 'insufficient' const isSyncing = syncProgress.phase === 'connecting' || syncProgress.phase === 'syncing' + // Handle continue with insufficient data + const handleContinueAnyway = useCallback(() => { + if (!samples?.hasData) { + // Can't continue with zero data + return + } + onProgress() + }, [samples, onProgress]) + // Check sync availability on mount useEffect(() => { const checkSync = async () => { @@ -324,27 +389,52 @@ export function DataCard({ samples, samplesLoading, onProgress, onSyncComplete } )} - {/* No data state - only if sync not available */} - {!samples?.hasData && !showSyncUI && ( + {/* Inline capture toggle - always show when not syncing */} + {!isSyncing && ( +
+ + + {showCapture && ( +
+ onSyncComplete?.()} /> +
+ )} +
+ )} + + {/* No data state */} + {!samples?.hasData && !showCapture && (
๐Ÿ“ท
No training data collected yet
- - Collect Training Data - +
{requirements.message}
)} @@ -395,8 +485,10 @@ export function DataCard({ samples, samplesLoading, onProgress, onSyncComplete }
Distribution
{Object.entries(samples.digits).map(([digit, data]) => { + const digitNum = parseInt(digit, 10) const maxCount = Math.max(...Object.values(samples.digits).map((d) => d.count)) const barHeight = maxCount > 0 ? (data.count / maxCount) * 30 : 0 + const isMissing = requirements.missingDigits.includes(digitNum) return (
- + {digit}
@@ -425,31 +524,131 @@ export function DataCard({ samples, samplesLoading, onProgress, onSyncComplete }
- {/* Ready indicator and continue */} - {isReady && !isSyncing && ( -
-
- โœ“ - Ready to train -
+ {/* Requirements message when insufficient */} + {requirements.needsMore && ( +
+ โš ๏ธ {requirements.message} +
+ )} - + {/* Ready indicator and continue */} + {!isSyncing && ( +
+ {isReady ? ( + <> +
+ โœ“ + + Ready to train + +
+ + + ) : ( + <> + {/* Continue anyway with warning */} + {!showContinueWarning ? ( + + ) : ( +
+
+ โš ๏ธ Warning: Training with insufficient data may produce a + poor model. Results may be inaccurate. +
+
+ + +
+
+ )} + + )}
)} diff --git a/apps/web/src/app/vision-training/train/components/wizard/cards/DependencyCard.tsx b/apps/web/src/app/vision-training/train/components/wizard/cards/DependencyCard.tsx new file mode 100644 index 00000000..e04cc26c --- /dev/null +++ b/apps/web/src/app/vision-training/train/components/wizard/cards/DependencyCard.tsx @@ -0,0 +1,379 @@ +'use client' + +import { useCallback, useEffect, useRef, useState } from 'react' +import { css } from '../../../../../../../styled-system/css' +import type { PreflightInfo } from '../types' + +interface DependencyCardProps { + preflightInfo: PreflightInfo | null + preflightLoading: boolean + fetchPreflight: () => void + onProgress: () => void +} + +const AUTO_PROGRESS_DELAY = 2000 + +export function DependencyCard({ + preflightInfo, + preflightLoading, + fetchPreflight, + onProgress, +}: DependencyCardProps) { + const [countdown, setCountdown] = useState(AUTO_PROGRESS_DELAY) + const timerRef = useRef(null) + const isReady = preflightInfo?.ready && !preflightLoading + + // Auto-progress countdown + useEffect(() => { + if (!isReady) { + setCountdown(AUTO_PROGRESS_DELAY) + return + } + + // Start countdown + const startTime = Date.now() + const tick = () => { + const elapsed = Date.now() - startTime + const remaining = Math.max(0, AUTO_PROGRESS_DELAY - elapsed) + setCountdown(remaining) + + if (remaining <= 0) { + onProgress() + } else { + timerRef.current = setTimeout(tick, 50) + } + } + + timerRef.current = setTimeout(tick, 50) + + return () => { + if (timerRef.current) { + clearTimeout(timerRef.current) + } + } + }, [isReady, onProgress]) + + const handleSkip = useCallback(() => { + if (timerRef.current) { + clearTimeout(timerRef.current) + } + onProgress() + }, [onProgress]) + + const handleRetry = useCallback(() => { + fetchPreflight() + }, [fetchPreflight]) + + if (preflightLoading) { + return ( +
+
+ ๐Ÿ“ฆ +
+
Checking dependencies...
+
+ ) + } + + // Show error if there's a platform or venv issue + if (preflightInfo && (!preflightInfo.platform.supported || preflightInfo.venv.error)) { + const errorMessage = preflightInfo.platform.reason || preflightInfo.venv.error + + return ( +
+
๐Ÿšซ
+
+ Environment Error +
+
+ {errorMessage} +
+ +
+ ) + } + + // Show missing dependencies + if (preflightInfo && !preflightInfo.dependencies.allInstalled) { + const { missing, installed, error } = preflightInfo.dependencies + + return ( +
+
โš ๏ธ
+
+ Missing Dependencies +
+ + {error && ( +
{error}
+ )} + + {/* Missing packages */} +
+
+ Missing ({missing.length}) +
+
+ {missing.map((pkg) => ( + + {pkg.name} + + ))} +
+
+ + {/* Installed packages */} + {installed.length > 0 && ( +
+
+ Installed ({installed.length}) +
+
+ {installed.map((pkg) => ( + + {pkg.name} + + ))} +
+
+ )} + + {/* Instructions */} +
+ Install missing packages manually, then retry: +
+ pip install {missing.map((p) => p.pipName).join(' ')} +
+
+ + +
+ ) + } + + if (!preflightInfo) { + return ( +
+
No dependency info
+ +
+ ) + } + + // All dependencies installed! + const { installed } = preflightInfo.dependencies + const progressPercent = ((AUTO_PROGRESS_DELAY - countdown) / AUTO_PROGRESS_DELAY) * 100 + + return ( +
+ {/* Success Icon */} +
โœ…
+ + {/* Title */} +
+ All Dependencies Ready +
+ + {/* Badge */} +
+ {installed.length} packages installed +
+ + {/* Installed packages list */} +
+
+ {installed.map((pkg) => ( + + {pkg.name} + + ))} +
+
+ + {/* Auto-progress bar */} +
+
+
+
+
+ + {/* Continue button */} + +
+ ) +} diff --git a/apps/web/src/app/vision-training/train/components/wizard/types.ts b/apps/web/src/app/vision-training/train/components/wizard/types.ts index cbea4aad..a50e162f 100644 --- a/apps/web/src/app/vision-training/train/components/wizard/types.ts +++ b/apps/web/src/app/vision-training/train/components/wizard/types.ts @@ -3,6 +3,7 @@ export type PhaseId = 'preparation' | 'training' | 'results' export type CardId = | 'data' | 'hardware' + | 'dependencies' | 'config' | 'setup' | 'loading' @@ -28,7 +29,7 @@ export const PHASES: PhaseDefinition[] = [ { id: 'preparation', title: 'Preparation', - cards: ['data', 'hardware', 'config'], + cards: ['data', 'hardware', 'dependencies', 'config'], }, { id: 'training', @@ -66,6 +67,13 @@ export const CARDS: Record = { autoProgress: true, autoProgressDelay: 2000, }, + dependencies: { + id: 'dependencies', + title: 'Dependencies', + icon: '๐Ÿ“ฆ', + autoProgress: true, + autoProgressDelay: 2000, + }, config: { id: 'config', title: 'Configuration', @@ -140,6 +148,27 @@ export interface HardwareInfo { hint?: string } +export interface PreflightInfo { + ready: boolean + platform: { + supported: boolean + reason?: string + } + venv: { + exists: boolean + python: string + isAppleSilicon: boolean + hasGpu: boolean + error?: string + } + dependencies: { + allInstalled: boolean + installed: { name: string; pipName: string }[] + missing: { name: string; pipName: string }[] + error?: string + } +} + export interface EpochData { epoch: number total_epochs: number diff --git a/apps/web/src/app/vision-training/train/page.tsx b/apps/web/src/app/vision-training/train/page.tsx index 26baf272..97fc2416 100644 --- a/apps/web/src/app/vision-training/train/page.tsx +++ b/apps/web/src/app/vision-training/train/page.tsx @@ -7,6 +7,7 @@ import { TrainingWizard } from './components/wizard/TrainingWizard' import type { SamplesData, HardwareInfo, + PreflightInfo, ServerPhase, TrainingConfig, EpochData, @@ -27,6 +28,10 @@ export default function TrainModelPage() { const [hardwareInfo, setHardwareInfo] = useState(null) const [hardwareLoading, setHardwareLoading] = useState(true) + // Preflight/dependency info + const [preflightInfo, setPreflightInfo] = useState(null) + const [preflightLoading, setPreflightLoading] = useState(true) + // Training state const [serverPhase, setServerPhase] = useState('idle') const [statusMessage, setStatusMessage] = useState('') @@ -79,10 +84,36 @@ export default function TrainModelPage() { } }, []) + // Fetch preflight/dependency info + const fetchPreflight = useCallback(async () => { + setPreflightLoading(true) + setPreflightInfo(null) + try { + const response = await fetch('/api/vision-training/preflight') + const data = await response.json() + setPreflightInfo(data) + } catch { + setPreflightInfo({ + ready: false, + platform: { supported: false, reason: 'Failed to check dependencies' }, + venv: { exists: false, python: '', isAppleSilicon: false, hasGpu: false }, + dependencies: { + allInstalled: false, + installed: [], + missing: [], + error: 'Failed to fetch', + }, + }) + } finally { + setPreflightLoading(false) + } + }, []) + useEffect(() => { fetchHardware() fetchSamples() - }, [fetchHardware, fetchSamples]) + fetchPreflight() + }, [fetchHardware, fetchSamples, fetchPreflight]) useEffect(() => { return () => { @@ -308,6 +339,9 @@ export default function TrainModelPage() { hardwareInfo={hardwareInfo} hardwareLoading={hardwareLoading} fetchHardware={fetchHardware} + preflightInfo={preflightInfo} + preflightLoading={preflightLoading} + fetchPreflight={fetchPreflight} config={config} setConfig={setConfig} serverPhase={serverPhase} diff --git a/apps/web/src/components/vision/CameraCapture.tsx b/apps/web/src/components/vision/CameraCapture.tsx new file mode 100644 index 00000000..b092a75c --- /dev/null +++ b/apps/web/src/components/vision/CameraCapture.tsx @@ -0,0 +1,568 @@ +'use client' + +import { useCallback, useEffect, useRef, useState } from 'react' +import { useDeskViewCamera } from '@/hooks/useDeskViewCamera' +import { useRemoteCameraDesktop } from '@/hooks/useRemoteCameraDesktop' +import { css } from '../../../styled-system/css' +import { RemoteCameraQRCode } from './RemoteCameraQRCode' +import { VisionCameraFeed } from './VisionCameraFeed' + +export type CameraSource = 'local' | 'phone' + +export interface CameraCaptureProps { + /** Initial camera source (default: 'local') */ + initialSource?: CameraSource + /** Called when a frame is captured (either from local video or phone) */ + onCapture?: (imageElement: HTMLImageElement | HTMLVideoElement) => void + /** Called when camera source changes */ + onSourceChange?: (source: CameraSource) => void + /** Called when phone connection status changes */ + onPhoneConnected?: (connected: boolean) => void + /** Show source selector tabs (default: true) */ + showSourceSelector?: boolean + /** Compact mode - smaller QR code, less padding */ + compact?: boolean +} + +/** + * CameraCapture - Reusable component for capturing frames from local or phone camera + * + * Provides: + * - Camera source tabs (local/phone) + * - Local camera feed with device selection + * - Phone camera via QR code connection + * - Access to current frame for capture + * + * Used by: + * - TrainingDataCapture (vision training wizard) + * - AbacusVisionBridge (practice sessions) - uses separate implementation for calibration + */ +export function CameraCapture({ + initialSource = 'local', + onCapture, + onSourceChange, + onPhoneConnected, + showSourceSelector = true, + compact = false, +}: CameraCaptureProps) { + const [cameraSource, setCameraSource] = useState(initialSource) + + // Local camera + const camera = useDeskViewCamera() + const videoRef = useRef(null) + + // Phone camera + const { + isPhoneConnected, + latestFrame, + frameRate, + currentSessionId, + subscribe, + unsubscribe, + getPersistedSessionId, + clearSession, + } = useRemoteCameraDesktop() + const [sessionId, setSessionId] = useState(null) + const imageRef = useRef(null) + + // Notify phone connection changes + useEffect(() => { + if (cameraSource === 'phone') { + onPhoneConnected?.(isPhoneConnected) + } + }, [cameraSource, isPhoneConnected, onPhoneConnected]) + + // Start local camera when source is local + useEffect(() => { + if (cameraSource === 'local') { + camera.requestCamera() + } + return () => { + if (cameraSource === 'local') { + camera.stopCamera() + } + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [cameraSource]) + + // Handle camera source change + const handleSourceChange = useCallback( + (source: CameraSource) => { + if (source === cameraSource) return + + setCameraSource(source) + onSourceChange?.(source) + + if (source === 'local') { + // Stop phone camera, start local + if (currentSessionId) { + unsubscribe() + } + camera.requestCamera() + } else { + // Stop local camera + camera.stopCamera() + // Check for persisted session + const persistedSessionId = getPersistedSessionId() + if (persistedSessionId) { + setSessionId(persistedSessionId) + } + } + }, + [cameraSource, currentSessionId, unsubscribe, camera, getPersistedSessionId, onSourceChange] + ) + + // Handle session created by QR code + const handleSessionCreated = useCallback( + (newSessionId: string) => { + setSessionId(newSessionId) + subscribe(newSessionId) + }, + [subscribe] + ) + + // Subscribe when session is set and source is phone + useEffect(() => { + if (sessionId && cameraSource === 'phone' && !currentSessionId) { + subscribe(sessionId) + } + }, [sessionId, cameraSource, currentSessionId, subscribe]) + + // Cleanup on unmount + useEffect(() => { + return () => { + if (currentSessionId) { + unsubscribe() + } + } + }, [currentSessionId, unsubscribe]) + + // Start fresh phone session + const handleStartFreshSession = useCallback(() => { + clearSession() + setSessionId(null) + }, [clearSession]) + + // Video ref callback for VisionCameraFeed - also calls onCapture + const handleVideoRef = useCallback( + (el: HTMLVideoElement | null) => { + videoRef.current = el + if (el && onCapture) { + onCapture(el) + } + }, + [onCapture] + ) + + // Image ref callback - also calls onCapture + const handleImageRef = useCallback( + (el: HTMLImageElement | null) => { + imageRef.current = el + if (el && onCapture) { + onCapture(el) + } + }, + [onCapture] + ) + + // Update capture element when phone frame changes + useEffect(() => { + if ( + cameraSource === 'phone' && + isPhoneConnected && + latestFrame && + imageRef.current && + onCapture + ) { + onCapture(imageRef.current) + } + }, [cameraSource, isPhoneConnected, latestFrame, onCapture]) + + // Determine if we have a usable frame + const hasFrame = + cameraSource === 'local' + ? camera.videoStream !== null + : isPhoneConnected && latestFrame !== null + + return ( +
+ {/* Camera source selector tabs */} + {showSourceSelector && ( +
+ + +
+ )} + + {/* Camera feed */} +
+ {cameraSource === 'local' ? ( + /* Local camera feed */ + <> + + + {/* Local camera toolbar */} + {camera.videoStream && ( +
+ {/* Camera selector */} + {camera.availableDevices.length > 0 && ( + + )} + + {/* Toolbar buttons */} +
+ {camera.availableDevices.length > 1 && ( + + )} + {camera.isTorchAvailable && ( + + )} +
+
+ )} + + {/* Camera error */} + {camera.error && ( +
+ {camera.error} +
+ )} + + ) : /* Phone camera feed */ !sessionId ? ( + /* Show QR code to connect phone */ +
+ +

+ Scan with your phone to connect +

+
+ ) : !isPhoneConnected ? ( + /* Waiting for phone */ +
+ +
+ + Waiting for phone + ยท + + +
+
+ ) : ( + /* Show camera frames */ +
+ {latestFrame ? ( + Remote camera view + ) : ( +
+ Waiting for frames... +
+ )} + + {/* Phone camera toolbar */} +
+
+ + {frameRate} fps +
+
+
+ )} +
+
+ ) +}