feat(blog): add vision detection stories and screenshots
- Create VisionDetection.stories.tsx with interactive demos - Add screenshot capture script for Storybook stories - Update blog post with captured screenshots - Include before/after comparison, progress gallery, and step demos 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
parent
77b8e6cfb4
commit
3c238dc550
|
|
@ -26,6 +26,8 @@ Until now, students using our practice system had two options:
|
|||
|
||||
Neither option fully captured the experience of a teacher looking over a student's shoulder, nodding as they correctly add each term, gently redirecting when they slip.
|
||||
|
||||

|
||||
|
||||
## How It Works
|
||||
|
||||
Our new vision system uses your device's camera to watch the physical abacus in real time. Here's what happens:
|
||||
|
|
@ -48,6 +50,8 @@ Here's where the magic happens. Our ML model analyzes each column of the abacus
|
|||
|
||||
The student sees their physical actions reflected instantly in the digital interface—validation that they're doing it right.
|
||||
|
||||

|
||||
|
||||
## The Technology Behind It
|
||||
|
||||
We built a custom machine learning model specifically for soroban column recognition. Unlike generic digit recognition, our model understands the unique structure of an abacus:
|
||||
|
|
@ -71,6 +75,17 @@ Raw ML predictions can be noisy—a brief hand shadow might cause a misread. Our
|
|||
|
||||
Research consistently shows that immediate feedback accelerates learning. When a student adds 45 and sees a checkmark appear, they know instantly that they've got it right. No waiting, no uncertainty.
|
||||
|
||||
<div style="display: flex; gap: 2rem; justify-content: center; flex-wrap: wrap;">
|
||||
<figure>
|
||||
<img src="/blog/vision-examples/first-term-completed.png" alt="First term completed with checkmark" style="max-height: 300px;" />
|
||||
<figcaption>First term detected</figcaption>
|
||||
</figure>
|
||||
<figure>
|
||||
<img src="/blog/vision-examples/two-terms-completed.png" alt="Two terms completed with checkmarks" style="max-height: 300px;" />
|
||||
<figcaption>Two terms detected</figcaption>
|
||||
</figure>
|
||||
</div>
|
||||
|
||||
### Scaffolded Independence
|
||||
|
||||
The system doesn't just watch—it helps. When a student shows a prefix sum (like 45 after the first term of 45 + 32), the system recognizes they might need guidance and offers help mode. But when they show 77 (the correct answer), it auto-submits and celebrates their success.
|
||||
|
|
|
|||
Binary file not shown.
|
After Width: | Height: | Size: 24 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 27 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 6.8 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 6.8 KiB |
|
|
@ -0,0 +1,92 @@
|
|||
/**
|
||||
* Capture screenshots from Storybook stories for blog posts.
|
||||
*
|
||||
* Usage:
|
||||
* 1. Start Storybook: pnpm storybook
|
||||
* 2. Run this script: npx tsx scripts/captureStoryScreenshots.ts
|
||||
*
|
||||
* Screenshots are saved to public/blog/vision-examples/
|
||||
*/
|
||||
|
||||
import { chromium } from 'playwright'
|
||||
import { mkdirSync, existsSync } from 'fs'
|
||||
import { join } from 'path'
|
||||
|
||||
const STORYBOOK_URL = 'http://localhost:6006'
|
||||
const OUTPUT_DIR = join(process.cwd(), 'public/blog/vision-examples')
|
||||
|
||||
// Stories to capture with their output filenames
|
||||
const STORIES_TO_CAPTURE = [
|
||||
{
|
||||
// Before/After comparison - great for showing the feature value
|
||||
storyId: 'vision-detection-feedback--before-after',
|
||||
filename: 'before-after-comparison.png',
|
||||
viewport: { width: 900, height: 500 },
|
||||
},
|
||||
{
|
||||
// Progress gallery showing step-by-step detection
|
||||
storyId: 'vision-detection-feedback--progress-gallery',
|
||||
filename: 'detection-progress-gallery.png',
|
||||
viewport: { width: 1200, height: 500 },
|
||||
},
|
||||
{
|
||||
// First term completed
|
||||
storyId: 'vision-detection-feedback--first-term-completed',
|
||||
filename: 'first-term-completed.png',
|
||||
viewport: { width: 400, height: 450 },
|
||||
},
|
||||
{
|
||||
// Two terms completed
|
||||
storyId: 'vision-detection-feedback--two-terms-completed',
|
||||
filename: 'two-terms-completed.png',
|
||||
viewport: { width: 400, height: 450 },
|
||||
},
|
||||
]
|
||||
|
||||
async function captureScreenshots() {
|
||||
// Ensure output directory exists
|
||||
if (!existsSync(OUTPUT_DIR)) {
|
||||
mkdirSync(OUTPUT_DIR, { recursive: true })
|
||||
console.log(`Created output directory: ${OUTPUT_DIR}`)
|
||||
}
|
||||
|
||||
console.log('Launching browser...')
|
||||
const browser = await chromium.launch()
|
||||
const context = await browser.newContext()
|
||||
|
||||
for (const story of STORIES_TO_CAPTURE) {
|
||||
console.log(`\nCapturing: ${story.filename}`)
|
||||
|
||||
const page = await context.newPage()
|
||||
await page.setViewportSize(story.viewport)
|
||||
|
||||
// Storybook iframe URL for isolated story view
|
||||
const storyUrl = `${STORYBOOK_URL}/iframe.html?id=${story.storyId}&viewMode=story`
|
||||
console.log(` URL: ${storyUrl}`)
|
||||
|
||||
try {
|
||||
await page.goto(storyUrl, { waitUntil: 'load', timeout: 60000 })
|
||||
|
||||
// Wait for the story to render and animations to settle
|
||||
await page.waitForTimeout(2000)
|
||||
|
||||
const outputPath = join(OUTPUT_DIR, story.filename)
|
||||
await page.screenshot({
|
||||
path: outputPath,
|
||||
type: 'png',
|
||||
})
|
||||
|
||||
console.log(` Saved: ${outputPath}`)
|
||||
} catch (error) {
|
||||
console.error(` Error capturing ${story.filename}:`, error)
|
||||
}
|
||||
|
||||
await page.close()
|
||||
}
|
||||
|
||||
await browser.close()
|
||||
console.log('\nDone!')
|
||||
}
|
||||
|
||||
// Run the script
|
||||
captureScreenshots().catch(console.error)
|
||||
|
|
@ -0,0 +1,522 @@
|
|||
import type { Meta, StoryObj } from '@storybook/react'
|
||||
import { useState, useEffect } from 'react'
|
||||
import { css } from '../../../styled-system/css'
|
||||
import { VerticalProblem } from './VerticalProblem'
|
||||
|
||||
/**
|
||||
* Stories demonstrating the vision-powered abacus detection feature.
|
||||
* These showcase how the system provides real-time feedback as students
|
||||
* work through problems on their physical abacus.
|
||||
*/
|
||||
const meta: Meta = {
|
||||
title: 'Vision/Detection Feedback',
|
||||
parameters: {
|
||||
layout: 'centered',
|
||||
},
|
||||
}
|
||||
|
||||
export default meta
|
||||
type Story = StoryObj
|
||||
|
||||
/**
|
||||
* Shows checkmarks appearing on terms as the vision system detects
|
||||
* the student completing each step on their physical abacus.
|
||||
*/
|
||||
function VisionProgressDemo() {
|
||||
const terms = [45, 32, 18]
|
||||
const prefixSums = [45, 77, 95] // Running totals after each term
|
||||
const [detectedIndex, setDetectedIndex] = useState<number | undefined>(undefined)
|
||||
const [detectedValue, setDetectedValue] = useState<number | null>(null)
|
||||
|
||||
return (
|
||||
<div
|
||||
className={css({
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
alignItems: 'center',
|
||||
gap: '2rem',
|
||||
padding: '2rem',
|
||||
minWidth: '400px',
|
||||
})}
|
||||
>
|
||||
<div
|
||||
className={css({
|
||||
textAlign: 'center',
|
||||
marginBottom: '1rem',
|
||||
})}
|
||||
>
|
||||
<h3
|
||||
className={css({
|
||||
fontSize: '1.25rem',
|
||||
fontWeight: 'bold',
|
||||
color: 'gray.800',
|
||||
marginBottom: '0.5rem',
|
||||
})}
|
||||
>
|
||||
Vision Detection Progress
|
||||
</h3>
|
||||
<p className={css({ fontSize: '0.875rem', color: 'gray.600' })}>
|
||||
Checkmarks appear as the camera detects completed terms
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<VerticalProblem
|
||||
terms={terms}
|
||||
userAnswer={detectedValue?.toString() ?? ''}
|
||||
isFocused={true}
|
||||
isCompleted={false}
|
||||
correctAnswer={95}
|
||||
size="large"
|
||||
detectedPrefixIndex={detectedIndex}
|
||||
/>
|
||||
|
||||
{/* Simulated detection display */}
|
||||
<div
|
||||
className={css({
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
alignItems: 'center',
|
||||
gap: '0.75rem',
|
||||
padding: '1rem',
|
||||
backgroundColor: 'gray.100',
|
||||
borderRadius: '12px',
|
||||
width: '100%',
|
||||
maxWidth: '300px',
|
||||
})}
|
||||
>
|
||||
<div
|
||||
className={css({
|
||||
fontSize: '0.75rem',
|
||||
fontWeight: '600',
|
||||
color: 'gray.500',
|
||||
textTransform: 'uppercase',
|
||||
letterSpacing: '0.05em',
|
||||
})}
|
||||
>
|
||||
Detected on Abacus
|
||||
</div>
|
||||
<div
|
||||
className={css({
|
||||
fontSize: '2.5rem',
|
||||
fontWeight: 'bold',
|
||||
fontFamily: 'mono',
|
||||
color: detectedValue !== null ? 'green.600' : 'gray.400',
|
||||
})}
|
||||
>
|
||||
{detectedValue !== null ? detectedValue : '—'}
|
||||
</div>
|
||||
|
||||
{/* Control buttons */}
|
||||
<div
|
||||
className={css({
|
||||
display: 'flex',
|
||||
gap: '0.5rem',
|
||||
marginTop: '0.5rem',
|
||||
})}
|
||||
>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => {
|
||||
setDetectedIndex(undefined)
|
||||
setDetectedValue(null)
|
||||
}}
|
||||
className={css({
|
||||
padding: '0.5rem 1rem',
|
||||
fontSize: '0.75rem',
|
||||
backgroundColor: 'gray.200',
|
||||
border: 'none',
|
||||
borderRadius: '6px',
|
||||
cursor: 'pointer',
|
||||
_hover: { backgroundColor: 'gray.300' },
|
||||
})}
|
||||
>
|
||||
Reset
|
||||
</button>
|
||||
{prefixSums.map((sum, i) => (
|
||||
<button
|
||||
key={i}
|
||||
type="button"
|
||||
onClick={() => {
|
||||
setDetectedIndex(i)
|
||||
setDetectedValue(sum)
|
||||
}}
|
||||
className={css({
|
||||
padding: '0.5rem 1rem',
|
||||
fontSize: '0.75rem',
|
||||
backgroundColor: detectedIndex === i ? 'green.500' : 'blue.500',
|
||||
color: 'white',
|
||||
border: 'none',
|
||||
borderRadius: '6px',
|
||||
cursor: 'pointer',
|
||||
_hover: { opacity: 0.9 },
|
||||
})}
|
||||
>
|
||||
{sum}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export const ProgressDemo: Meta = {
|
||||
render: () => <VisionProgressDemo />,
|
||||
}
|
||||
|
||||
/**
|
||||
* Static example showing the first term completed
|
||||
*/
|
||||
export const FirstTermCompleted: Meta = {
|
||||
render: () => (
|
||||
<div className={css({ padding: '2rem' })}>
|
||||
<VerticalProblem
|
||||
terms={[45, 32, 18]}
|
||||
userAnswer="45"
|
||||
isFocused={true}
|
||||
isCompleted={false}
|
||||
correctAnswer={95}
|
||||
size="large"
|
||||
detectedPrefixIndex={0}
|
||||
/>
|
||||
</div>
|
||||
),
|
||||
}
|
||||
|
||||
/**
|
||||
* Static example showing two terms completed
|
||||
*/
|
||||
export const TwoTermsCompleted: Meta = {
|
||||
render: () => (
|
||||
<div className={css({ padding: '2rem' })}>
|
||||
<VerticalProblem
|
||||
terms={[45, 32, 18]}
|
||||
userAnswer="77"
|
||||
isFocused={true}
|
||||
isCompleted={false}
|
||||
correctAnswer={95}
|
||||
size="large"
|
||||
detectedPrefixIndex={1}
|
||||
/>
|
||||
</div>
|
||||
),
|
||||
}
|
||||
|
||||
/**
|
||||
* Gallery showing the progression of vision detection
|
||||
*/
|
||||
function VisionProgressGallery() {
|
||||
const terms = [45, 32, 18]
|
||||
const stages = [
|
||||
{ label: 'Starting', userAnswer: '', detectedPrefixIndex: undefined },
|
||||
{ label: 'First term (45)', userAnswer: '45', detectedPrefixIndex: 0 },
|
||||
{ label: 'Two terms (77)', userAnswer: '77', detectedPrefixIndex: 1 },
|
||||
{ label: 'Final answer (95)', userAnswer: '95', detectedPrefixIndex: undefined },
|
||||
]
|
||||
|
||||
return (
|
||||
<div
|
||||
className={css({
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
gap: '1rem',
|
||||
padding: '1.5rem',
|
||||
})}
|
||||
>
|
||||
<h2
|
||||
className={css({
|
||||
fontSize: '1.5rem',
|
||||
fontWeight: 'bold',
|
||||
textAlign: 'center',
|
||||
marginBottom: '1rem',
|
||||
})}
|
||||
>
|
||||
Vision Detection: Step by Step
|
||||
</h2>
|
||||
|
||||
<div
|
||||
className={css({
|
||||
display: 'grid',
|
||||
gridTemplateColumns: 'repeat(4, 1fr)',
|
||||
gap: '1.5rem',
|
||||
})}
|
||||
>
|
||||
{stages.map((stage, i) => (
|
||||
<div
|
||||
key={i}
|
||||
className={css({
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
alignItems: 'center',
|
||||
gap: '0.75rem',
|
||||
})}
|
||||
>
|
||||
<div
|
||||
className={css({
|
||||
fontSize: '0.75rem',
|
||||
fontWeight: '600',
|
||||
color: 'gray.600',
|
||||
textAlign: 'center',
|
||||
})}
|
||||
>
|
||||
{stage.label}
|
||||
</div>
|
||||
<div
|
||||
className={css({
|
||||
padding: '1rem',
|
||||
backgroundColor: i === stages.length - 1 ? 'green.50' : 'white',
|
||||
borderRadius: '12px',
|
||||
border: '2px solid',
|
||||
borderColor: i === stages.length - 1 ? 'green.300' : 'gray.200',
|
||||
})}
|
||||
>
|
||||
<VerticalProblem
|
||||
terms={terms}
|
||||
userAnswer={stage.userAnswer}
|
||||
isFocused={i < stages.length - 1}
|
||||
isCompleted={i === stages.length - 1}
|
||||
correctAnswer={95}
|
||||
size="normal"
|
||||
detectedPrefixIndex={stage.detectedPrefixIndex}
|
||||
/>
|
||||
</div>
|
||||
{i < stages.length - 1 && (
|
||||
<div className={css({ fontSize: '1.5rem', color: 'gray.400' })}>→</div>
|
||||
)}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export const ProgressGallery: Meta = {
|
||||
render: () => <VisionProgressGallery />,
|
||||
parameters: {
|
||||
layout: 'fullscreen',
|
||||
},
|
||||
}
|
||||
|
||||
/**
|
||||
* Simulated live detection with animated value changes
|
||||
*/
|
||||
function LiveDetectionSimulation() {
|
||||
const terms = [25, 17, 8]
|
||||
const prefixSums = [25, 42, 50]
|
||||
const [step, setStep] = useState(0)
|
||||
const [isPlaying, setIsPlaying] = useState(false)
|
||||
|
||||
useEffect(() => {
|
||||
if (!isPlaying) return
|
||||
|
||||
const timer = setInterval(() => {
|
||||
setStep((s) => {
|
||||
if (s >= prefixSums.length) {
|
||||
setIsPlaying(false)
|
||||
return s
|
||||
}
|
||||
return s + 1
|
||||
})
|
||||
}, 2000)
|
||||
|
||||
return () => clearInterval(timer)
|
||||
}, [isPlaying, prefixSums.length])
|
||||
|
||||
const currentValue = step === 0 ? null : prefixSums[step - 1]
|
||||
const detectedIndex = step === 0 ? undefined : step - 1
|
||||
|
||||
return (
|
||||
<div
|
||||
className={css({
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
alignItems: 'center',
|
||||
gap: '2rem',
|
||||
padding: '2rem',
|
||||
})}
|
||||
>
|
||||
<div className={css({ textAlign: 'center' })}>
|
||||
<h3 className={css({ fontSize: '1.25rem', fontWeight: 'bold', marginBottom: '0.5rem' })}>
|
||||
Live Vision Detection
|
||||
</h3>
|
||||
<p className={css({ fontSize: '0.875rem', color: 'gray.600' })}>
|
||||
Watch checkmarks appear as values are detected
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{/* Detection indicator */}
|
||||
<div
|
||||
className={css({
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
gap: '1rem',
|
||||
padding: '1rem 1.5rem',
|
||||
backgroundColor: 'gray.900',
|
||||
borderRadius: '12px',
|
||||
color: 'white',
|
||||
})}
|
||||
>
|
||||
<div
|
||||
className={css({
|
||||
width: '12px',
|
||||
height: '12px',
|
||||
borderRadius: '50%',
|
||||
backgroundColor: currentValue !== null ? 'green.400' : 'gray.500',
|
||||
animation: currentValue !== null ? 'pulse 1s infinite' : 'none',
|
||||
})}
|
||||
/>
|
||||
<div className={css({ fontSize: '0.75rem', color: 'gray.400' })}>DETECTED</div>
|
||||
<div className={css({ fontSize: '2rem', fontFamily: 'mono', fontWeight: 'bold' })}>
|
||||
{currentValue ?? '—'}
|
||||
</div>
|
||||
<div className={css({ fontSize: '0.75rem', color: 'gray.500' })}>
|
||||
{currentValue !== null ? '98% confidence' : 'waiting...'}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<VerticalProblem
|
||||
terms={terms}
|
||||
userAnswer={currentValue?.toString() ?? ''}
|
||||
isFocused={step < prefixSums.length}
|
||||
isCompleted={step >= prefixSums.length}
|
||||
correctAnswer={50}
|
||||
size="large"
|
||||
detectedPrefixIndex={detectedIndex}
|
||||
/>
|
||||
|
||||
<div className={css({ display: 'flex', gap: '1rem' })}>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => {
|
||||
setStep(0)
|
||||
setIsPlaying(true)
|
||||
}}
|
||||
disabled={isPlaying}
|
||||
className={css({
|
||||
padding: '0.75rem 1.5rem',
|
||||
backgroundColor: isPlaying ? 'gray.400' : 'green.500',
|
||||
color: 'white',
|
||||
border: 'none',
|
||||
borderRadius: '8px',
|
||||
fontWeight: '600',
|
||||
cursor: isPlaying ? 'not-allowed' : 'pointer',
|
||||
_hover: { opacity: isPlaying ? 1 : 0.9 },
|
||||
})}
|
||||
>
|
||||
{isPlaying ? 'Detecting...' : 'Start Simulation'}
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => {
|
||||
setStep(0)
|
||||
setIsPlaying(false)
|
||||
}}
|
||||
className={css({
|
||||
padding: '0.75rem 1.5rem',
|
||||
backgroundColor: 'gray.200',
|
||||
border: 'none',
|
||||
borderRadius: '8px',
|
||||
fontWeight: '600',
|
||||
cursor: 'pointer',
|
||||
_hover: { backgroundColor: 'gray.300' },
|
||||
})}
|
||||
>
|
||||
Reset
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export const LiveSimulation: Meta = {
|
||||
render: () => <LiveDetectionSimulation />,
|
||||
}
|
||||
|
||||
/**
|
||||
* Simple side-by-side comparison for blog screenshot
|
||||
*/
|
||||
function BeforeAfterComparison() {
|
||||
return (
|
||||
<div
|
||||
className={css({
|
||||
display: 'flex',
|
||||
gap: '3rem',
|
||||
padding: '2rem',
|
||||
alignItems: 'flex-start',
|
||||
})}
|
||||
>
|
||||
<div className={css({ textAlign: 'center' })}>
|
||||
<div
|
||||
className={css({
|
||||
fontSize: '0.875rem',
|
||||
fontWeight: '600',
|
||||
color: 'gray.600',
|
||||
marginBottom: '1rem',
|
||||
})}
|
||||
>
|
||||
Without Vision
|
||||
</div>
|
||||
<div
|
||||
className={css({
|
||||
padding: '1.5rem',
|
||||
backgroundColor: 'gray.50',
|
||||
borderRadius: '12px',
|
||||
border: '2px solid',
|
||||
borderColor: 'gray.200',
|
||||
})}
|
||||
>
|
||||
<VerticalProblem
|
||||
terms={[45, 32, 18]}
|
||||
userAnswer=""
|
||||
isFocused={true}
|
||||
isCompleted={false}
|
||||
correctAnswer={95}
|
||||
size="large"
|
||||
/>
|
||||
</div>
|
||||
<div className={css({ fontSize: '0.75rem', color: 'gray.500', marginTop: '0.75rem' })}>
|
||||
No feedback until answer is entered
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className={css({ textAlign: 'center' })}>
|
||||
<div
|
||||
className={css({
|
||||
fontSize: '0.875rem',
|
||||
fontWeight: '600',
|
||||
color: 'green.600',
|
||||
marginBottom: '1rem',
|
||||
})}
|
||||
>
|
||||
With Vision Detection
|
||||
</div>
|
||||
<div
|
||||
className={css({
|
||||
padding: '1.5rem',
|
||||
backgroundColor: 'green.50',
|
||||
borderRadius: '12px',
|
||||
border: '2px solid',
|
||||
borderColor: 'green.300',
|
||||
})}
|
||||
>
|
||||
<VerticalProblem
|
||||
terms={[45, 32, 18]}
|
||||
userAnswer="77"
|
||||
isFocused={true}
|
||||
isCompleted={false}
|
||||
correctAnswer={95}
|
||||
size="large"
|
||||
detectedPrefixIndex={1}
|
||||
/>
|
||||
</div>
|
||||
<div className={css({ fontSize: '0.75rem', color: 'green.600', marginTop: '0.75rem' })}>
|
||||
Real-time checkmarks as terms complete
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export const BeforeAfter: Meta = {
|
||||
render: () => <BeforeAfterComparison />,
|
||||
}
|
||||
Loading…
Reference in New Issue