feat(backend): scan-product accuracy 66.2% -> 79.7% + frozen validation benchmark
Accuracy work on the 79-image product-scan validation set (user goal: 90%): - classify_ocr_server.py: 0/90/180/270-degree expiry-date search (stops at first hit, 0-degree fallback); classification decoupled onto the upright image (rotated frames regressed DINOv2 -6pts until this); cross-line date stitching; tiled full-res OCR pass (defeats the 4000px downscale that killed small inkjet dates); VL-pipeline expiry fallback with keyword-anchored anti-hallucination guard; VL text lines merged into text_lines + VL SKU retry. Visualization endpoints removed entirely (Visual/Spotting grids - unused by frontend, 3x per-scan GPU cost). - product-scan.ts: coverage-normalized OCR-evidence re-ranking of DINOv2 top-K (tuned offline: +8/-0 on top-1 misses), re-ranked class mapped to sku_master by SKU prefix; classifier timeout 90s->240s for fallback paths. - Frozen benchmark: product-test-images-fixed/ (79 renamed images) + freeze/seed/build-undetected/capture/experiment scripts; labels trimmed to the 79 validation entries (training rows kept in .bak-with-training); 5 TRAINED-ON SKUs replaced with fresh held-out photos. - manual-label-scan page: shows last batch-test AI prediction under every field by default (new /api/product-scan-results); serves the fixed folder; fixed total hydration failure via allowedDevOrigins 127.0.0.1. - Measured (all-79, zero failures): sku/name 87.3%, expiry 64.6%, overall 79.7%. Tiles/VL-evidence/VL-SKU deployed but not yet batch-measured. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Gr6HH7JrdsXX8AARejQboM
This commit is contained in:
1 parent
19f1facf9b
commit
e76ccb60a6
156 files changed
+17150
-1405
No files matched your search
@@ -18,7 +18,8 @@ export default function ManualLabelScanPage() {
|
||||
notes: ""
|
||||
});
|
||||
const [aiPredicted, setAiPredicted] = useState<AiPredictedData | null>(null);
|
||||
|
||||
const [aiSource, setAiSource] = useState<{ type: "batch" | "live"; timestamp: string; method?: string; confidence?: number } | null>(null);
|
||||
|
||||
const [skuList, setSkuList] = useState<Array<{ no_sku: string; nama_item: string }>>([]);
|
||||
const [isScanning, setIsScanning] = useState(false);
|
||||
const [savingGT, setSavingGT] = useState(false);
|
||||
@@ -39,20 +40,11 @@ export default function ManualLabelScanPage() {
|
||||
setSkuList(skuData.skus || []);
|
||||
}
|
||||
|
||||
// Fetch Training Images
|
||||
const pfmRes = await fetch("/api/produk-pfm");
|
||||
let trainingFiles: { url: string; filename: string }[] = [];
|
||||
if (pfmRes.ok) {
|
||||
const pfmData = await pfmRes.json();
|
||||
trainingFiles = (pfmData.products || []).flatMap((p: any) =>
|
||||
p.images.map((url: string) => ({
|
||||
url,
|
||||
filename: url.replace(/^\/produk-pfm\/foto-kemasan-v2\//, "")
|
||||
}))
|
||||
);
|
||||
}
|
||||
|
||||
// Fetch Test Images
|
||||
// Fetch Test Images — the frozen 79-image Validation Set
|
||||
// (product-test-images-fixed/), the only set the accuracy harness
|
||||
// scores. Gallery/training photos (foto-kemasan-v2/) are not shown
|
||||
// here: they don't need per-photo ground truth, only correct
|
||||
// SKU-folder placement for classifier training.
|
||||
const testRes = await fetch("/api/product-images");
|
||||
let testFiles: { url: string; filename: string }[] = [];
|
||||
if (testRes.ok) {
|
||||
@@ -63,9 +55,8 @@ export default function ManualLabelScanPage() {
|
||||
}));
|
||||
}
|
||||
|
||||
const combined = [...testFiles, ...trainingFiles];
|
||||
setFiles(combined);
|
||||
if (combined.length > 0) setCurrentIndex(0);
|
||||
setFiles(testFiles);
|
||||
if (testFiles.length > 0) setCurrentIndex(0);
|
||||
|
||||
} catch (err) {
|
||||
console.error("Error initializing page", err);
|
||||
@@ -91,11 +82,33 @@ export default function ManualLabelScanPage() {
|
||||
expiry_date: data.expiry_date || "",
|
||||
notes: data.notes || ""
|
||||
});
|
||||
setAiPredicted(null); // Reset AI predictions on new file load
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("Error fetching label", err);
|
||||
}
|
||||
|
||||
// Default-load the AI prediction from the last batch accuracy run
|
||||
// (not a live re-scan) so failures are visible immediately while
|
||||
// browsing - "Scan with AI" below can still be used to get a fresh
|
||||
// live result for this exact image.
|
||||
setAiPredicted(null);
|
||||
setAiSource(null);
|
||||
try {
|
||||
const aiRes = await fetch(`/api/product-scan-results?filename=${encodeURIComponent(file.filename)}`);
|
||||
if (aiRes.ok) {
|
||||
const aiData = await aiRes.json();
|
||||
if (aiData.found) {
|
||||
setAiPredicted({
|
||||
no_sku: aiData.no_sku,
|
||||
nama_item: aiData.nama_item,
|
||||
expiry_date: aiData.expiry_date
|
||||
});
|
||||
setAiSource({ type: "batch", timestamp: aiData.timestamp, method: aiData.method, confidence: aiData.confidence });
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("Error fetching batch AI result", err);
|
||||
}
|
||||
};
|
||||
loadLabel();
|
||||
}, [currentIndex, files]);
|
||||
@@ -138,13 +151,27 @@ export default function ManualLabelScanPage() {
|
||||
|
||||
if (!scanRes.ok) throw new Error("Pipeline API error");
|
||||
const scanData = await scanRes.json();
|
||||
|
||||
|
||||
// Compare against the sku_master-resolved best match (what the app
|
||||
// actually shows/saves as nama_item, and what the accuracy harness
|
||||
// scores), not classification.top1_name - that's the classifier's raw
|
||||
// internal class label (e.g. the foto-kemasan-v2 folder name), which
|
||||
// structurally never matches a sku_master-style ground truth string
|
||||
// even when the classification itself is correct.
|
||||
const bestMatch = (scanData.possibleMatches || []).find((m: { isBestMatch?: boolean }) => m.isBestMatch);
|
||||
|
||||
setAiPredicted({
|
||||
no_sku: scanData.classification?.top1_name ? skuList.find(s => s.nama_item === scanData.classification.top1_name)?.no_sku : undefined,
|
||||
nama_item: scanData.classification?.top1_name,
|
||||
no_sku: bestMatch?.no_sku,
|
||||
nama_item: bestMatch?.nama_item,
|
||||
expiry_date: scanData.ocr?.extracted_expired_date
|
||||
});
|
||||
|
||||
setAiSource({
|
||||
type: "live",
|
||||
timestamp: new Date().toISOString(),
|
||||
method: scanData.classification?.method,
|
||||
confidence: scanData.classification?.top1_confidence
|
||||
});
|
||||
|
||||
showToast("AI Scan complete!");
|
||||
} catch (err) {
|
||||
showToast(getErrorMessage(err, undefined, "AI Scan failed"), true);
|
||||
@@ -206,6 +233,7 @@ export default function ManualLabelScanPage() {
|
||||
<Editor
|
||||
formData={formData}
|
||||
aiPredicted={aiPredicted}
|
||||
aiSource={aiSource}
|
||||
skuList={skuList}
|
||||
isScanning={isScanning}
|
||||
onScanWithAi={handleScanWithAi}
|
||||
|
||||
Reference in new issue
Block a user