import fs from "node:fs"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { execSync } from "node:child_process"; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); const APP_ROOT = path.join(__dirname, "..", "pfm-web-app"); const SOURCES_DIR = path.join(__dirname, "..", "sources"); const LABELS_PATH = path.join(SOURCES_DIR, "product_manual_labels.json"); const HISTORY_PATH = path.join(SOURCES_DIR, "product_accuracy_history.jsonl"); const FETCH_TIMEOUT_MS = 120_000; interface GroundTruth { filename: string; no_sku: string; nama_item: string; expiry_date: string; } interface ScanResponse { classification?: { top1_name: string; top1_confidence: number; }; ocr?: { extracted_expired_date: string; }; possibleMatches?: Array<{ no_sku: string; nama_item: string; isBestMatch: boolean; }>; } interface Check { field: "no_sku" | "nama_item" | "expiry_date"; match: boolean; } interface HistoryEntry { timestamp: string; commit: string; imageCount: { training: number; validation: number }; failedImages: string[]; fields: { training: Record; validation: Record; }; } function parseArgs(argv: string[]) { return { baseUrl: process.env.ACCURACY_BASE_URL || "http://localhost:3000", }; } function norm(v: unknown): string { if (!v) return ""; return String(v).replace(/\s+/g, " ").trim().toUpperCase(); } function isMatch(a: unknown, b: unknown): boolean { return norm(a) === norm(b); } function getImagePath(filename: string): string { if (filename.includes("/")) { return path.join(APP_ROOT, "public", "produk-pfm", "foto-kemasan-v2", filename); } return path.join(SOURCES_DIR, "product-test-images", filename); } async function checkServerReachable(baseUrl: string) { try { const res = await fetch(`${baseUrl}/api/v1/health`, { signal: AbortSignal.timeout(5000) }); if (!res.ok) throw new Error(`HTTP ${res.status}`); } catch (err) { throw new Error(`Next dev server not reachable at ${baseUrl}. Ensure it's running.`); } } async function fetchScan(baseUrl: string, base64: string): Promise { const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS); try { const res = await fetch(`${baseUrl}/api/scan-pfm`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ image_base64: base64 }), signal: controller.signal }); if (!res.ok) throw new Error(`HTTP ${res.status}`); return await res.json(); } finally { clearTimeout(timer); } } function getGitCommit(): string { try { return execSync("git rev-parse --short HEAD", { cwd: __dirname }).toString().trim(); } catch { return "unknown"; } } function pct(c: number, t: number) { return t === 0 ? 0 : (c / t) * 100; } function fmtPct(n: number) { return `${n.toFixed(1)}%`; } async function main() { const args = parseArgs(process.argv.slice(2)); await checkServerReachable(args.baseUrl); if (!fs.existsSync(LABELS_PATH)) { console.error(`No labels found at ${LABELS_PATH}`); process.exit(1); } const labels: GroundTruth[] = JSON.parse(fs.readFileSync(LABELS_PATH, "utf8")); const results = { training: [] as { gt: GroundTruth; checks: Check[] }[], validation: [] as { gt: GroundTruth; checks: Check[] }[], failed: [] as string[] }; for (const gt of labels) { if (gt.filename.startsWith("uploaded-")) continue; // Skip phantom const imgPath = getImagePath(gt.filename); if (!fs.existsSync(imgPath)) { console.warn(`Warning: Image missing from disk: ${imgPath}`); continue; } process.stdout.write(`Scanning ${gt.filename}... `); try { const b64 = "data:image/jpeg;base64," + fs.readFileSync(imgPath, "base64"); const parsed = await fetchScan(args.baseUrl, b64); const bestMatchSku = parsed.possibleMatches?.find(m => m.isBestMatch)?.no_sku || ""; const predictedItemName = parsed.classification?.top1_name || ""; const predictedExpiry = parsed.ocr?.extracted_expired_date || ""; const checks: Check[] = [ { field: "no_sku", match: isMatch(gt.no_sku, bestMatchSku) }, { field: "nama_item", match: isMatch(gt.nama_item, predictedItemName) }, { field: "expiry_date", match: isMatch(gt.expiry_date, predictedExpiry) } ]; if (gt.filename.includes("/")) { results.training.push({ gt, checks }); } else { results.validation.push({ gt, checks }); } const score = checks.filter(c => c.match).length; console.log(`done (${score}/3)`); } catch (err) { console.log(`FAILED (${(err as Error).message})`); results.failed.push(gt.filename); } } const aggregate = (list: { checks: Check[] }[]) => { const agg: Record = { no_sku: { correct: 0, total: 0 }, nama_item: { correct: 0, total: 0 }, expiry_date: { correct: 0, total: 0 } }; for (const item of list) { for (const check of item.checks) { agg[check.field].total++; if (check.match) agg[check.field].correct++; } } return agg; }; const trainAgg = aggregate(results.training); const valAgg = aggregate(results.validation); console.log("\n=== Product Scan Accuracy Summary ==="); console.log(`Training Images: ${results.training.length} | Validation Images: ${results.validation.length} | Failed: ${results.failed.length}\n`); console.log("Field | Training Set | Validation Set"); console.log("---------------|--------------|---------------"); ["no_sku", "nama_item", "expiry_date"].forEach(f => { const t = trainAgg[f].total ? fmtPct(pct(trainAgg[f].correct, trainAgg[f].total)) : "n/a"; const v = valAgg[f].total ? fmtPct(pct(valAgg[f].correct, valAgg[f].total)) : "n/a"; console.log(`${f.padEnd(14)} | ${t.padEnd(12)} | ${v.padEnd(14)}`); }); console.log(""); const entry: HistoryEntry = { timestamp: new Date().toISOString(), commit: getGitCommit(), imageCount: { training: results.training.length, validation: results.validation.length }, failedImages: results.failed, fields: { training: trainAgg, validation: valAgg } }; fs.appendFileSync(HISTORY_PATH, JSON.stringify(entry) + "\n"); } main().catch(err => { console.error("Fatal error:", err); process.exit(1); });