import { config as loadDotenv } from "dotenv"; import { existsSync, readFileSync } from "node:fs"; import path from "node:path"; for (const candidate of [".env", "../.env", "../../.env"]) { const p = path.resolve(process.cwd(), candidate); if (existsSync(p)) { loadDotenv({ path: p }); break; } } import { fileURLToPath } from "node:url"; import { GeminiAamosClient, type BudgetStore } from "@app/ai-contracts"; /** * Live-evaluering av Gemini-lärar-tier för kylskåps-/skafferiskanning. * * pnpm --filter @app/worker eval:scan * * Kräver AAMOS_MODE=gemini + GEMINI_API_KEY. Använder lokala fixture-bilder * (committade i repot) – inga externa hämtningar, inga 429. */ const __filename = fileURLToPath(import.meta.url); const FIXTURES = path.resolve(path.dirname(__filename), "fixtures"); function fileToDataUrl(filePath: string): string { const buf = readFileSync(filePath); const ext = path.extname(filePath).toLowerCase(); const mime = ext === ".png" ? "image/png" : ext === ".webp" ? "image/webp" : "image/jpeg"; return `data:${mime};base64,${buf.toString("base64")}`; } interface ScanEvalCase { id: string; imageUrls: string[]; locationType: "fridge" | "pantry"; marketLocale: string; checks: (items: Array<{ detectedName: string; brand: string | null; estimatedQuantity: number | null; unit: string | null; confidence: number; requiresConfirmation: boolean; }>) => { name: string; passed: boolean }[]; } const CASES: ScanEvalCase[] = [ { id: "fridge-1", imageUrls: [fileToDataUrl(path.join(FIXTURES, "fridge-1.jpg"))], locationType: "fridge", marketLocale: "sv-SE", checks: (items) => { const milk = items.find((i) => /mjölk|milk|grädde|cream/i.test(i.detectedName)); return [ { name: "hittade mjölkprodukt", passed: !!milk }, { name: "konfidens > 0.5", passed: !!milk && milk.confidence > 0.5 }, { name: "kräver bekräftelse om låg konfidens", passed: !!milk && (milk.confidence >= 0.92 || milk.requiresConfirmation) }, ]; }, }, { id: "fridge-2", imageUrls: [fileToDataUrl(path.join(FIXTURES, "fridge-2.jpg"))], locationType: "fridge", marketLocale: "sv-SE", checks: (items) => { const found = items.find((i) => /gurka|inlagd|konserverad|burk|pickle/i.test(i.detectedName)); return [ { name: "hittade konserverad produkt", passed: !!found }, { name: "konfidens > 0.5", passed: !!found && found.confidence > 0.5 }, { name: "minst ett item med confidence > 0.7", passed: items.some((i) => i.confidence > 0.7) }, ]; }, }, ]; class SilentBudgetStore implements BudgetStore { private spend = 0; async getDailySpendUsd(): Promise { return this.spend; } async incrementDailySpendUsd(amountUsd: number): Promise { this.spend += amountUsd; return this.spend; } } async function main() { const mode = process.env.AAMOS_MODE ?? "mock"; const apiKey = process.env.GEMINI_API_KEY; if (mode !== "gemini" || !apiKey) { console.error("[eval:scan] KRÄVER AAMOS_MODE=gemini + GEMINI_API_KEY."); process.exit(1); } const client = new GeminiAamosClient({ apiKey, model: process.env.GEMINI_MODEL, timeoutMs: process.env.GEMINI_TIMEOUT_MS ? Number(process.env.GEMINI_TIMEOUT_MS) : 60_000, dailyBudgetUsd: process.env.GEMINI_DAILY_BUDGET_USD ? Number(process.env.GEMINI_DAILY_BUDGET_USD) : 10, budgetStore: new SilentBudgetStore(), }); console.log(`[eval:scan] Gemini-lärar-tier live-evaluering – ${CASES.length} fall\n`); let totalChecks = 0; let failedChecks = 0; let totalCostUsd = 0; let totalLatencyMs = 0; for (const evalCase of CASES) { const started = Date.now(); let checks: { name: string; passed: boolean }[] = []; let error: string | null = null; try { const result = await client.runTask( evalCase.locationType === "fridge" ? "ANALYZE_FRIDGE_IMAGE" : "ANALYZE_PANTRY_IMAGE", { imageUrls: evalCase.imageUrls, locationType: evalCase.locationType, marketLocale: evalCase.marketLocale, knownItems: [], }, ); if (result.status !== "ok" || !result.output) { error = `status=${result.status}: ${result.error ?? "okänt fel"}`; checks = [{ name: "Gemini svarade ok", passed: false }]; } else { const items = result.output.items; checks = [{ name: "Gemini svarade ok", passed: true }, ...evalCase.checks(items)]; } if (result.costUsd) totalCostUsd += result.costUsd; if (result.latencyMs) totalLatencyMs += result.latencyMs; } catch (err) { error = (err as Error).message; checks = [{ name: "Gemini svarade ok", passed: false }]; } const ms = Date.now() - started; const passed = checks.every((c) => c.passed); totalChecks += checks.length; failedChecks += checks.filter((c) => !c.passed).length; console.log(`${passed ? "✓" : "✗"} ${evalCase.id} (${ms} ms)`); for (const c of checks) if (!c.passed) console.log(` ✗ ${c.name}`); if (error) console.log(` fel: ${error}`); } console.log(`\n[eval:scan] ${CASES.length} fall, ${totalChecks} kontroller, ${failedChecks} fallerade.`); console.log(`[eval:scan] Total latens: ${totalLatencyMs} ms, total kostnad: ~$${totalCostUsd.toFixed(6)}`); if (failedChecks > 0) { console.log("[eval:scan] UNDERKÄND – åtgärda innan Skiva 1 går till fälttest."); process.exit(1); } console.log("[eval:scan] GODKÄND – fälttest kan börja."); process.exit(0); } main().catch((err) => { console.error("[eval:scan] KRASCH:", err); process.exit(1); });