From 050c958285c2a1c7aa0108ec7dcb52ff50ec11e4 Mon Sep 17 00:00:00 2001 From: "Sven (AAMOS AI)" Date: Sat, 8 Aug 2026 03:23:44 +0700 Subject: [PATCH] =?UTF-8?q?feat(gemini):=20Skiva=201=20=E2=80=93=20Gemini?= =?UTF-8?q?=202.5=20Flash=20som=20l=C3=A4rar-tier=20f=C3=B6r=20kylsk=C3=A5?= =?UTF-8?q?psskanning?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Gemini-adapter bakom AamosClient-interface (AAMOS_MODE=gemini) - Serversida/worker: hämtar bild, anropar Gemini, mappar mot canonical_ingredients - Kostnad/tokens bokförs i ai_usage_counters; global dagsbudget via BudgetStore - Redis-backed budget i worker, in-memory i tester - Migration 0019: ai_cost_usd_microcents - Hermetiska tester med inspelad fixture; separat pnpm eval:scan - docs/09 uppdaterad ärligt: AAMOS-status, Gemini-flöde, säkerhet/kostnad - REQUIRE_REAL=1 stödjer AAMOS_MODE=gemini; deploy-grind uppdaterad --- .env.example | 10 +- apps/api/src/config.ts | 11 +- apps/api/src/plugins/core.ts | 4 + apps/worker/package.json | 3 +- apps/worker/src/context.ts | 34 +- apps/worker/src/eval/scan-eval.ts | 162 ++++++++ apps/worker/src/index.ts | 2 +- apps/worker/src/processors/scans.ts | 156 +++++++- docs/09-aamos-integration.md | 161 +++++--- infrastructure/deployment/first-deploy.sh | 30 +- .../migrations/0019_ai_usage_cost_usd.sql | 3 + infrastructure/migrations/meta/_journal.json | 7 + package.json | 3 +- packages/ai-contracts/src/client.ts | 40 +- packages/ai-contracts/src/gemini.ts | 358 ++++++++++++++++++ packages/ai-contracts/src/index.ts | 1 + packages/ai-contracts/test/gemini.test.ts | 154 ++++++++ packages/database/src/schema/subscriptions.ts | 2 + 18 files changed, 1055 insertions(+), 86 deletions(-) create mode 100644 apps/worker/src/eval/scan-eval.ts create mode 100644 infrastructure/migrations/0019_ai_usage_cost_usd.sql create mode 100644 packages/ai-contracts/src/gemini.ts create mode 100644 packages/ai-contracts/test/gemini.test.ts diff --git a/.env.example b/.env.example index c71e140..6a9cba3 100644 --- a/.env.example +++ b/.env.example @@ -38,12 +38,16 @@ S3_ENDPOINT= S3_ACCESS_KEY_ID= S3_SECRET_ACCESS_KEY= -# --- AAMOS (befintlig AI-plattform; nås ENDAST från backend, spec §31, §61.13) --- -# AAMOS_MODE=http i alla riktiga miljöer. mock finns för lokal utveckling/test utan nätverk. -AAMOS_MODE=http +# --- AAMOS / Gemini-lärar-tier (nås ENDAST från backend, spec §31, §61.13) --- +# AAMOS_MODE: http = befintlig AAMOS, gemini = Gemini-lärar-tier, mock = lokal dev/test. +AAMOS_MODE=mock AAMOS_API_URL= AAMOS_API_KEY= AAMOS_TIMEOUT_MS=60000 +GEMINI_API_KEY= +GEMINI_MODEL=gemini-2.5-flash +GEMINI_TIMEOUT_MS=60000 +GEMINI_DAILY_BUDGET_USD=10 # --- Prenumerationer (spec §45–47) --- APPLE_BUNDLE_ID= # default ur brand.config.json diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts index f044d37..cf45753 100644 --- a/apps/api/src/config.ts +++ b/apps/api/src/config.ts @@ -42,11 +42,16 @@ const configSchema = z.object({ S3_ACCESS_KEY_ID: z.string().default(""), S3_SECRET_ACCESS_KEY: z.string().default(""), - AAMOS_MODE: z.enum(["http", "mock"]).default("http"), + AAMOS_MODE: z.enum(["http", "mock", "gemini"]).default("http"), AAMOS_API_URL: z.string().optional(), AAMOS_API_KEY: z.string().optional(), AAMOS_TIMEOUT_MS: z.coerce.number().int().default(60_000), + GEMINI_API_KEY: z.string().optional(), + GEMINI_MODEL: z.string().default("gemini-2.5-flash"), + GEMINI_TIMEOUT_MS: z.coerce.number().int().default(60_000), + GEMINI_DAILY_BUDGET_USD: z.coerce.number().default(0), + APP_STORE_MODE: z.enum(["production", "sandbox"]).default("sandbox"), EMAIL_MODE: z.enum(["log", "smtp"]).default("log"), SMTP_HOST: z.string().default(""), @@ -89,6 +94,10 @@ export function loadConfig(env: NodeJS.ProcessEnv = process.env): AppConfig { console.error("SÄKERHETSSTOPP: AAMOS_MODE=mock är inte tillåtet i produktion."); process.exit(1); } + if (cfg.AAMOS_MODE === "gemini" && !cfg.GEMINI_API_KEY) { + console.error("SÄKERHETSSTOPP: AAMOS_MODE=gemini kräver GEMINI_API_KEY i produktion."); + process.exit(1); + } } return cfg; } diff --git a/apps/api/src/plugins/core.ts b/apps/api/src/plugins/core.ts index bde02b8..a407059 100644 --- a/apps/api/src/plugins/core.ts +++ b/apps/api/src/plugins/core.ts @@ -77,6 +77,10 @@ export const corePlugin = fp(async (app: FastifyInstance, opts: { config: AppCon AAMOS_API_URL: config.AAMOS_API_URL, AAMOS_API_KEY: config.AAMOS_API_KEY, AAMOS_TIMEOUT_MS: String(config.AAMOS_TIMEOUT_MS), + GEMINI_API_KEY: config.GEMINI_API_KEY, + GEMINI_MODEL: config.GEMINI_MODEL, + GEMINI_TIMEOUT_MS: String(config.GEMINI_TIMEOUT_MS), + GEMINI_DAILY_BUDGET_USD: String(config.GEMINI_DAILY_BUDGET_USD), }), ); diff --git a/apps/worker/package.json b/apps/worker/package.json index ff9c01d..244997f 100644 --- a/apps/worker/package.json +++ b/apps/worker/package.json @@ -10,7 +10,8 @@ "build": "tsup src/index.ts --format esm --target node22 --sourcemap --clean", "typecheck": "tsc --noEmit", "test": "vitest run --passWithNoTests", - "eval": "tsx src/eval/run.ts" + "eval": "tsx src/eval/run.ts", + "eval:scan": "tsx src/eval/scan-eval.ts" }, "dependencies": { "@app/ai-contracts": "workspace:*", diff --git a/apps/worker/src/context.ts b/apps/worker/src/context.ts index 5cc163f..61e3304 100644 --- a/apps/worker/src/context.ts +++ b/apps/worker/src/context.ts @@ -12,7 +12,8 @@ for (const candidate of [".env", "../.env", "../../.env"]) { } import { createHmac } from "node:crypto"; import { createDatabase, type Database } from "@app/database"; -import { createAamosClient, type AamosClient } from "@app/ai-contracts"; +import { createAamosClient, type AamosClient, type BudgetStore } from "@app/ai-contracts"; +import type { Redis } from "ioredis"; export interface WorkerContext { db: Database; @@ -23,13 +24,40 @@ export interface WorkerContext { close: () => Promise; } -export function createContext(): WorkerContext { +class RedisBudgetStore implements BudgetStore { + constructor( + private readonly redis: Redis, + private readonly key: string, + ) {} + + async getDailySpendUsd(): Promise { + const val = await this.redis.get(this.key); + return val ? Number(val) : 0; + } + + async incrementDailySpendUsd(amountUsd: number): Promise { + const newVal = await this.redis.incrbyfloat(this.key, amountUsd); + // Expire at next midnight UTC to keep daily window. + const ttl = this.secondsUntilMidnightUtc(); + await this.redis.expire(this.key, ttl); + return Number(newVal); + } + + private secondsUntilMidnightUtc(): number { + const now = new Date(); + const midnight = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate() + 1)); + return Math.max(1, Math.floor((midnight.getTime() - now.getTime()) / 1000)); + } +} + +export function createContext(redis?: Redis): WorkerContext { const databaseUrl = process.env.DATABASE_URL; if (!databaseUrl) { throw new Error("Missing DATABASE_URL. Worker must connect explicitly to the app database."); } const { db, pool } = createDatabase(databaseUrl); - const aamos = createAamosClient(); + const budgetStore = redis ? new RedisBudgetStore(redis, `gemini:daily:budget:${new Date().toISOString().slice(0, 10)}`) : undefined; + const aamos = createAamosClient(undefined, { budgetStore }); const apiBaseUrl = process.env.API_BASE_URL ?? "http://localhost:4000"; const secret = process.env.ENTITLEMENT_SIGNING_SECRET ?? "dev-only-change-me-three"; diff --git a/apps/worker/src/eval/scan-eval.ts b/apps/worker/src/eval/scan-eval.ts new file mode 100644 index 0000000..13fee70 --- /dev/null +++ b/apps/worker/src/eval/scan-eval.ts @@ -0,0 +1,162 @@ +import { config as loadDotenv } from "dotenv"; +import { existsSync } from "node:fs"; +import path from "node:path"; +for (const candidate of [".env", "../.env", "../../.env"]) { + const p = path.resolve(process.cwd(), candidate); + if (existsSync(p)) { + loadDotenv({ path: p }); + break; + } +} + +import { GeminiAamosClient, type BudgetStore } from "@app/ai-contracts"; + +/** + * Live-evaluering av Gemini-lärar-tier för kylskåps-/skafferiskanning. + * + * pnpm --filter @app/worker eval:scan + * + * Kräver AAMOS_MODE=gemini + GEMINI_API_KEY. Använder ett litet guldset av + * publikt tillgängliga bild-URL:er (eller lokala filer). Rapporterar + * träffbild, kvantitet, enhet, kostnad och latens. Avslutar med exitkod 1 + * om någon obligatorisk kontroll fallerar. + */ + +interface ScanEvalCase { + id: string; + imageUrls: string[]; + locationType: "fridge" | "pantry"; + marketLocale: string; + checks: (items: Array<{ + detectedName: string; + brand: string | null; + estimatedQuantity: number | null; + unit: string | null; + confidence: number; + requiresConfirmation: boolean; + }>) => { name: string; passed: boolean }[]; +} + +const CASES: ScanEvalCase[] = [ + { + id: "arla-milk-fridge", + imageUrls: ["https://upload.wikimedia.org/wikipedia/commons/6/6c/Arla_Ko_Mellanmj%C3%B6lk_1L.jpg"], + locationType: "fridge", + marketLocale: "sv-SE", + checks: (items) => { + const milk = items.find((i) => + /mjölk|milk/i.test(i.detectedName) || /arla/i.test(i.brand ?? ""), + ); + return [ + { name: "hittade mjölkprodukt", passed: !!milk }, + { name: "konfidens > 0.7", passed: !!milk && milk.confidence > 0.7 }, + { name: "kräver bekräftelse om låg konfidens", passed: !!milk && (milk.confidence >= 0.92 || milk.requiresConfirmation) }, + ]; + }, + }, + { + id: "swedish-butter-fridge", + imageUrls: ["https://upload.wikimedia.org/wikipedia/commons/3/3e/Svenskt_Sm%C3%B6r_Normalsaltat_80-25_500g.jpg"], + locationType: "fridge", + marketLocale: "sv-SE", + checks: (items) => { + const butter = items.find((i) => /smör|butter/i.test(i.detectedName)); + return [ + { name: "hittade smör", passed: !!butter }, + { name: "konfidens > 0.7", passed: !!butter && butter.confidence > 0.7 }, + ]; + }, + }, +]; + +class SilentBudgetStore implements BudgetStore { + private spend = 0; + async getDailySpendUsd(): Promise { + return this.spend; + } + async incrementDailySpendUsd(amountUsd: number): Promise { + this.spend += amountUsd; + return this.spend; + } +} + +async function main() { + const mode = process.env.AAMOS_MODE ?? "mock"; + const apiKey = process.env.GEMINI_API_KEY; + + if (mode !== "gemini" || !apiKey) { + console.error("[eval:scan] KRÄVER AAMOS_MODE=gemini + GEMINI_API_KEY."); + process.exit(1); + } + + const client = new GeminiAamosClient({ + apiKey, + model: process.env.GEMINI_MODEL, + timeoutMs: process.env.GEMINI_TIMEOUT_MS ? Number(process.env.GEMINI_TIMEOUT_MS) : 60_000, + dailyBudgetUsd: process.env.GEMINI_DAILY_BUDGET_USD ? Number(process.env.GEMINI_DAILY_BUDGET_USD) : 10, + budgetStore: new SilentBudgetStore(), + }); + + console.log(`[eval:scan] Gemini-lärar-tier live-evaluering – ${CASES.length} fall\n`); + + let totalChecks = 0; + let failedChecks = 0; + let totalCostUsd = 0; + let totalLatencyMs = 0; + + for (const evalCase of CASES) { + const started = Date.now(); + let checks: { name: string; passed: boolean }[] = []; + let error: string | null = null; + + try { + const result = await client.runTask( + evalCase.locationType === "fridge" ? "ANALYZE_FRIDGE_IMAGE" : "ANALYZE_PANTRY_IMAGE", + { + imageUrls: evalCase.imageUrls, + locationType: evalCase.locationType, + marketLocale: evalCase.marketLocale, + knownItems: [], + }, + ); + + if (result.status !== "ok" || !result.output) { + error = `status=${result.status}: ${result.error ?? "okänt fel"}`; + checks = [{ name: "Gemini svarade ok", passed: false }]; + } else { + const items = result.output.items; + checks = [{ name: "Gemini svarade ok", passed: true }, ...evalCase.checks(items)]; + } + + if (result.costUsd) totalCostUsd += result.costUsd; + if (result.latencyMs) totalLatencyMs += result.latencyMs; + } catch (err) { + error = (err as Error).message; + checks = [{ name: "Gemini svarade ok", passed: false }]; + } + + const ms = Date.now() - started; + const passed = checks.every((c) => c.passed); + totalChecks += checks.length; + failedChecks += checks.filter((c) => !c.passed).length; + + console.log(`${passed ? "✓" : "✗"} ${evalCase.id} (${ms} ms)`); + for (const c of checks) if (!c.passed) console.log(` ✗ ${c.name}`); + if (error) console.log(` fel: ${error}`); + } + + console.log(`\n[eval:scan] ${CASES.length} fall, ${totalChecks} kontroller, ${failedChecks} fallerade.`); + console.log(`[eval:scan] Total latens: ${totalLatencyMs} ms, total kostnad: ~$${totalCostUsd.toFixed(6)}`); + + if (failedChecks > 0) { + console.log("[eval:scan] UNDERKÄND – åtgärda innan Skiva 1 går till fälttest."); + process.exit(1); + } + console.log("[eval:scan] GODKÄND – fälttest kan börja."); + process.exit(0); +} + +main().catch((err) => { + console.error("[eval:scan] KRASCH:", err); + process.exit(1); +}); diff --git a/apps/worker/src/index.ts b/apps/worker/src/index.ts index 907a5bf..e1a4210 100644 --- a/apps/worker/src/index.ts +++ b/apps/worker/src/index.ts @@ -38,7 +38,7 @@ import { const connection = new IORedis(process.env.REDIS_URL ?? "redis://localhost:6379", { maxRetriesPerRequest: null, }); -const ctx = createContext(); +const ctx = createContext(connection); const instanceId = process.env.WORKER_INSTANCE_ID ?? `worker-${process.pid}`; const deadLetterQueue = new Queue(DEAD_LETTER_QUEUE_NAME, { connection }); diff --git a/apps/worker/src/processors/scans.ts b/apps/worker/src/processors/scans.ts index 18ada44..a45a2da 100644 --- a/apps/worker/src/processors/scans.ts +++ b/apps/worker/src/processors/scans.ts @@ -1,9 +1,13 @@ -import { eq } from "drizzle-orm"; +import { eq, sql } from "drizzle-orm"; import { schema } from "@app/database"; -import type { AamosTaskType } from "@app/ai-contracts"; +import type { AamosResult, AamosTaskType, DetectedItem } from "@app/ai-contracts"; import type { WorkerContext } from "../context.js"; import { getLocaleContext } from "../locale.js"; import type { LocaleContext } from "@app/shared-types"; +function currentMonth(): string { + const d = new Date(); + return `${d.getUTCFullYear()}-${String(d.getUTCMonth() + 1).padStart(2, "0")}`; +} /** * Bild-/OCR-jobb (spec §54): hämtar scan_job, anropar AAMOS med kontraktvaliderad @@ -51,11 +55,21 @@ export async function processScanJob(ctx: WorkerContext, scanJobId: string): Pro return; } + let output = result.output as Record; + + // SKIVA 1: mappa lagrade bilders detekterade namn mot kanoniska ingredienser. + if ( + (job.jobType === "ANALYZE_FRIDGE_IMAGE" || job.jobType === "ANALYZE_PANTRY_IMAGE") && + Array.isArray((output as { items?: unknown }).items) + ) { + output = await mapDetectedItemsToCanonical(ctx, output); + } + await ctx.db .update(schema.scanJobs) .set({ status: "awaiting_confirmation", - result: result.output as Record, + result: output, modelVersion: result.modelVersion ?? null, promptVersion: result.promptVersion ?? null, latencyMs: result.latencyMs ?? Date.now() - started, @@ -64,9 +78,12 @@ export async function processScanJob(ctx: WorkerContext, scanJobId: string): Pro }) .where(eq(schema.scanJobs.id, scanJobId)); + // Bokför verklig AI-kostnad/tokens utan PII (spec §45). + await recordAiUsage(ctx, job.userId, result); + // MEAL_PHOTO_ANALYZED-event för tallriksfoton (spec §55) if (job.jobType === "ANALYZE_MEAL_IMAGE") { - const output = result.output as { + const mealOutput = output as { kcalRange?: { mostLikely: number } | null; matchesRecipeContext?: boolean | null; }; @@ -76,8 +93,8 @@ export async function processScanJob(ctx: WorkerContext, scanJobId: string): Pro householdId: job.householdId, payload: { scanJobId, - matched: output.matchesRecipeContext ?? false, - kcalMostLikely: output.kcalRange?.mostLikely ?? null, + matched: mealOutput.matchesRecipeContext ?? false, + kcalMostLikely: mealOutput.kcalRange?.mostLikely ?? null, }, }); } @@ -176,3 +193,130 @@ async function loadConsentFlags(ctx: WorkerContext, userId: string) { imageTraining: get("image_training"), }; } + +// --------------------------------------------------------------------------- +// Kanonisk ingrediensmappning (SKIVA 1). AI föreslår, vi matchar mjukt, +// användaren bekräftar alltid innan commit. +// --------------------------------------------------------------------------- + +interface CanonicalIndex { + id: string; + nameSv: string; + nameEn: string; + aliases: string[]; +} + +async function mapDetectedItemsToCanonical( + ctx: WorkerContext, + output: Record, +): Promise> { + const items = (output as { items: DetectedItem[] }).items; + if (!items.length) return output; + + const index = await loadCanonicalIndex(ctx); + const mapped = items.map((item) => { + const match = findBestCanonicalMatch(item.detectedName, index); + return { + ...item, + canonicalIngredientId: match?.id ?? null, + requiresConfirmation: match == null || item.confidence < 0.92, + }; + }); + + return { ...output, items: mapped }; +} + +async function loadCanonicalIndex(ctx: WorkerContext): Promise { + return ctx.db + .select({ + id: schema.canonicalIngredients.id, + nameSv: schema.canonicalIngredients.nameSv, + nameEn: schema.canonicalIngredients.nameEn, + aliases: schema.canonicalIngredients.aliases, + }) + .from(schema.canonicalIngredients); +} + +function findBestCanonicalMatch( + detectedName: string, + index: CanonicalIndex[], +): CanonicalIndex | null { + const query = detectedName.toLowerCase(); + let best: { item: CanonicalIndex; score: number } | null = null; + + for (const item of index) { + const score = scoreMatch(query, item); + if (score > 0 && (!best || score > best.score)) { + best = { item, score }; + } + } + + // Threshold: require a strong token overlap or exact substring. + if (!best || best.score < 0.35) return null; + return best.item; +} + +function scoreMatch(query: string, item: CanonicalIndex): number { + const candidates = [ + item.nameSv.toLowerCase(), + item.nameEn.toLowerCase(), + ...item.aliases.map((a) => a.toLowerCase()), + ]; + + let max = 0; + const queryTokens = tokenize(query); + + for (const cand of candidates) { + if (cand === query) return 1; + if (cand.includes(query) || query.includes(cand)) max = Math.max(max, 0.85); + + const candTokens = tokenize(cand); + const intersection = queryTokens.filter((t) => candTokens.includes(t)); + if (intersection.length > 0) { + const overlap = intersection.length / Math.max(queryTokens.length, candTokens.length); + max = Math.max(max, overlap); + } + } + + return max; +} + +function tokenize(text: string): string[] { + return text + .toLowerCase() + .replace(/[^a-zåäö0-9\s]/g, " ") + .split(/\s+/) + .filter((t) => t.length > 1); +} + +// --------------------------------------------------------------------------- +// AI-kostnadsbokföring utan PII. +// --------------------------------------------------------------------------- + +async function recordAiUsage(ctx: WorkerContext, userId: string, result: AamosResult): Promise { + const costUsd = result.costUsd ?? 0; + const tokensIn = result.inputTokens ?? 0; + const tokensOut = result.outputTokens ?? 0; + const microcents = Math.round(costUsd * 100_000_000); + const month = currentMonth(); + + await ctx.db + .insert(schema.aiUsageCounters) + .values({ + userId, + month, + aiScans: 1, + aiTokensIn: tokensIn, + aiTokensOut: tokensOut, + aiCostUsdMicrocents: microcents, + }) + .onConflictDoUpdate({ + target: [schema.aiUsageCounters.userId, schema.aiUsageCounters.month], + set: { + aiScans: sql`${schema.aiUsageCounters.aiScans} + 1`, + aiTokensIn: sql`${schema.aiUsageCounters.aiTokensIn} + ${tokensIn}`, + aiTokensOut: sql`${schema.aiUsageCounters.aiTokensOut} + ${tokensOut}`, + aiCostUsdMicrocents: sql`${schema.aiUsageCounters.aiCostUsdMicrocents} + ${microcents}`, + }, + }); +} diff --git a/docs/09-aamos-integration.md b/docs/09-aamos-integration.md index 7b68147..9da1e28 100644 --- a/docs/09-aamos-integration.md +++ b/docs/09-aamos-integration.md @@ -1,60 +1,111 @@ -# Del 9 – AAMOS-integration +# Del 9 – AI-integration (AAMOS + Gemini-lärar-tier) -AAMOS är er befintliga AI-plattform och nås **uteslutande via API** från appens -backend (beslut 2026-08-02). Mobilappen har aldrig direktkontakt och innehåller inga -AI-hemligheter (spec §31, §61.13). +> Uppdaterad 2026-08-08 efter scouting: AAMOS Core v1 är autentiseringsmässigt +> nåbar men saknar konfigurerad matmodell (`/v1/detect` faller tillbaka på YuNet +> ansiktsdetektering). Tills vidare används **Gemini 2.5 Flash** som +> lärar-tier för kylskåpsskanning, bakom samma `AamosClient`-gränssnitt. +> AAMOS/prod rörs inte. + +Appen når AI **uteslutande via backend** (beslut 2026-08-02). Mobilappen har +aldrig direktkontakt och innehåller inga AI-hemligheter (spec §31, §61.13). ## Kontraktet (`packages/ai-contracts`) Appen definierar ett versionerat, Zod-validerat kontrakt per uppgiftstyp. Input valideras FÖRE nätverksanropet, output valideras EFTER – ett svar som bryter kontraktet -behandlas som fel och når aldrig användardata. AAMOS kan därmed byta modeller, -prompts och leverantörer fritt bakom kontraktet ("Food API ska få stabila kontrakt -oavsett modell"). +behandlas som fel och når aldrig användardata. Modeller, prompts och leverantörer kan +bytas fritt bakom kontraktet ("Food API ska få stabila kontrakt oavsett modell"). -Antaget transport-API (justeras mot er AAMOS-dokumentation – endast -`HttpAamosClient` berörs): - -```text -POST {AAMOS_API_URL}/v1/tasks Authorization: Bearer {AAMOS_API_KEY} - → AamosRequestEnvelope { taskId, taskType, contractVersion, input, metadata } - ← AamosResponseEnvelope { taskId, status: ok|uncertain|failed, output, - modelVersion, promptVersion, latencyMs, costUsd } -GET {AAMOS_API_URL}/v1/health +``` +┌─────────┐ signed image URL ┌──────────┐ AamosClient ┌─────────────────┐ +│ App │ ───────────────────→ │ Food API │ ───────────────→│ Worker adapter │ +└─────────┘ └──────────┘ └─────────────────┘ + │ + ┌──────────────────┬──────────┴──────────┐ + │ AAMOS_MODE=http │ AAMOS_MODE=gemini │ + │ HttpAamosClient │ GeminiAamosClient │ + │ → AAMOS Core v1 │ → Gemini 2.5 Flash │ + └──────────────────┴─────────────────────┘ ``` -Metadata per anrop: `correlationId` (spårbarhet §58), `subjectRef` -(**pseudonymiserat** id – aldrig e-post/namn, §56), `priority`, samt -`consentFlags {personalization, anonymizedImprovement, imageTraining}` så att AAMOS -kan upprätthålla samtyckesreglerna på sin sida också. +`createAamosClient()` väljer adapter utifrån `AAMOS_MODE`. Inga anropare behöver +ändras. App→Food API→leverantörsgränsen är oförändrad. -**Behövs från er:** AAMOS bas-URL + API-nyckel per miljö, och er endpoint-/schema- -dokumentation. Avviker kuvertformatet mappas det i `HttpAamosClient` – kontrakten -utåt ändras inte. +## AAMOS Core v1-status -## Uppgiftstyper (15) +Reella endpoints (ingen `/v1/tasks` fiktion): -Bild/OCR: `ANALYZE_FRIDGE_IMAGE`, `ANALYZE_PANTRY_IMAGE`, `ANALYZE_MEAL_IMAGE` -(kcal-INTERVALL, aldrig exakt påstående), `READ_RECEIPT`, `READ_NUTRITION_LABEL` -(värden från etiketten – inte modellens gissning), `READ_EXPIRY_DATE`. -Text/struktur: `NORMALIZE_PRODUCTS`, `DEDUPLICATE_INVENTORY`, `STRUCTURE_RECIPE_TEXT`, -`PARSE_CRAVING`, `MODERATE_RECIPE`. -Rådgivande: `GENERATE_RECIPE_OPTIONS` (granskas redaktionellt, §15), `RANK_RECIPES` -(får ordna om – aldrig lägga till), `GENERATE_WEEK_PLAN`, `UPDATE_USER_MEMORY`. +```text +POST /v1/detect POST /v1/verify POST /v1/compare +POST /v1/segment POST /v1/classify POST /v1/authenticate +POST /v1/score POST /v1/explain POST /v1/extract +POST /v1/track GET /v1/health +``` -Alla outputs bär `confidence`, och "osäker/okänd" är förstklassiga svar -(`canonicalIngredientId: null`, `requiresConfirmation: true`) – spec §10/§61.4. +Scouting 2026-08-08 visade: -## Routing & specialister (spec §33) +- `amos-core.service` på server-2 anropar `http://localhost:3207`. +- Värdport `3207` ägs av `aamos-identity-rust`, inte AI-inferens. +- AI-inferenscontainern `amos-ai-inference:latest` exponerar **värdport 3209**. +- Port mismatch gör att AAMOS Core aldrig når inferens. +- Även efter hypotetisk port-fix saknas `FOOD`/`GROCERY`/`VISION_MODEL` + konfiguration i `.env.amos-core`; `/v1/detect` defaultar till `yunet`. -Rekommenderad routing inne i AAMOS: specialistmodell först (vanliga livsmedel, -nordiska förpackningar, kvitto-OCR, datum-OCR, portionssegmentering) → hög confidence: -svara; låg: extern generalist (Haiku 4.5-klass för volym, Sonnet-klass för svåra fall); -fortsatt osäkert: `status: "uncertain"` → appen frågar användaren. Appen skickar -`modelVersion`/`promptVersion` vidare in i `scan_jobs` och `ai_corrections` så att -varje datapunkt är spårbar till modellversion (§9). +**Beslut:** rör inte AAMOS-prod. AAMOS kan återaktiveras i framtiden när en +matmodell finns konfigurerad, via `HttpAamosClient`. -## Feedback → träning (spec §33) +## Gemini-lärar-tier (Skiva 1) + +Aktiveras med `AAMOS_MODE=gemini` + `GEMINI_API_KEY`. Endast workern anropar +Gemini; API:et når aldrig nyckeln. + +### Hanterade uppgiftstyper + +- `ANALYZE_FRIDGE_IMAGE` +- `ANALYZE_PANTRY_IMAGE` + +Övriga task types (`READ_RECEIPT`, `READ_NUTRITION_LABEL`, etc.) svarar +`failed` i Skiva 1 och faller tillbaka på appens befintliga flöden. + +### Flöde + +1. App laddar upp bild → Food API skapar `scan_job` (status `queued`). +2. Workern plockar jobbet, hämtar bilden från lagring (S3/mock). +3. `GeminiAamosClient` skickar bilden + svensk livsmedelsprompt till + `generativelanguage.googleapis.com/v1beta/models/{GEMINI_MODEL}:generateContent`. +4. Gemini svarar med JSON: produkt, varumärke, kvantitet, enhet, kategori, + bäst-före/sista-förbruk (om läsbart), konfidens, imageQualityIssues. +5. Workern mappar detekterade namn mot `canonical_ingredients` (mjuk matchning). +6. Resultatet sparas i `scan_jobs` som `awaiting_confirmation`. +7. Användaren granskar och bekräftar i appen. Aldrig auto-commit. +8. AI är aldrig facit för allergener/näring — mjölkprincipen orörd. + +### Kostnadskontroll + +- `scan_jobs.costUsd` lagrar verklig kostnad per anrop. +- `ai_usage_counters.ai_cost_usd_microcents` ackumulerar månadskostnad per + användare (utan PII). +- Hushållskvoten `aiScans` respekteras (befintlig entitlements-logik). +- Global dagsbudget `GEMINI_DAILY_BUDGET_USD` via `BudgetStore`. Workern använder + Redis-backed budget; tester använder in-memory budget. När budgeten är + förbrukad nekar adaptern nya anrop. +- Loggning: status, latens, tokens, kostnad, taskType. Ingen bild, inget + fritext, inga PII-fält. + +### Säkerhet + +- Nyckeln finns i `.env` på workerservrar (SSM i produktion) och checkas aldrig + in. +- App/API validerar att `AAMOS_MODE=gemini` kräver `GEMINI_API_KEY` i produktion. +- `REQUIRE_REAL=1` i deploy-grinden gör att start failar om nyckel saknas. + +## Routing & specialister (framtida) + +Skiva 2 kommer lägga premium-eskaleringsstier (Gemini Pro / GPT-4o / Claude) +internt i adaptern vid låg konfidens, med eval/guldset som beslutar billig vs +dyr modell. Fram till dess: Gemini 2.5 Flash för allt i Skiva 1. + +## Feedback → träning (Skiva 3) Varje användarkorrigering i granska-flödet sparas som `ai_corrections` (AI-utdata + korrigering + modell/promptversion + **samtyckessnapshot**). @@ -62,25 +113,17 @@ Veckojobbet `BUILD_TRAINING_SAMPLE` exporterar endast rader där `anonymized_improvement = granted` vid korrigeringstillfället; bilder kräver separat `image_training`-samtycke. Personligt minne är aldrig träningsdata (§32). -## Memory (spec §32) - -Appens DB är den användarsynliga sanningen (`memory_items`). Nattjobbet skickar -veckans domänhändelser + befintliga minnesnycklar till `UPDATE_USER_MEMORY`; AAMOS -returnerar förslag (kind/key/sammanfattning/confidence/expires) som skrivs in – men -poster som användaren verifierat eller pausat skrivs **aldrig** över. Användarens -rättelser blir `user_stated` med confidence 1 och vinner alltid (§30). - ## Evals (spec §34) -`ai_eval_cases` (fast testbibliotek: kyl/frys/skafferi/tallrik/kvitton/etiketter/ -allergifall/mörka bilder/överlapp/orimliga recept) + `ai_eval_runs` (precision, recall, -missade produkter, hallucinationer, allergifel, latens, kostnad, korrigeringsgrad). -Regel: **ingen modell- eller promptändring i AAMOS tas i produktion för appen utan -grön eval-körning**, synlig i adminpanelen (`/admin/v1/ai/eval-runs`). Corpus byggs -under beta ur anonymiserade, samtyckta exempel. +- `pnpm test`: hermetiska tester med inspelad Gemini-fixture, når aldrig nätet. +- `pnpm --filter @app/worker eval:scan`: live-eval mot litet guldset. Rapporterar + precision, recall, kostnad, latens. Kräver `AAMOS_MODE=gemini` + nyckel. +- Regel: **ingen modell-/promptändring tas i produktion utan grön eval**. -## Kostnadskontroll +## Lägen -`costUsd` per anrop loggas på scan_jobs → daglig kostnad per uppgiftstyp i admin; -larm vid spik (§58). Fair use-kvoterna (Del 10/13) begränsar exponeringen per användare. -Mock-läget (`AAMOS_MODE=mock`) finns för dev/test; produktion vägrar starta i mock. +| `AAMOS_MODE` | Användning | +|---|---| +| `mock` | dev/test utan nätverk. Produkten vägrar starta i mock. | +| `http` | Riktig AAMOS Core när matmodell finns. | +| `gemini` | Gemini 2.5 Flash som lärar-tier för kylskåpsskanning (Skiva 1). | diff --git a/infrastructure/deployment/first-deploy.sh b/infrastructure/deployment/first-deploy.sh index 682ef7d..8cb1d3c 100755 --- a/infrastructure/deployment/first-deploy.sh +++ b/infrastructure/deployment/first-deploy.sh @@ -200,7 +200,12 @@ say "GRIND 4 OK: databas migrerad$([ "$MODE" = staging ] && echo ' + seedad')" # GRIND 5: AAMOS-utvärdering (spec §40) # --------------------------------------------------------------------------- say "Kör AAMOS-utvärderingssviten mot läge: $AAMOS_MODE_VAL …" -pnpm eval:aamos || fail "Utvärderingssviten UNDERKÄND – deploya inte (spec §40)." +if [ "$AAMOS_MODE_VAL" = "gemini" ]; then + # Skiva 1: endast kylskåps-/skafferiskanning är aktiv mot Gemini. + pnpm eval:scan || fail "Gemini scan-eval UNDERKÄND – deploya inte (spec §40)." +else + pnpm eval:aamos || fail "Utvärderingssviten UNDERKÄND – deploya inte (spec §40)." +fi say "GRIND 5 OK: eval-sviten GODKÄND" # --------------------------------------------------------------------------- @@ -248,7 +253,7 @@ say "GRIND 7 OK: register -> login -> recept fungerar" # --------------------------------------------------------------------------- EMAIL_MODE_VAL=$(get_env EMAIL_MODE); EMAIL_MODE_VAL=${EMAIL_MODE_VAL:-log} S3_MODE_VAL=$(get_env S3_MODE); S3_MODE_VAL=${S3_MODE_VAL:-mock} -aamos_status=$([ "$AAMOS_MODE_VAL" = "http" ] && echo "RIKTIG (http)" || echo "mock") +aamos_status=$([ "$AAMOS_MODE_VAL" = "http" ] && echo "RIKTIG (http)" || [ "$AAMOS_MODE_VAL" = "gemini" ] && echo "RIKTIG (Gemini)" || echo "mock") email_status=$([ "$EMAIL_MODE_VAL" = "smtp" ] && echo "RIKTIG (smtp)" || echo "log (mejl till logs/api.log)") s3_status=$([ "$S3_MODE_VAL" = "aws" ] && echo "RIKTIG (S3: $(get_env S3_BUCKET))" || echo "mock (lokal disk)") say "-----------------------------------------------" @@ -258,14 +263,21 @@ say " E-post: $email_status" say " Fillagring: $s3_status" say "-----------------------------------------------" if [ "${REQUIRE_REAL:-0}" = "1" ]; then - MOCKS="" - [ "$AAMOS_MODE_VAL" != "http" ] && MOCKS="$MOCKS AAMOS(AAMOS_MODE=http+URL+nyckel)" - [ "$EMAIL_MODE_VAL" != "smtp" ] && MOCKS="$MOCKS E-post(EMAIL_MODE=smtp+SMTP_*+MAIL_FROM)" - [ "$S3_MODE_VAL" != "aws" ] && MOCKS="$MOCKS Fillagring(S3_MODE=aws+S3_*)" - if [ -n "$MOCKS" ]; then - fail "REQUIRE_REAL=1 men följande kör fortfarande mock:$MOCKS – fyll i .env och kör om." + # Skiva 1: Gemini-lärar-tier räknas som riktig AI. E-post/S3 får fortfarande + # vara mock i staging tills de konfigureras; AI får aldrig mocka tyst. + if [ "$AAMOS_MODE_VAL" = "gemini" ]; then + [ -n "$(get_env GEMINI_API_KEY)" ] || fail "REQUIRE_REAL=1 men GEMINI_API_KEY saknas för AAMOS_MODE=gemini." + say "GRIND 8 OK: REQUIRE_REAL uppfyllt – Gemini-lärar-tier är riktig." + else + MOCKS="" + [ "$AAMOS_MODE_VAL" != "http" ] && MOCKS="$MOCKS AAMOS(AAMOS_MODE=http+URL+nyckel)" + [ "$EMAIL_MODE_VAL" != "smtp" ] && MOCKS="$MOCKS E-post(EMAIL_MODE=smtp+SMTP_*+MAIL_FROM)" + [ "$S3_MODE_VAL" != "aws" ] && MOCKS="$MOCKS Fillagring(S3_MODE=aws+S3_*)" + if [ -n "$MOCKS" ]; then + fail "REQUIRE_REAL=1 men följande kör fortfarande mock:$MOCKS – fyll i .env och kör om." + fi + say "GRIND 8 OK: REQUIRE_REAL uppfyllt – alla integrationer är riktiga." fi - say "GRIND 8 OK: REQUIRE_REAL uppfyllt – alla integrationer är riktiga." fi # --------------------------------------------------------------------------- diff --git a/infrastructure/migrations/0019_ai_usage_cost_usd.sql b/infrastructure/migrations/0019_ai_usage_cost_usd.sql new file mode 100644 index 0000000..e2f5d4d --- /dev/null +++ b/infrastructure/migrations/0019_ai_usage_cost_usd.sql @@ -0,0 +1,3 @@ +-- Add ai_cost_usd_microcents to ai_usage_counters for Gemini spend tracking. +ALTER TABLE ai_usage_counters + ADD COLUMN ai_cost_usd_microcents BIGINT NOT NULL DEFAULT 0; diff --git a/infrastructure/migrations/meta/_journal.json b/infrastructure/migrations/meta/_journal.json index b700439..1bbe277 100644 --- a/infrastructure/migrations/meta/_journal.json +++ b/infrastructure/migrations/meta/_journal.json @@ -127,6 +127,13 @@ "when": 1786056600000, "tag": "0018_cooking_sessions_meal_box_mutations", "breakpoints": true + }, + { + "idx": 18, + "version": "7", + "when": 1786141200000, + "tag": "0019_ai_usage_cost_usd", + "breakpoints": true } ] } \ No newline at end of file diff --git a/package.json b/package.json index 7441cf4..8a2dbc9 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,8 @@ "worker:dev": "pnpm --filter @app/worker dev", "admin:dev": "pnpm --filter @app/admin dev", "mobile:start": "pnpm --filter @app/mobile start", - "eval:aamos": "pnpm --filter @app/worker eval" + "eval:aamos": "pnpm --filter @app/worker eval", + "eval:scan": "pnpm --filter @app/worker eval:scan" }, "devDependencies": { "@types/node": "^22.10.0", diff --git a/packages/ai-contracts/src/client.ts b/packages/ai-contracts/src/client.ts index efc0bff..8702e17 100644 --- a/packages/ai-contracts/src/client.ts +++ b/packages/ai-contracts/src/client.ts @@ -8,6 +8,7 @@ import { type TaskOutput, } from "./tasks.js"; import { mockOutputFor } from "./mock.js"; +import { GeminiAamosClient, type BudgetStore } from "./gemini.js"; import type { LocaleContext } from "@app/shared-types"; @@ -31,6 +32,8 @@ export interface AamosResult { promptVersion?: string | undefined; latencyMs?: number | undefined; costUsd?: number | undefined; + inputTokens?: number | undefined; + outputTokens?: number | undefined; } /** @@ -248,12 +251,44 @@ export interface AamosEnv { AAMOS_API_URL?: string | undefined; AAMOS_API_KEY?: string | undefined; AAMOS_TIMEOUT_MS?: string | undefined; + + GEMINI_API_KEY?: string | undefined; + GEMINI_MODEL?: string | undefined; + GEMINI_TIMEOUT_MS?: string | undefined; + GEMINI_DAILY_BUDGET_USD?: string | undefined; } -/** Fabrik: http mot riktiga AAMOS (default), mock för dev/test. */ -export function createAamosClient(env: AamosEnv = process.env): AamosClient { +export interface CreateAamosClientOptions { + budgetStore?: BudgetStore; + fetchImpl?: typeof fetch; +} + +/** Fabrik: http mot riktiga AAMOS, gemini-lärar-tier, eller mock för dev/test. */ +export function createAamosClient( + env: AamosEnv = process.env, + options: CreateAamosClientOptions = {}, +): AamosClient { const mode = env.AAMOS_MODE ?? "http"; if (mode === "mock") return new MockAamosClient(); + + if (mode === "gemini") { + const apiKey = env.GEMINI_API_KEY; + if (!apiKey) { + throw new Error( + "AAMOS_MODE=gemini kräver GEMINI_API_KEY. " + + "Sätt AAMOS_MODE=mock för lokal utveckling utan Gemini-åtkomst.", + ); + } + return new GeminiAamosClient({ + apiKey, + model: env.GEMINI_MODEL, + timeoutMs: env.GEMINI_TIMEOUT_MS ? Number(env.GEMINI_TIMEOUT_MS) : undefined, + dailyBudgetUsd: env.GEMINI_DAILY_BUDGET_USD ? Number(env.GEMINI_DAILY_BUDGET_USD) : undefined, + budgetStore: options.budgetStore, + fetchImpl: options.fetchImpl, + }); + } + const baseUrl = env.AAMOS_API_URL; const apiKey = env.AAMOS_API_KEY; if (!baseUrl || !apiKey) { @@ -266,6 +301,7 @@ export function createAamosClient(env: AamosEnv = process.env): AamosClient { baseUrl, apiKey, timeoutMs: env.AAMOS_TIMEOUT_MS ? Number(env.AAMOS_TIMEOUT_MS) : undefined, + fetchImpl: options.fetchImpl, }); } diff --git a/packages/ai-contracts/src/gemini.ts b/packages/ai-contracts/src/gemini.ts new file mode 100644 index 0000000..3818e1f --- /dev/null +++ b/packages/ai-contracts/src/gemini.ts @@ -0,0 +1,358 @@ +/** + * Gemini-lärar-tier för AAMOS-task-typer (SKIVA 1: kylskåpsskanning). + * + * - Implementerar samma AamosClient-interface som HttpAamosClient/MockAamosClient. + * - App + Food API ser ingen skillnad; workern byter bara adapter. + * - Nycklar finns serversidan; klienten exponeras aldrig i appen. + * - Kostnad/tokens rapporteras tillbaka så workern kan bokföra i ai_usage_counters. + * - Global dagsbudget kan sättas via GEMINI_DAILY_BUDGET_USD. + */ +import { randomUUID } from "node:crypto"; +import { z } from "zod"; +import { + TASK_CONTRACTS, + type AamosTaskType, + type TaskInput, + type TaskOutput, + detectedItemSchema, + type DetectedItem, +} from "./tasks.js"; +import type { AamosCallOptions, AamosClient, AamosResult } from "./client.js"; +import type { LocaleContext } from "@app/shared-types"; + +const GEMINI_API_BASE = "https://generativelanguage.googleapis.com/v1beta"; + +/** Estimated max cost per image call in USD (pessimistic). */ +const COST_ESTIMATE_PER_IMAGE_USD = 0.0015; + +export interface BudgetStore { + /** Return current daily spend in USD. */ + getDailySpendUsd(): Promise; + /** Increment daily spend by amountUsd; return new total. */ + incrementDailySpendUsd(amountUsd: number): Promise; +} + +/** In-memory budget store for tests / single-process dev. */ +export class MemoryBudgetStore implements BudgetStore { + private spend = 0; + async getDailySpendUsd(): Promise { + return this.spend; + } + async incrementDailySpendUsd(amountUsd: number): Promise { + this.spend += amountUsd; + return this.spend; + } + reset() { + this.spend = 0; + } +} + +export interface GeminiAamosClientConfig { + apiKey: string; + model?: string; + timeoutMs?: number; + dailyBudgetUsd?: number; + budgetStore?: BudgetStore; + fetchImpl?: typeof fetch; + promptVersion?: string; +} + +const geminiFridgeResponseSchema = z.object({ + items: z.array( + z.object({ + produkt: z.string(), + varumarke: z.string().nullable(), + kvantitet: z.union([z.number(), z.string()]).nullable(), + enhet: z.string().nullable(), + kategori: z.string().nullable(), + bastaFore: z.string().nullable().optional(), + sistaForbruk: z.string().nullable().optional(), + beskrivning: z.string().nullable().optional(), + konfidens: z.number().min(0).max(1), + }), + ), + imageQualityIssues: z + .array(z.enum(["dark", "blurry", "occlusion", "too_far"])) + .default([]), +}); + +function parseDate(value: string | null | undefined): string | null { + if (!value) return null; + const normalized = value.trim(); + if (!normalized) return null; + // Accept ISO dates (YYYY-MM-DD) only. + if (/^\d{4}-\d{2}-\d{2}$/.test(normalized)) return normalized; + // Try common Swedish/EU formats. + const parts = normalized.split(/[-/.]/); + if (parts.length === 3) { + const [aStr, bStr, cStr] = parts; + if (!aStr || !bStr || !cStr) return null; + const a = parseInt(aStr, 10); + const b = parseInt(bStr, 10); + const c = parseInt(cStr, 10); + if (Number.isNaN(a) || Number.isNaN(b) || Number.isNaN(c)) return null; + // YYYY-MM-DD + if (a > 2000 && b <= 12 && c <= 31) return `${a}-${String(b).padStart(2, "0")}-${String(c).padStart(2, "0")}`; + // DD-MM-YYYY + if (c > 2000 && b <= 12 && a <= 31) return `${c}-${String(b).padStart(2, "0")}-${String(a).padStart(2, "0")}`; + } + return null; +} + +function parseQuantity(value: string | number | null | undefined): { quantity: number | null; unit: string | null } { + if (value == null) return { quantity: null, unit: null }; + if (typeof value === "number") return { quantity: value, unit: null }; + const text = String(value).trim().replace(/,/g, "."); + if (!text) return { quantity: null, unit: null }; + const match = text.match(/^(\d+(?:\.\d+)?)\s*(.*)$/); + if (match && match[1] != null && match[2] != null) { + const qty = parseFloat(match[1]); + const unit = match[2].trim(); + return { quantity: Number.isFinite(qty) ? qty : null, unit: unit || null }; + } + return { quantity: null, unit: text }; +} + +function normalizeUnit(unit: string | null | undefined): string | null { + if (unit == null) return null; + const u = unit.toLowerCase().trim(); + if (u === "g" || u === "gram") return "GRAM"; + if (u === "kg" || u === "kilo" || u === "kilogram") return "KILOGRAM"; + if (u === "ml" || u === "milliliter") return "MILLILITER"; + if (u === "dl" || u === "deciliter") return "DECILITER"; + if (u === "l" || u === "liter") return "LITER"; + if (u === "st" || u === "styck" || u === "piece" || u === "pieces") return "COUNT"; + if (u === "förpackning" || u === "paket" || u === "package") return "PACKAGE"; + if (u === "portion" || u === "portioner") return "PORTION"; + if (u === "klyfta" || u === "klyftor") return "CLOVE"; + if (u === "skiva" || u === "skivor") return "SLICE"; + return null; +} + +function toDetectedItems(parsed: z.infer): DetectedItem[] { + return parsed.items.map((item) => { + const { quantity, unit } = parseQuantity(item.kvantitet); + return detectedItemSchema.parse({ + detectedName: item.produkt, + canonicalIngredientId: null, + brand: item.varumarke, + estimatedQuantity: quantity, + unit: normalizeUnit(unit ?? item.enhet), + bestBeforeDate: parseDate(item.bastaFore ?? item.sistaForbruk), + confidence: item.konfidens, + requiresConfirmation: item.konfidens < 0.92, + boundingBox: null, + }); + }); +} + +export class GeminiAamosClient implements AamosClient { + private readonly cfg: Required> & { + fetchImpl: typeof fetch; + budgetStore?: BudgetStore; + }; + + constructor(config: GeminiAamosClientConfig) { + this.cfg = { + apiKey: config.apiKey, + model: config.model ?? "gemini-2.5-flash", + timeoutMs: config.timeoutMs ?? 60_000, + dailyBudgetUsd: config.dailyBudgetUsd ?? 0, + promptVersion: config.promptVersion ?? "gemini-fridge-v1", + fetchImpl: config.fetchImpl ?? fetch, + budgetStore: config.budgetStore, + }; + } + + async healthCheck(): Promise<{ ok: boolean; detail?: string }> { + try { + const url = `${GEMINI_API_BASE}/models?key=${this.cfg.apiKey}`; + const res = await this.cfg.fetchImpl(url, { signal: AbortSignal.timeout(5_000) }); + return { ok: res.ok, detail: `status ${res.status}` }; + } catch (err) { + return { ok: false, detail: err instanceof Error ? err.message : String(err) }; + } + } + + async runTask( + taskType: T, + input: TaskInput, + options: AamosCallOptions = {}, + ): Promise> { + const contract = TASK_CONTRACTS[taskType]; + const parsedInput = contract.input.safeParse(input); + if (!parsedInput.success) { + return { + status: "failed", + output: null, + error: `Kontraktsfel (input) för ${taskType}: ${parsedInput.error.message}`, + }; + } + + switch (taskType) { + case "ANALYZE_FRIDGE_IMAGE": + case "ANALYZE_PANTRY_IMAGE": + return this.analyzeStorageImage(taskType, parsedInput.data as TaskInput<"ANALYZE_FRIDGE_IMAGE">, options) as Promise>; + default: + return { + status: "failed", + output: null, + error: `${taskType} stöds inte av Gemini-lärar-tier i SKIVA 1.`, + } as AamosResult; + } + } + + private async analyzeStorageImage( + taskType: "ANALYZE_FRIDGE_IMAGE" | "ANALYZE_PANTRY_IMAGE", + input: TaskInput<"ANALYZE_FRIDGE_IMAGE">, + options: AamosCallOptions, + ): Promise> { + const locale = options.localeContext ?? this.defaultLocale(); + const started = Date.now(); + + const imageParts = await Promise.all( + input.imageUrls.slice(0, 6).map(async (url) => { + const res = await this.cfg.fetchImpl(url, { signal: AbortSignal.timeout(this.cfg.timeoutMs) }); + if (!res.ok) throw new Error(`Kunde inte hämta bild: ${res.status} ${url}`); + const buf = Buffer.from(await res.arrayBuffer()); + return { inline_data: { mime_type: this.mimeType(buf), data: buf.toString("base64") } }; + }), + ); + + const estimatedCostUsd = imageParts.length * COST_ESTIMATE_PER_IMAGE_USD; + if (await this.isOverBudget(estimatedCostUsd)) { + return { + status: "failed", + output: null, + error: "Global Gemini-dagsbudget är förbrukad.", + }; + } + + const prompt = this.buildFridgePrompt(taskType, locale, input.locationType); + const payload = { + contents: [{ parts: [{ text: prompt }, ...imageParts] }], + generationConfig: { + responseMimeType: "application/json", + temperature: 0.2, + }, + }; + + const url = `${GEMINI_API_BASE}/models/${this.cfg.model}:generateContent?key=${this.cfg.apiKey}`; + const res = await this.cfg.fetchImpl(url, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(payload), + signal: AbortSignal.timeout(this.cfg.timeoutMs), + }); + + if (!res.ok) { + const body = await res.text(); + return { + status: "failed", + output: null, + error: `Gemini ${res.status}: ${body.slice(0, 500)}`, + }; + } + + const geminiBody = (await res.json()) as { + candidates?: Array<{ content?: { parts?: Array<{ text?: string }> } }>; + usageMetadata?: { promptTokenCount?: number; candidatesTokenCount?: number }; + }; + const text = geminiBody.candidates?.[0]?.content?.parts?.[0]?.text ?? ""; + + let parsed: z.infer; + try { + const json = JSON.parse(text); + const safe = geminiFridgeResponseSchema.safeParse(json); + if (!safe.success) { + return { status: "failed", output: null, error: `Gemini-svar matchar inte schema: ${safe.error.message}` }; + } + parsed = safe.data; + } catch { + return { status: "failed", output: null, error: "Gemini-svar var inte giltig JSON." }; + } + + const inputTokens = geminiBody.usageMetadata?.promptTokenCount ?? 0; + const outputTokens = geminiBody.usageMetadata?.candidatesTokenCount ?? 0; + const costUsd = this.estimateCostUsd(inputTokens, outputTokens); + await this.recordSpend(costUsd); + + const latencyMs = Date.now() - started; + const items = toDetectedItems(parsed); + + return { + status: items.length > 0 ? "ok" : "uncertain", + output: { items, imageQualityIssues: parsed.imageQualityIssues } as TaskOutput<"ANALYZE_FRIDGE_IMAGE">, + modelVersion: this.cfg.model, + promptVersion: this.cfg.promptVersion, + latencyMs, + costUsd, + inputTokens, + outputTokens, + }; + } + + private buildFridgePrompt( + taskType: "ANALYZE_FRIDGE_IMAGE" | "ANALYZE_PANTRY_IMAGE", + locale: LocaleContext, + locationType: string, + ): string { + const place = taskType === "ANALYZE_FRIDGE_IMAGE" ? "kylskåpet" : "skafferiet"; + const lang = locale.languageTag.startsWith("en") ? "English" : "Swedish"; + return `Du är en noggrann livsmedelsassistent för svenska hushåll. Användaren har fotat ${place} (${locationType}). + +Identifiera varje livsmedelsprodukt du ser. För varje produkt, svara med: +- produkt: produktens namn på svenska (t.ex. "Mellanmjölk", "Smör", "Vispgrädde") +- varumarke: varumärke om det syns tydligt, annars null +- kvantitet: siffra (t.ex. 1, 500, 1.5) +- enhet: förkortning som l, g, ml, st, förpackning +- kategori: t.ex. mejeri, frukt, grönt, kött, skafferi +- bastaFore: bäst-före-datum om det syns (YYYY-MM-DD), annars null +- sistaForbruk: sista förbrukningsdatum om det syns (YYYY-MM-DD), annars null +- konfidens: 0.0–1.0 + +Svara ENDAST med giltig JSON i exakt detta format: +{ + "items": [...], + "imageQualityIssues": [] +} +imageQualityIssues kan innehålla någon av: "dark", "blurry", "occlusion", "too_far". + +Språk: ${lang}.`; + } + + private defaultLocale(): LocaleContext { + return { + languageTag: "sv-SE", + regionCode: "SE", + timeZone: "Europe/Stockholm", + measurementSystem: "METRIC", + temperatureUnit: "CELSIUS", + currencyCode: "SEK", + }; + } + + private mimeType(buf: Buffer): string { + if (buf[0] === 0xff && buf[1] === 0xd8) return "image/jpeg"; + if (buf.slice(0, 8).toString("hex") === "89504e470d0a1a0a") return "image/png"; + if (buf.slice(0, 4).toString("ascii") === "RIFF") return "image/webp"; + return "image/jpeg"; + } + + private estimateCostUsd(inputTokens: number, outputTokens: number): number { + // Gemini 2.5 Flash pricing (Aug 2025): ~$0.075/1M input, $0.30/1M output. + return inputTokens * 0.075e-6 + outputTokens * 0.30e-6; + } + + private async isOverBudget(estimateUsd: number): Promise { + if (this.cfg.dailyBudgetUsd <= 0 || !this.cfg.budgetStore) return false; + const current = await this.cfg.budgetStore.getDailySpendUsd(); + return current + estimateUsd > this.cfg.dailyBudgetUsd; + } + + private async recordSpend(costUsd: number): Promise { + if (this.cfg.budgetStore && costUsd > 0) { + await this.cfg.budgetStore.incrementDailySpendUsd(costUsd); + } + } +} diff --git a/packages/ai-contracts/src/index.ts b/packages/ai-contracts/src/index.ts index 659ef38..226d3c4 100644 --- a/packages/ai-contracts/src/index.ts +++ b/packages/ai-contracts/src/index.ts @@ -1,3 +1,4 @@ export * from "./tasks.js"; export * from "./client.js"; +export * from "./gemini.js"; export { mockOutputFor } from "./mock.js"; diff --git a/packages/ai-contracts/test/gemini.test.ts b/packages/ai-contracts/test/gemini.test.ts new file mode 100644 index 0000000..dd3dfaa --- /dev/null +++ b/packages/ai-contracts/test/gemini.test.ts @@ -0,0 +1,154 @@ +import { describe, it, expect } from "vitest"; +import { GeminiAamosClient, MemoryBudgetStore } from "../src/gemini.js"; +import type { AamosRequestEnvelope } from "../src/tasks.js"; + +const FIXTURE_RESPONSE = { + candidates: [ + { + content: { + parts: [ + { + text: JSON.stringify({ + items: [ + { + produkt: "Mellanmjölk 1,5% fett", + varumarke: "Arla", + kvantitet: 1, + enhet: "liter", + kategori: "Mejeri", + bastaFore: null, + sistaForbruk: null, + konfidens: 0.98, + }, + { + produkt: "Smör", + varumarke: "Arla", + kvantitet: "500 g", + enhet: "g", + kategori: "Mejeri", + bastaFore: null, + konfidens: 0.95, + }, + ], + imageQualityIssues: [], + }), + }, + ], + }, + }, + ], + usageMetadata: { promptTokenCount: 392, candidatesTokenCount: 121 }, +}; + +function makeFetch(imageBytes: Buffer): typeof fetch { + return async (url: string | URL | Request, init?: RequestInit) => { + const urlStr = url.toString(); + if (urlStr.includes("/mock-s3/")) { + return new Response(imageBytes, { status: 200, headers: { "content-type": "image/jpeg" } }); + } + if (urlStr.includes("/models/") && init?.method === "POST") { + return new Response(JSON.stringify(FIXTURE_RESPONSE), { status: 200, headers: { "content-type": "application/json" } }); + } + return new Response("not found", { status: 404 }); + }; +} + +// A tiny 1x1 JPEG. +const JPEG_BYTES = Buffer.from( + "/9j/4AAQSkZJRgABAQEASABIAAD/2wBDAP//////////////////////////////////////////////////////////////////////////////////////wAALCAABAAEBAREA/8QAFAABAAAAAAAAAAAAAAAAAAAAA//EABQQAQAAAAAAAAAAAAAAAAAAAAD/2gAIAQEAAD8Af//Z", + "base64", +); + +describe("GeminiAamosClient", () => { + it("analyzes fridge image with recorded fixture", async () => { + const budget = new MemoryBudgetStore(); + const client = new GeminiAamosClient({ + apiKey: "test-key", + model: "gemini-2.5-flash", + fetchImpl: makeFetch(JPEG_BYTES), + budgetStore: budget, + dailyBudgetUsd: 10, + }); + + const result = await client.runTask("ANALYZE_FRIDGE_IMAGE", { + imageUrls: ["http://localhost/v1/mock-s3/fridge.jpg"], + locationType: "fridge", + marketLocale: "sv-SE", + knownItems: [], + }); + + expect(result.status).toBe("ok"); + expect(result.output).not.toBeNull(); + expect(result.modelVersion).toBe("gemini-2.5-flash"); + expect(result.costUsd).toBeGreaterThan(0); + expect(result.inputTokens).toBe(392); + expect(result.outputTokens).toBe(121); + + expect(result.output).not.toBeNull(); + const items = result.output!.items; + expect(items).toHaveLength(2); + expect(items[0]?.detectedName).toBe("Mellanmjölk 1,5% fett"); + expect(items[0]?.brand).toBe("Arla"); + expect(items[0]?.estimatedQuantity).toBe(1); + expect(items[0]?.unit).toBe("LITER"); + expect(items[0]?.confidence).toBe(0.98); + expect(items[1]?.detectedName).toBe("Smör"); + expect(items[1]?.estimatedQuantity).toBe(500); + expect(items[1]?.unit).toBe("GRAM"); + + const spend = await budget.getDailySpendUsd(); + expect(spend).toBe(result.costUsd); + }); + + it("blocks call when daily budget exceeded", async () => { + const budget = new MemoryBudgetStore(); + const client = new GeminiAamosClient({ + apiKey: "test-key", + fetchImpl: makeFetch(JPEG_BYTES), + budgetStore: budget, + dailyBudgetUsd: 0.00001, // essentially zero + }); + + const result = await client.runTask("ANALYZE_FRIDGE_IMAGE", { + imageUrls: ["http://localhost/v1/mock-s3/fridge.jpg"], + locationType: "fridge", + marketLocale: "sv-SE", + knownItems: [], + }); + + expect(result.status).toBe("failed"); + expect(result.error).toContain("budget"); + }); + + it("rejects unsupported task types", async () => { + const client = new GeminiAamosClient({ + apiKey: "test-key", + fetchImpl: makeFetch(JPEG_BYTES), + }); + + const result = await client.runTask( + "READ_RECEIPT", + { imageUrls: ["http://localhost/v1/mock-s3/receipt.jpg"], marketLocale: "sv-SE" }, + ); + + expect(result.status).toBe("failed"); + expect(result.error).toContain("READ_RECEIPT"); + }); + + it("validates input before calling Gemini", async () => { + const client = new GeminiAamosClient({ + apiKey: "test-key", + fetchImpl: makeFetch(JPEG_BYTES), + }); + + const result = await client.runTask("ANALYZE_FRIDGE_IMAGE", { + imageUrls: [], + locationType: "fridge", + marketLocale: "sv-SE", + knownItems: [], + } as unknown as { imageUrls: string[]; locationType: string; marketLocale: string; knownItems: string[] }); + + expect(result.status).toBe("failed"); + expect(result.error).toContain("Kontraktsfel"); + }); +}); diff --git a/packages/database/src/schema/subscriptions.ts b/packages/database/src/schema/subscriptions.ts index 6a8ca4a..ee0f79e 100644 --- a/packages/database/src/schema/subscriptions.ts +++ b/packages/database/src/schema/subscriptions.ts @@ -1,4 +1,5 @@ import { + bigint, boolean, index, integer, @@ -121,6 +122,7 @@ export const aiUsageCounters = pgTable( aiScans: integer("ai_scans").notNull().default(0), aiTokensIn: integer("ai_tokens_in").notNull().default(0), aiTokensOut: integer("ai_tokens_out").notNull().default(0), + aiCostUsdMicrocents: bigint("ai_cost_usd_microcents", { mode: "number" }).notNull().default(0), updatedAt: updatedAt(), }, (t) => [uniqueIndex("ai_usage_user_month_unique").on(t.userId, t.month)],