feat(gemini): Skiva 1 – Gemini 2.5 Flash som lärar-tier för kylskåpsskanning

- Gemini-adapter bakom AamosClient-interface (AAMOS_MODE=gemini) - Serversida/worker: hämtar bild, anropar Gemini, mappar mot canonical_ingredients - Kostnad/tokens bokförs i ai_usage_counters; global dagsbudget via BudgetStore - Redis-backed budget i worker, in-memory i tester - Migration 0019: ai_cost_usd_microcents - Hermetiska tester med inspelad fixture; separat pnpm eval:scan - docs/09 uppdaterad ärligt: AAMOS-status, Gemini-flöde, säkerhet/kostnad - REQUIRE_REAL=1 stödjer AAMOS_MODE=gemini; deploy-grind uppdaterad
2026-08-08 03:23:44 +07:00
parent 7894a1ec06
commit 050c958285
18 changed files with 1055 additions and 86 deletions
@@ -42,11 +42,16 @@ const configSchema = z.object({
  S3_ACCESS_KEY_ID: z.string().default(""),
  S3_SECRET_ACCESS_KEY: z.string().default(""),

-  AAMOS_MODE: z.enum(["http", "mock"]).default("http"),
+  AAMOS_MODE: z.enum(["http", "mock", "gemini"]).default("http"),
  AAMOS_API_URL: z.string().optional(),
  AAMOS_API_KEY: z.string().optional(),
  AAMOS_TIMEOUT_MS: z.coerce.number().int().default(60_000),

+  GEMINI_API_KEY: z.string().optional(),
+  GEMINI_MODEL: z.string().default("gemini-2.5-flash"),
+  GEMINI_TIMEOUT_MS: z.coerce.number().int().default(60_000),
+  GEMINI_DAILY_BUDGET_USD: z.coerce.number().default(0),
+
  APP_STORE_MODE: z.enum(["production", "sandbox"]).default("sandbox"),
  EMAIL_MODE: z.enum(["log", "smtp"]).default("log"),
  SMTP_HOST: z.string().default(""),
@@ -89,6 +94,10 @@ export function loadConfig(env: NodeJS.ProcessEnv = process.env): AppConfig {
      console.error("SÄKERHETSSTOPP: AAMOS_MODE=mock är inte tillåtet i produktion.");
      process.exit(1);
    }
+    if (cfg.AAMOS_MODE === "gemini" && !cfg.GEMINI_API_KEY) {
+      console.error("SÄKERHETSSTOPP: AAMOS_MODE=gemini kräver GEMINI_API_KEY i produktion.");
+      process.exit(1);
+    }
  }
  return cfg;
 }
@@ -77,6 +77,10 @@ export const corePlugin = fp(async (app: FastifyInstance, opts: { config: AppCon
      AAMOS_API_URL: config.AAMOS_API_URL,
      AAMOS_API_KEY: config.AAMOS_API_KEY,
      AAMOS_TIMEOUT_MS: String(config.AAMOS_TIMEOUT_MS),
+      GEMINI_API_KEY: config.GEMINI_API_KEY,
+      GEMINI_MODEL: config.GEMINI_MODEL,
+      GEMINI_TIMEOUT_MS: String(config.GEMINI_TIMEOUT_MS),
+      GEMINI_DAILY_BUDGET_USD: String(config.GEMINI_DAILY_BUDGET_USD),
    }),
  );

@@ -10,7 +10,8 @@
    "build": "tsup src/index.ts --format esm --target node22 --sourcemap --clean",
    "typecheck": "tsc --noEmit",
    "test": "vitest run --passWithNoTests",
-    "eval": "tsx src/eval/run.ts"
+    "eval": "tsx src/eval/run.ts",
+    "eval:scan": "tsx src/eval/scan-eval.ts"
  },
  "dependencies": {
    "@app/ai-contracts": "workspace:*",
@@ -12,7 +12,8 @@ for (const candidate of [".env", "../.env", "../../.env"]) {
 }
 import { createHmac } from "node:crypto";
 import { createDatabase, type Database } from "@app/database";
-import { createAamosClient, type AamosClient } from "@app/ai-contracts";
+import { createAamosClient, type AamosClient, type BudgetStore } from "@app/ai-contracts";
+import type { Redis } from "ioredis";

 export interface WorkerContext {
  db: Database;
@@ -23,13 +24,40 @@ export interface WorkerContext {
  close: () => Promise<void>;
 }

-export function createContext(): WorkerContext {
+class RedisBudgetStore implements BudgetStore {
+  constructor(
+    private readonly redis: Redis,
+    private readonly key: string,
+  ) {}
+
+  async getDailySpendUsd(): Promise<number> {
+    const val = await this.redis.get(this.key);
+    return val ? Number(val) : 0;
+  }
+
+  async incrementDailySpendUsd(amountUsd: number): Promise<number> {
+    const newVal = await this.redis.incrbyfloat(this.key, amountUsd);
+    // Expire at next midnight UTC to keep daily window.
+    const ttl = this.secondsUntilMidnightUtc();
+    await this.redis.expire(this.key, ttl);
+    return Number(newVal);
+  }
+
+  private secondsUntilMidnightUtc(): number {
+    const now = new Date();
+    const midnight = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate() + 1));
+    return Math.max(1, Math.floor((midnight.getTime() - now.getTime()) / 1000));
+  }
+}
+
+export function createContext(redis?: Redis): WorkerContext {
  const databaseUrl = process.env.DATABASE_URL;
  if (!databaseUrl) {
    throw new Error("Missing DATABASE_URL. Worker must connect explicitly to the app database.");
  }
  const { db, pool } = createDatabase(databaseUrl);
-  const aamos = createAamosClient();
+  const budgetStore = redis ? new RedisBudgetStore(redis, `gemini:daily:budget:${new Date().toISOString().slice(0, 10)}`) : undefined;
+  const aamos = createAamosClient(undefined, { budgetStore });
  const apiBaseUrl = process.env.API_BASE_URL ?? "http://localhost:4000";
  const secret = process.env.ENTITLEMENT_SIGNING_SECRET ?? "dev-only-change-me-three";

@@ -0,0 +1,162 @@
+import { config as loadDotenv } from "dotenv";
+import { existsSync } from "node:fs";
+import path from "node:path";
+for (const candidate of [".env", "../.env", "../../.env"]) {
+  const p = path.resolve(process.cwd(), candidate);
+  if (existsSync(p)) {
+    loadDotenv({ path: p });
+    break;
+  }
+}
+
+import { GeminiAamosClient, type BudgetStore } from "@app/ai-contracts";
+
+/**
+ * Live-evaluering av Gemini-lärar-tier för kylskåps-/skafferiskanning.
+ *
+ *   pnpm --filter @app/worker eval:scan
+ *
+ * Kräver AAMOS_MODE=gemini + GEMINI_API_KEY. Använder ett litet guldset av
+ * publikt tillgängliga bild-URL:er (eller lokala filer). Rapporterar
+ * träffbild, kvantitet, enhet, kostnad och latens. Avslutar med exitkod 1
+ * om någon obligatorisk kontroll fallerar.
+ */
+
+interface ScanEvalCase {
+  id: string;
+  imageUrls: string[];
+  locationType: "fridge" | "pantry";
+  marketLocale: string;
+  checks: (items: Array<{
+    detectedName: string;
+    brand: string | null;
+    estimatedQuantity: number | null;
+    unit: string | null;
+    confidence: number;
+    requiresConfirmation: boolean;
+  }>) => { name: string; passed: boolean }[];
+}
+
+const CASES: ScanEvalCase[] = [
+  {
+    id: "arla-milk-fridge",
+    imageUrls: ["https://upload.wikimedia.org/wikipedia/commons/6/6c/Arla_Ko_Mellanmj%C3%B6lk_1L.jpg"],
+    locationType: "fridge",
+    marketLocale: "sv-SE",
+    checks: (items) => {
+      const milk = items.find((i) =>
+        /mjölk|milk/i.test(i.detectedName) || /arla/i.test(i.brand ?? ""),
+      );
+      return [
+        { name: "hittade mjölkprodukt", passed: !!milk },
+        { name: "konfidens > 0.7", passed: !!milk && milk.confidence > 0.7 },
+        { name: "kräver bekräftelse om låg konfidens", passed: !!milk && (milk.confidence >= 0.92 || milk.requiresConfirmation) },
+      ];
+    },
+  },
+  {
+    id: "swedish-butter-fridge",
+    imageUrls: ["https://upload.wikimedia.org/wikipedia/commons/3/3e/Svenskt_Sm%C3%B6r_Normalsaltat_80-25_500g.jpg"],
+    locationType: "fridge",
+    marketLocale: "sv-SE",
+    checks: (items) => {
+      const butter = items.find((i) => /smör|butter/i.test(i.detectedName));
+      return [
+        { name: "hittade smör", passed: !!butter },
+        { name: "konfidens > 0.7", passed: !!butter && butter.confidence > 0.7 },
+      ];
+    },
+  },
+];
+
+class SilentBudgetStore implements BudgetStore {
+  private spend = 0;
+  async getDailySpendUsd(): Promise<number> {
+    return this.spend;
+  }
+  async incrementDailySpendUsd(amountUsd: number): Promise<number> {
+    this.spend += amountUsd;
+    return this.spend;
+  }
+}
+
+async function main() {
+  const mode = process.env.AAMOS_MODE ?? "mock";
+  const apiKey = process.env.GEMINI_API_KEY;
+
+  if (mode !== "gemini" || !apiKey) {
+    console.error("[eval:scan] KRÄVER AAMOS_MODE=gemini + GEMINI_API_KEY.");
+    process.exit(1);
+  }
+
+  const client = new GeminiAamosClient({
+    apiKey,
+    model: process.env.GEMINI_MODEL,
+    timeoutMs: process.env.GEMINI_TIMEOUT_MS ? Number(process.env.GEMINI_TIMEOUT_MS) : 60_000,
+    dailyBudgetUsd: process.env.GEMINI_DAILY_BUDGET_USD ? Number(process.env.GEMINI_DAILY_BUDGET_USD) : 10,
+    budgetStore: new SilentBudgetStore(),
+  });
+
+  console.log(`[eval:scan] Gemini-lärar-tier live-evaluering – ${CASES.length} fall\n`);
+
+  let totalChecks = 0;
+  let failedChecks = 0;
+  let totalCostUsd = 0;
+  let totalLatencyMs = 0;
+
+  for (const evalCase of CASES) {
+    const started = Date.now();
+    let checks: { name: string; passed: boolean }[] = [];
+    let error: string | null = null;
+
+    try {
+      const result = await client.runTask(
+        evalCase.locationType === "fridge" ? "ANALYZE_FRIDGE_IMAGE" : "ANALYZE_PANTRY_IMAGE",
+        {
+          imageUrls: evalCase.imageUrls,
+          locationType: evalCase.locationType,
+          marketLocale: evalCase.marketLocale,
+          knownItems: [],
+        },
+      );
+
+      if (result.status !== "ok" || !result.output) {
+        error = `status=${result.status}: ${result.error ?? "okänt fel"}`;
+        checks = [{ name: "Gemini svarade ok", passed: false }];
+      } else {
+        const items = result.output.items;
+        checks = [{ name: "Gemini svarade ok", passed: true }, ...evalCase.checks(items)];
+      }
+
+      if (result.costUsd) totalCostUsd += result.costUsd;
+      if (result.latencyMs) totalLatencyMs += result.latencyMs;
+    } catch (err) {
+      error = (err as Error).message;
+      checks = [{ name: "Gemini svarade ok", passed: false }];
+    }
+
+    const ms = Date.now() - started;
+    const passed = checks.every((c) => c.passed);
+    totalChecks += checks.length;
+    failedChecks += checks.filter((c) => !c.passed).length;
+
+    console.log(`${passed ? "✓" : "✗"} ${evalCase.id} (${ms} ms)`);
+    for (const c of checks) if (!c.passed) console.log(`    ✗ ${c.name}`);
+    if (error) console.log(`    fel: ${error}`);
+  }
+
+  console.log(`\n[eval:scan] ${CASES.length} fall, ${totalChecks} kontroller, ${failedChecks} fallerade.`);
+  console.log(`[eval:scan] Total latens: ${totalLatencyMs} ms, total kostnad: ~$${totalCostUsd.toFixed(6)}`);
+
+  if (failedChecks > 0) {
+    console.log("[eval:scan] UNDERKÄND – åtgärda innan Skiva 1 går till fälttest.");
+    process.exit(1);
+  }
+  console.log("[eval:scan] GODKÄND – fälttest kan börja.");
+  process.exit(0);
+}
+
+main().catch((err) => {
+  console.error("[eval:scan] KRASCH:", err);
+  process.exit(1);
+});
@@ -38,7 +38,7 @@ import {
 const connection = new IORedis(process.env.REDIS_URL ?? "redis://localhost:6379", {
  maxRetriesPerRequest: null,
 });
-const ctx = createContext();
+const ctx = createContext(connection);
 const instanceId = process.env.WORKER_INSTANCE_ID ?? `worker-${process.pid}`;

 const deadLetterQueue = new Queue(DEAD_LETTER_QUEUE_NAME, { connection });
@@ -1,9 +1,13 @@
-import { eq } from "drizzle-orm";
+import { eq, sql } from "drizzle-orm";
 import { schema } from "@app/database";
-import type { AamosTaskType } from "@app/ai-contracts";
+import type { AamosResult, AamosTaskType, DetectedItem } from "@app/ai-contracts";
 import type { WorkerContext } from "../context.js";
 import { getLocaleContext } from "../locale.js";
 import type { LocaleContext } from "@app/shared-types";
+function currentMonth(): string {
+  const d = new Date();
+  return `${d.getUTCFullYear()}-${String(d.getUTCMonth() + 1).padStart(2, "0")}`;
+}

 /**
 * Bild-/OCR-jobb (spec §54): hämtar scan_job, anropar AAMOS med kontraktvaliderad
@@ -51,11 +55,21 @@ export async function processScanJob(ctx: WorkerContext, scanJobId: string): Pro
    return;
  }

+  let output = result.output as Record<string, unknown>;
+
+  // SKIVA 1: mappa lagrade bilders detekterade namn mot kanoniska ingredienser.
+  if (
+    (job.jobType === "ANALYZE_FRIDGE_IMAGE" || job.jobType === "ANALYZE_PANTRY_IMAGE") &&
+    Array.isArray((output as { items?: unknown }).items)
+  ) {
+    output = await mapDetectedItemsToCanonical(ctx, output);
+  }
+
  await ctx.db
    .update(schema.scanJobs)
    .set({
      status: "awaiting_confirmation",
-      result: result.output as Record<string, unknown>,
+      result: output,
      modelVersion: result.modelVersion ?? null,
      promptVersion: result.promptVersion ?? null,
      latencyMs: result.latencyMs ?? Date.now() - started,
@@ -64,9 +78,12 @@ export async function processScanJob(ctx: WorkerContext, scanJobId: string): Pro
    })
    .where(eq(schema.scanJobs.id, scanJobId));

+  // Bokför verklig AI-kostnad/tokens utan PII (spec §45).
+  await recordAiUsage(ctx, job.userId, result);
+
  // MEAL_PHOTO_ANALYZED-event för tallriksfoton (spec §55)
  if (job.jobType === "ANALYZE_MEAL_IMAGE") {
-    const output = result.output as {
+    const mealOutput = output as {
      kcalRange?: { mostLikely: number } | null;
      matchesRecipeContext?: boolean | null;
    };
@@ -76,8 +93,8 @@ export async function processScanJob(ctx: WorkerContext, scanJobId: string): Pro
      householdId: job.householdId,
      payload: {
        scanJobId,
-        matched: output.matchesRecipeContext ?? false,
-        kcalMostLikely: output.kcalRange?.mostLikely ?? null,
+        matched: mealOutput.matchesRecipeContext ?? false,
+        kcalMostLikely: mealOutput.kcalRange?.mostLikely ?? null,
      },
    });
  }
@@ -176,3 +193,130 @@ async function loadConsentFlags(ctx: WorkerContext, userId: string) {
    imageTraining: get("image_training"),
  };
 }
+
+// ---------------------------------------------------------------------------
+// Kanonisk ingrediensmappning (SKIVA 1). AI föreslår, vi matchar mjukt,
+// användaren bekräftar alltid innan commit.
+// ---------------------------------------------------------------------------
+
+interface CanonicalIndex {
+  id: string;
+  nameSv: string;
+  nameEn: string;
+  aliases: string[];
+}
+
+async function mapDetectedItemsToCanonical(
+  ctx: WorkerContext,
+  output: Record<string, unknown>,
+): Promise<Record<string, unknown>> {
+  const items = (output as { items: DetectedItem[] }).items;
+  if (!items.length) return output;
+
+  const index = await loadCanonicalIndex(ctx);
+  const mapped = items.map((item) => {
+    const match = findBestCanonicalMatch(item.detectedName, index);
+    return {
+      ...item,
+      canonicalIngredientId: match?.id ?? null,
+      requiresConfirmation: match == null || item.confidence < 0.92,
+    };
+  });
+
+  return { ...output, items: mapped };
+}
+
+async function loadCanonicalIndex(ctx: WorkerContext): Promise<CanonicalIndex[]> {
+  return ctx.db
+    .select({
+      id: schema.canonicalIngredients.id,
+      nameSv: schema.canonicalIngredients.nameSv,
+      nameEn: schema.canonicalIngredients.nameEn,
+      aliases: schema.canonicalIngredients.aliases,
+    })
+    .from(schema.canonicalIngredients);
+}
+
+function findBestCanonicalMatch(
+  detectedName: string,
+  index: CanonicalIndex[],
+): CanonicalIndex | null {
+  const query = detectedName.toLowerCase();
+  let best: { item: CanonicalIndex; score: number } | null = null;
+
+  for (const item of index) {
+    const score = scoreMatch(query, item);
+    if (score > 0 && (!best || score > best.score)) {
+      best = { item, score };
+    }
+  }
+
+  // Threshold: require a strong token overlap or exact substring.
+  if (!best || best.score < 0.35) return null;
+  return best.item;
+}
+
+function scoreMatch(query: string, item: CanonicalIndex): number {
+  const candidates = [
+    item.nameSv.toLowerCase(),
+    item.nameEn.toLowerCase(),
+    ...item.aliases.map((a) => a.toLowerCase()),
+  ];
+
+  let max = 0;
+  const queryTokens = tokenize(query);
+
+  for (const cand of candidates) {
+    if (cand === query) return 1;
+    if (cand.includes(query) || query.includes(cand)) max = Math.max(max, 0.85);
+
+    const candTokens = tokenize(cand);
+    const intersection = queryTokens.filter((t) => candTokens.includes(t));
+    if (intersection.length > 0) {
+      const overlap = intersection.length / Math.max(queryTokens.length, candTokens.length);
+      max = Math.max(max, overlap);
+    }
+  }
+
+  return max;
+}
+
+function tokenize(text: string): string[] {
+  return text
+    .toLowerCase()
+    .replace(/[^a-zåäö0-9\s]/g, " ")
+    .split(/\s+/)
+    .filter((t) => t.length > 1);
+}
+
+// ---------------------------------------------------------------------------
+// AI-kostnadsbokföring utan PII.
+// ---------------------------------------------------------------------------
+
+async function recordAiUsage(ctx: WorkerContext, userId: string, result: AamosResult<AamosTaskType>): Promise<void> {
+  const costUsd = result.costUsd ?? 0;
+  const tokensIn = result.inputTokens ?? 0;
+  const tokensOut = result.outputTokens ?? 0;
+  const microcents = Math.round(costUsd * 100_000_000);
+  const month = currentMonth();
+
+  await ctx.db
+    .insert(schema.aiUsageCounters)
+    .values({
+      userId,
+      month,
+      aiScans: 1,
+      aiTokensIn: tokensIn,
+      aiTokensOut: tokensOut,
+      aiCostUsdMicrocents: microcents,
+    })
+    .onConflictDoUpdate({
+      target: [schema.aiUsageCounters.userId, schema.aiUsageCounters.month],
+      set: {
+        aiScans: sql`${schema.aiUsageCounters.aiScans} + 1`,
+        aiTokensIn: sql`${schema.aiUsageCounters.aiTokensIn} + ${tokensIn}`,
+        aiTokensOut: sql`${schema.aiUsageCounters.aiTokensOut} + ${tokensOut}`,
+        aiCostUsdMicrocents: sql`${schema.aiUsageCounters.aiCostUsdMicrocents} + ${microcents}`,
+      },
+    });
+}