feat(worker/ai): Gemini-lärar-tier för scan/detect/allergen + shadow-capture

- Utöka GeminiAamosClient med ANALYZE_MEAL_IMAGE, READ_NUTRITION_LABEL, READ_EXPIRY_DATE och READ_RECEIPT (vision-prompts mot kontraktsscheman). - Lägg till apps/worker/src/lib/shadow-capture.ts: S3/local fallback, imageTraining-gating, ingen PII, aldrig faila användarvägen. - Integrera capture i scan-processorn efter lyckad runTask med consentFlags. - Lägg till @aws-sdk/client-s3 i worker samt eval:capture-skript. - Uppdatera eval:scan till Wikimedia Special:FilePath + redirect-following. - Exkludera .terraform i brand-guard för att undvika false positives. - Uppdatera gemini-test för DEDUPLICATE_INVENTORY som unsupported.
2026-08-12 02:51:05 +07:00
parent 515dd12cc7
commit 9a9f8a6a3c
9 changed files with 615 additions and 20 deletions
@@ -1,5 +1,5 @@
 /**
- * Gemini-lärar-tier för AAMOS-task-typer (SKIVA 1: kylskåpsskanning).
+ * Gemini-lärar-tier för AAMOS-task-typer (SKIVA 1).
 *
 * - Implementerar samma AamosClient-interface som HttpAamosClient/MockAamosClient.
 * - App + Food API ser ingen skillnad; workern byter bara adapter.
@@ -7,7 +7,6 @@
 * - Kostnad/tokens rapporteras tillbaka så workern kan bokföra i ai_usage_counters.
 * - Global dagsbudget kan sättas via GEMINI_DAILY_BUDGET_USD.
 */
-import { randomUUID } from "node:crypto";
 import { z } from "zod";
 import {
  TASK_CONTRACTS,
@@ -17,14 +16,22 @@ import {
  detectedItemSchema,
  type DetectedItem,
  generateRecipeCandidatesOutput,
+  analyzeMealImageOutput,
+  readNutritionLabelOutput,
+  readExpiryDateOutput,
+  readReceiptOutput,
 } from "./tasks.js";
 import type { AamosCallOptions, AamosClient, AamosResult } from "./client.js";
 import type { LocaleContext } from "@app/shared-types";

 const GEMINI_API_BASE = "https://generativelanguage.googleapis.com/v1beta";

-/** Custom fetch using Node https module — works around IPv6 hangs on WSL. */
+/** Custom fetch using Node https module — works around IPv6 hangs on WSL. Follows redirects. */
 function customNodeFetch(input: string | URL | Request, init?: RequestInit): Promise<Response> {
+  return requestOnce(input, init, 5);
+}
+
+function requestOnce(input: string | URL | Request, init: RequestInit | undefined, redirectsLeft: number): Promise<Response> {
  return new Promise((resolve, reject) => {
    const url = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url;
    const u = new URL(url);
@@ -45,6 +52,9 @@ function customNodeFetch(input: string | URL | Request, init?: RequestInit): Pro
      if (postData && !headers["content-length"]) {
        headers["content-length"] = String(Buffer.byteLength(postData));
      }
+      if (!headers["user-agent"]) {
+        headers["user-agent"] = "Mozilla/5.0 (compatible; Cibello/1.0)";
+      }

      const req = httpMod.request(
        {
@@ -56,13 +66,23 @@ function customNodeFetch(input: string | URL | Request, init?: RequestInit): Pro
          timeout: 120_000,
        },
        (res) => {
+          const status = res.statusCode ?? 200;
+          const location = res.headers.location;
+          if (status >= 300 && status < 400 && location && redirectsLeft > 0) {
+            const nextUrl = new URL(location, url).toString();
+            requestOnce(nextUrl, { ...init, method: "GET", body: undefined }, redirectsLeft - 1)
+              .then(resolve)
+              .catch(reject);
+            return;
+          }
+
          let body = "";
          res.setEncoding("utf8");
          res.on("data", (chunk) => { body += chunk; });
          res.on("end", () => {
            resolve(
              new Response(body, {
-                status: res.statusCode ?? 200,
+                status,
                statusText: res.statusMessage ?? "OK",
                headers: new Headers(Object.entries(res.headers).map(([k, v]) => [k, String(v)])),
              }),
@@ -81,6 +101,9 @@ function customNodeFetch(input: string | URL | Request, init?: RequestInit): Pro
 /** Estimated max cost per image call in USD (pessimistic). */
 const COST_ESTIMATE_PER_IMAGE_USD = 0.0015;

+/** Estimated max cost per simple OCR image call in USD. */
+const COST_ESTIMATE_PER_OCR_IMAGE_USD = 0.001;
+
 export interface BudgetStore {
  /** Return current daily spend in USD. */
  getDailySpendUsd(): Promise<number>;
@@ -222,7 +245,7 @@ export class GeminiAamosClient implements AamosClient {

  async healthCheck(): Promise<{ ok: boolean; detail?: string }> {
    try {
-      const url = `${GEMINI_API_BASE}/models?key=${this.cfg.apiKey}`;
+      const url = `${GEMINI_API_BASE}/models?key=${encodeURIComponent(this.cfg.apiKey)}`;
      const res = await this.cfg.fetchImpl(url, { signal: AbortSignal.timeout(5_000) });
      return { ok: res.ok, detail: `status ${res.status}` };
    } catch (err) {
@@ -251,6 +274,14 @@ export class GeminiAamosClient implements AamosClient {
        return this.analyzeStorageImage(taskType, parsedInput.data as TaskInput<"ANALYZE_FRIDGE_IMAGE">, options) as Promise<AamosResult<T>>;
      case "GENERATE_RECIPE_CANDIDATES":
        return this.generateRecipeCandidates(parsedInput.data as TaskInput<"GENERATE_RECIPE_CANDIDATES">, options) as Promise<AamosResult<T>>;
+      case "ANALYZE_MEAL_IMAGE":
+        return this.analyzeMealImage(parsedInput.data as TaskInput<"ANALYZE_MEAL_IMAGE">, options) as Promise<AamosResult<T>>;
+      case "READ_NUTRITION_LABEL":
+        return this.readNutritionLabel(parsedInput.data as TaskInput<"READ_NUTRITION_LABEL">, options) as Promise<AamosResult<T>>;
+      case "READ_EXPIRY_DATE":
+        return this.readExpiryDate(parsedInput.data as TaskInput<"READ_EXPIRY_DATE">, options) as Promise<AamosResult<T>>;
+      case "READ_RECEIPT":
+        return this.readReceipt(parsedInput.data as TaskInput<"READ_RECEIPT">, options) as Promise<AamosResult<T>>;
      default:
        return {
          status: "failed",
@@ -295,7 +326,7 @@ export class GeminiAamosClient implements AamosClient {
      },
    };

-    const url = `${GEMINI_API_BASE}/models/${this.cfg.model}:generateContent?key=${this.cfg.apiKey}`;
+    const url = `${GEMINI_API_BASE}/models/${this.cfg.model}:generateContent?key=${encodeURIComponent(this.cfg.apiKey)}`;
    const res = await this.cfg.fetchImpl(url, {
      method: "POST",
      headers: { "content-type": "application/json" },
@@ -350,6 +381,186 @@ export class GeminiAamosClient implements AamosClient {
    };
  }

+  private async runVisionTask<T extends AamosTaskType>(
+    taskType: T,
+    imageUrls: string[],
+    prompt: string,
+    outputSchema: z.ZodSchema<unknown>,
+    mapOutput: (parsed: unknown) => TaskOutput<T>,
+    options: AamosCallOptions,
+    promptVersion: string,
+    costEstimatePerImage: number,
+  ): Promise<AamosResult<T>> {
+    const locale = options.localeContext ?? this.defaultLocale();
+    const started = Date.now();
+
+    const imageParts = await this.fetchImageParts(imageUrls, this.cfg.timeoutMs);
+    const estimatedCostUsd = imageParts.length * costEstimatePerImage;
+    if (await this.isOverBudget(estimatedCostUsd)) {
+      return {
+        status: "failed",
+        output: null,
+        error: "Global Gemini-dagsbudget är förbrukad.",
+      } as AamosResult<T>;
+    }
+
+    const payload = {
+      contents: [{ parts: [{ text: prompt }, ...imageParts] }],
+      generationConfig: {
+        responseMimeType: "application/json",
+        temperature: 0.2,
+      },
+    };
+
+    const url = `${GEMINI_API_BASE}/models/${this.cfg.model}:generateContent?key=${encodeURIComponent(this.cfg.apiKey)}`;
+    const res = await this.cfg.fetchImpl(url, {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify(payload),
+      signal: AbortSignal.timeout(this.cfg.timeoutMs),
+    });
+
+    if (!res.ok) {
+      const body = await res.text();
+      return {
+        status: "failed",
+        output: null,
+        error: `Gemini ${res.status}: ${body.slice(0, 500)}`,
+      } as AamosResult<T>;
+    }
+
+    const geminiBody = (await res.json()) as {
+      candidates?: Array<{ content?: { parts?: Array<{ text?: string }> } }>;
+      usageMetadata?: { promptTokenCount?: number; candidatesTokenCount?: number };
+    };
+    const text = geminiBody.candidates?.[0]?.content?.parts?.[0]?.text ?? "";
+
+    let parsed: unknown;
+    try {
+      const json = JSON.parse(text);
+      const safe = outputSchema.safeParse(json);
+      if (!safe.success) {
+        return {
+          status: "failed",
+          output: null,
+          error: `Gemini-svar matchar inte schema: ${safe.error.message}`,
+        } as AamosResult<T>;
+      }
+      parsed = safe.data;
+    } catch {
+      return {
+        status: "failed",
+        output: null,
+        error: "Gemini-svar var inte giltig JSON.",
+      } as AamosResult<T>;
+    }
+
+    const inputTokens = geminiBody.usageMetadata?.promptTokenCount ?? 0;
+    const outputTokens = geminiBody.usageMetadata?.candidatesTokenCount ?? 0;
+    const costUsd = this.estimateCostUsd(inputTokens, outputTokens);
+    await this.recordSpend(costUsd);
+
+    const latencyMs = Date.now() - started;
+    const output = mapOutput(parsed) as TaskOutput<T>;
+
+    return {
+      status: "ok",
+      output,
+      modelVersion: this.cfg.model,
+      promptVersion,
+      latencyMs,
+      costUsd,
+      inputTokens,
+      outputTokens,
+    };
+  }
+
+  private async analyzeMealImage(
+    input: TaskInput<"ANALYZE_MEAL_IMAGE">,
+    options: AamosCallOptions,
+  ): Promise<AamosResult<"ANALYZE_MEAL_IMAGE">> {
+    const locale = options.localeContext ?? this.defaultLocale();
+    const prompt = this.buildMealPrompt(locale, input.recipeContext);
+    return this.runVisionTask(
+      "ANALYZE_MEAL_IMAGE",
+      input.imageUrls,
+      prompt,
+      analyzeMealImageOutput,
+      (parsed) => parsed as TaskOutput<"ANALYZE_MEAL_IMAGE">,
+      options,
+      "gemini-meal-v1",
+      COST_ESTIMATE_PER_IMAGE_USD,
+    );
+  }
+
+  private async readNutritionLabel(
+    input: TaskInput<"READ_NUTRITION_LABEL">,
+    options: AamosCallOptions,
+  ): Promise<AamosResult<"READ_NUTRITION_LABEL">> {
+    const locale = options.localeContext ?? this.defaultLocale();
+    const prompt = this.buildNutritionPrompt(locale);
+    return this.runVisionTask(
+      "READ_NUTRITION_LABEL",
+      input.imageUrls,
+      prompt,
+      readNutritionLabelOutput,
+      (parsed) => parsed as TaskOutput<"READ_NUTRITION_LABEL">,
+      options,
+      "gemini-nutrition-v1",
+      COST_ESTIMATE_PER_OCR_IMAGE_USD,
+    );
+  }
+
+  private async readExpiryDate(
+    input: TaskInput<"READ_EXPIRY_DATE">,
+    options: AamosCallOptions,
+  ): Promise<AamosResult<"READ_EXPIRY_DATE">> {
+    const locale = options.localeContext ?? this.defaultLocale();
+    const prompt = this.buildExpiryPrompt(locale);
+    return this.runVisionTask(
+      "READ_EXPIRY_DATE",
+      input.imageUrls,
+      prompt,
+      readExpiryDateOutput,
+      (parsed) => parsed as TaskOutput<"READ_EXPIRY_DATE">,
+      options,
+      "gemini-expiry-v1",
+      COST_ESTIMATE_PER_OCR_IMAGE_USD,
+    );
+  }
+
+  private async readReceipt(
+    input: TaskInput<"READ_RECEIPT">,
+    options: AamosCallOptions,
+  ): Promise<AamosResult<"READ_RECEIPT">> {
+    const locale = options.localeContext ?? this.defaultLocale();
+    const prompt = this.buildReceiptPrompt(locale);
+    return this.runVisionTask(
+      "READ_RECEIPT",
+      input.imageUrls,
+      prompt,
+      readReceiptOutput,
+      (parsed) => parsed as TaskOutput<"READ_RECEIPT">,
+      options,
+      "gemini-receipt-v1",
+      COST_ESTIMATE_PER_IMAGE_USD,
+    );
+  }
+
+  private async fetchImageParts(
+    imageUrls: string[],
+    timeoutMs: number,
+  ): Promise<Array<{ inline_data: { mime_type: string; data: string } }>> {
+    return Promise.all(
+      imageUrls.slice(0, 6).map(async (url) => {
+        const res = await this.cfg.fetchImpl(url, { signal: AbortSignal.timeout(timeoutMs) });
+        if (!res.ok) throw new Error(`Kunde inte hämta bild: ${res.status} ${url}`);
+        const buf = Buffer.from(await res.arrayBuffer());
+        return { inline_data: { mime_type: this.mimeType(buf), data: buf.toString("base64") } };
+      }),
+    );
+  }
+
  private async generateRecipeCandidates(
    input: TaskInput<"GENERATE_RECIPE_CANDIDATES">,
    options: AamosCallOptions,
@@ -377,7 +588,7 @@ export class GeminiAamosClient implements AamosClient {
      },
    };

-    const url = `${GEMINI_API_BASE}/models/${this.cfg.model}:generateContent?key=${this.cfg.apiKey}`;
+    const url = `${GEMINI_API_BASE}/models/${this.cfg.model}:generateContent?key=${encodeURIComponent(this.cfg.apiKey)}`;
    const res = await this.cfg.fetchImpl(url, {
      method: "POST",
      headers: { "content-type": "application/json" },
@@ -553,6 +764,129 @@ Svara ENDAST med giltig JSON i exakt detta format:
 }
 imageQualityIssues kan innehålla någon av: "dark", "blurry", "occlusion", "too_far".

+Språk: ${lang}.`;
+  }
+
+  private buildMealPrompt(locale: LocaleContext, recipeContext: unknown): string {
+    const lang = locale.languageTag.startsWith("en") ? "English" : "Swedish";
+    const ctx = recipeContext as {
+      recipeId?: string;
+      titleSv?: string;
+      nutritionPerPortion?: Record<string, number>;
+      portions?: number;
+    } | null;
+
+    let contextLine = "";
+    if (ctx?.titleSv) {
+      contextLine = `Användaren säger att måltiden ska matcha receptet "${ctx.titleSv}" (${ctx.portions ?? "?"} portioner). `;
+    }
+
+    return `Du är en svensk näringsassistent. ${contextLine}Bilden visar en serverad måltid.
+
+Uppskatta:
+- matchesRecipeContext: true/false/null – matchar bilden receptet användaren nämnde?
+- portionFractionEstimate: hur stor del av en normalportion ser det ut att vara? 0.0–5.0, eller null.
+- kcalRange: { min, max, mostLikely } för hela måltiden, eller null.
+- components: lista med uppskattade komponenter (namn, ev. canonicalIngredientId som null, estimatedGrams, confidence).
+- confidence: 0.0–1.0 för hela tolkningen.
+
+Svara ENDAST med giltig JSON som matchar detta schema:
+{
+  "matchesRecipeContext": boolean | null,
+  "portionFractionEstimate": number | null,
+  "kcalRange": { "min": number, "max": number, "mostLikely": number } | null,
+  "components": [
+    { "name": string, "canonicalIngredientId": string | null, "estimatedGrams": number | null, "confidence": number }
+  ],
+  "confidence": number
+}
+
+Språk: ${lang}.`;
+  }
+
+  private buildNutritionPrompt(locale: LocaleContext): string {
+    const lang = locale.languageTag.startsWith("en") ? "English" : "Swedish";
+    return `Du är en OCR-assistent för svenska livsmedelsförpackningar. Bilden visar en näringsdeklaration/ingredienslista.
+
+Läs av exakt det som står på etiketten. Gissa aldrig värden som inte syns. Returnera:
+- basis: "per_100_g", "per_100_ml" eller "per_portion" – vad gäller tabellen?
+- values: { kcal, proteinG, carbsG, fatG, saturatedFatG, fiberG, sugarG, saltG } – null om ej synligt.
+- ingredientsText: rå text med ingredienser, eller null.
+- allergensDeclared: lista med allergener som explicit nämns (t.ex. ["gluten", "mjölk", "nötter"]).
+- gtin: streckkod/EAN om synlig, annars null.
+- productName: produktnamn om synligt, annars null.
+- brand: varumärke om synligt, annars null.
+- confidence: 0.0–1.0.
+
+Svara ENDAST med giltig JSON som matchar detta schema:
+{
+  "basis": "per_100_g" | "per_100_ml" | "per_portion" | null,
+  "values": { "kcal": number|null, "proteinG": number|null, "carbsG": number|null, "fatG": number|null, "saturatedFatG": number|null, "fiberG": number|null, "sugarG": number|null, "saltG": number|null } | null,
+  "ingredientsText": string | null,
+  "allergensDeclared": string[],
+  "gtin": string | null,
+  "productName": string | null,
+  "brand": string | null,
+  "confidence": number
+}
+
+Språk: ${lang}.`;
+  }
+
+  private buildExpiryPrompt(locale: LocaleContext): string {
+    const lang = locale.languageTag.startsWith("en") ? "English" : "Swedish";
+    return `Du är en OCR-assistent för svenska livsmedelsförpackningar. Bilden visar ett bäst-före-datum eller sista förbrukningsdatum.
+
+Returnera:
+- date: datum i format YYYY-MM-DD, eller null om ej läsbart.
+- dateKind: "best_before" för "bäst före", "use_by" för "sista förbrukningsdag", annars null.
+- confidence: 0.0–1.0.
+
+Svara ENDAST med giltig JSON:
+{
+  "date": "YYYY-MM-DD" | null,
+  "dateKind": "best_before" | "use_by" | null,
+  "confidence": number
+}
+
+Språk: ${lang}.`;
+  }
+
+  private buildReceiptPrompt(locale: LocaleContext): string {
+    const lang = locale.languageTag.startsWith("en") ? "English" : "Swedish";
+    return `Du är en OCR-assistent för svenska kvitton. Bilden visar ett butikskvitto.
+
+Läs av:
+- storeName: butikens namn, eller null.
+- purchaseDate: datum i format YYYY-MM-DD, eller null.
+- lines: varje rad med { rawText (exakt text), normalizedName (förenklat livsmedelsnamn eller null), canonicalIngredientId (alltid null), quantity (antal/enhet om går att utläsa, annars null), unit (enhetskod eller null), unitPriceMinor (ören per styck om synligt, annars null), totalPriceMinor (radens totalpris i ören om synligt, annars null), isDiscount (true om det är en rabatt/avdrag-rad), confidence (0.0–1.0) }.
+- totalMinor: kvittots totalsumma i ören, eller null.
+- discountTotalMinor: total rabatt i ören, eller null.
+- confidence: 0.0–1.0 för hela tolkningen.
+
+Svara ENDAST med giltig JSON som matchar detta schema:
+{
+  "storeName": string | null,
+  "purchaseDate": "YYYY-MM-DD" | null,
+  "lines": [
+    {
+      "rawText": string,
+      "normalizedName": string | null,
+      "canonicalIngredientId": null,
+      "quantity": number | null,
+      "unit": string | null,
+      "unitPriceMinor": number | null,
+      "totalPriceMinor": number | null,
+      "isDiscount": boolean,
+      "confidence": number
+    }
+  ],
+  "totalMinor": number | null,
+  "discountTotalMinor": number | null,
+  "confidence": number
+}
+
+Använd endast de enhetskoder du kan utläsa (GRAM, KILOGRAM, LITER, etc.), annars null. quantity är alltid ett decimaltal.
 Språk: ${lang}.`;
  }

@@ -127,12 +127,12 @@ describe("GeminiAamosClient", () => {
    });

    const result = await client.runTask(
-      "READ_RECEIPT",
-      { imageUrls: ["http://localhost/v1/mock-s3/receipt.jpg"], marketLocale: "sv-SE" },
+      "DEDUPLICATE_INVENTORY",
+      { items: [] },
    );

    expect(result.status).toBe("failed");
-    expect(result.error).toContain("READ_RECEIPT");
+    expect(result.error).toContain("DEDUPLICATE_INVENTORY");
  });

  it("validates input before calling Gemini", async () => {