/** * Gemini-lärar-tier för AAMOS-task-typer (SKIVA 1: kylskåpsskanning). * * - Implementerar samma AamosClient-interface som HttpAamosClient/MockAamosClient. * - App + Food API ser ingen skillnad; workern byter bara adapter. * - Nycklar finns serversidan; klienten exponeras aldrig i appen. * - Kostnad/tokens rapporteras tillbaka så workern kan bokföra i ai_usage_counters. * - Global dagsbudget kan sättas via GEMINI_DAILY_BUDGET_USD. */ import { randomUUID } from "node:crypto"; import { z } from "zod"; import { TASK_CONTRACTS, type AamosTaskType, type TaskInput, type TaskOutput, detectedItemSchema, type DetectedItem, generateRecipeCandidatesOutput, } from "./tasks.js"; import type { AamosCallOptions, AamosClient, AamosResult } from "./client.js"; import type { LocaleContext } from "@app/shared-types"; const GEMINI_API_BASE = "https://generativelanguage.googleapis.com/v1beta"; /** Custom fetch using Node https module — works around IPv6 hangs on WSL. */ function customNodeFetch(input: string | URL | Request, init?: RequestInit): Promise { return new Promise((resolve, reject) => { const url = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url; const u = new URL(url); const isHttps = u.protocol === "https:"; const mod = isHttps ? import("node:https") : import("node:http"); mod.then((httpMod) => { const postData = init?.body ? String(init.body) : undefined; const headers: Record = {}; if (init?.headers) { if (init.headers instanceof Headers) { init.headers.forEach((v, k) => { headers[k] = v; }); } else if (Array.isArray(init.headers)) { (init.headers as [string, string][]).forEach(([k, v]) => { if (k != null) headers[k] = v; }); } else { Object.assign(headers, init.headers as Record); } } if (postData && !headers["content-length"]) { headers["content-length"] = String(Buffer.byteLength(postData)); } const req = httpMod.request( { hostname: u.hostname, port: u.port || (isHttps ? 443 : 80), path: u.pathname + u.search, method: init?.method || "GET", headers, timeout: 120_000, }, (res) => { let body = ""; res.setEncoding("utf8"); res.on("data", (chunk) => { body += chunk; }); res.on("end", () => { resolve( new Response(body, { status: res.statusCode ?? 200, statusText: res.statusMessage ?? "OK", headers: new Headers(Object.entries(res.headers).map(([k, v]) => [k, String(v)])), }), ); }); }, ); req.on("error", (err) => reject(err)); req.on("timeout", () => { req.destroy(); reject(new Error("Request timeout")); }); if (postData) req.write(postData); req.end(); }).catch(reject); }); } /** Estimated max cost per image call in USD (pessimistic). */ const COST_ESTIMATE_PER_IMAGE_USD = 0.0015; export interface BudgetStore { /** Return current daily spend in USD. */ getDailySpendUsd(): Promise; /** Increment daily spend by amountUsd; return new total. */ incrementDailySpendUsd(amountUsd: number): Promise; } /** In-memory budget store for tests / single-process dev. */ export class MemoryBudgetStore implements BudgetStore { private spend = 0; async getDailySpendUsd(): Promise { return this.spend; } async incrementDailySpendUsd(amountUsd: number): Promise { this.spend += amountUsd; return this.spend; } reset() { this.spend = 0; } } export interface GeminiAamosClientConfig { apiKey: string; model?: string; timeoutMs?: number; dailyBudgetUsd?: number; budgetStore?: BudgetStore; fetchImpl?: typeof fetch; promptVersion?: string; } const geminiFridgeResponseSchema = z.object({ items: z.array( z.object({ produkt: z.string(), varumarke: z.string().nullable(), kvantitet: z.union([z.number(), z.string()]).nullable(), enhet: z.string().nullable(), kategori: z.string().nullable(), bastaFore: z.string().nullable().optional(), sistaForbruk: z.string().nullable().optional(), beskrivning: z.string().nullable().optional(), konfidens: z.number().min(0).max(1), }), ), imageQualityIssues: z .array(z.enum(["dark", "blurry", "occlusion", "too_far"])) .default([]), }); function parseDate(value: string | null | undefined): string | null { if (!value) return null; const normalized = value.trim(); if (!normalized) return null; // Accept ISO dates (YYYY-MM-DD) only. if (/^\d{4}-\d{2}-\d{2}$/.test(normalized)) return normalized; // Try common Swedish/EU formats. const parts = normalized.split(/[-/.]/); if (parts.length === 3) { const [aStr, bStr, cStr] = parts; if (!aStr || !bStr || !cStr) return null; const a = parseInt(aStr, 10); const b = parseInt(bStr, 10); const c = parseInt(cStr, 10); if (Number.isNaN(a) || Number.isNaN(b) || Number.isNaN(c)) return null; // YYYY-MM-DD if (a > 2000 && b <= 12 && c <= 31) return `${a}-${String(b).padStart(2, "0")}-${String(c).padStart(2, "0")}`; // DD-MM-YYYY if (c > 2000 && b <= 12 && a <= 31) return `${c}-${String(b).padStart(2, "0")}-${String(a).padStart(2, "0")}`; } return null; } function parseQuantity(value: string | number | null | undefined): { quantity: number | null; unit: string | null } { if (value == null) return { quantity: null, unit: null }; if (typeof value === "number") return { quantity: value, unit: null }; const text = String(value).trim().replace(/,/g, "."); if (!text) return { quantity: null, unit: null }; const match = text.match(/^(\d+(?:\.\d+)?)\s*(.*)$/); if (match && match[1] != null && match[2] != null) { const qty = parseFloat(match[1]); const unit = match[2].trim(); return { quantity: Number.isFinite(qty) ? qty : null, unit: unit || null }; } return { quantity: null, unit: text }; } function normalizeUnit(unit: string | null | undefined): string | null { if (unit == null) return null; const u = unit.toLowerCase().trim(); if (u === "g" || u === "gram") return "GRAM"; if (u === "kg" || u === "kilo" || u === "kilogram") return "KILOGRAM"; if (u === "ml" || u === "milliliter") return "MILLILITER"; if (u === "dl" || u === "deciliter") return "DECILITER"; if (u === "l" || u === "liter") return "LITER"; if (u === "st" || u === "styck" || u === "piece" || u === "pieces") return "COUNT"; if (u === "förpackning" || u === "paket" || u === "package") return "PACKAGE"; if (u === "portion" || u === "portioner") return "PORTION"; if (u === "klyfta" || u === "klyftor") return "CLOVE"; if (u === "skiva" || u === "skivor") return "SLICE"; return null; } function toDetectedItems(parsed: z.infer): DetectedItem[] { return parsed.items.map((item) => { const { quantity, unit } = parseQuantity(item.kvantitet); return detectedItemSchema.parse({ detectedName: item.produkt, canonicalIngredientId: null, brand: item.varumarke, estimatedQuantity: quantity, unit: normalizeUnit(unit ?? item.enhet), bestBeforeDate: parseDate(item.bastaFore ?? item.sistaForbruk), confidence: item.konfidens, requiresConfirmation: item.konfidens < 0.92, boundingBox: null, }); }); } export class GeminiAamosClient implements AamosClient { private readonly cfg: Required> & { fetchImpl: typeof fetch; budgetStore?: BudgetStore; }; constructor(config: GeminiAamosClientConfig) { this.cfg = { apiKey: config.apiKey, model: config.model ?? "gemini-2.5-flash", timeoutMs: config.timeoutMs ?? 60_000, dailyBudgetUsd: config.dailyBudgetUsd ?? 0, promptVersion: config.promptVersion ?? "gemini-fridge-v1", fetchImpl: config.fetchImpl ?? customNodeFetch, budgetStore: config.budgetStore, }; } async healthCheck(): Promise<{ ok: boolean; detail?: string }> { try { const url = `${GEMINI_API_BASE}/models?key=${this.cfg.apiKey}`; const res = await this.cfg.fetchImpl(url, { signal: AbortSignal.timeout(5_000) }); return { ok: res.ok, detail: `status ${res.status}` }; } catch (err) { return { ok: false, detail: err instanceof Error ? err.message : String(err) }; } } async runTask( taskType: T, input: TaskInput, options: AamosCallOptions = {}, ): Promise> { const contract = TASK_CONTRACTS[taskType]; const parsedInput = contract.input.safeParse(input); if (!parsedInput.success) { return { status: "failed", output: null, error: `Kontraktsfel (input) för ${taskType}: ${parsedInput.error.message}`, }; } switch (taskType) { case "ANALYZE_FRIDGE_IMAGE": case "ANALYZE_PANTRY_IMAGE": return this.analyzeStorageImage(taskType, parsedInput.data as TaskInput<"ANALYZE_FRIDGE_IMAGE">, options) as Promise>; case "GENERATE_RECIPE_CANDIDATES": return this.generateRecipeCandidates(parsedInput.data as TaskInput<"GENERATE_RECIPE_CANDIDATES">, options) as Promise>; default: return { status: "failed", output: null, error: `${taskType} stöds inte av Gemini-lärar-tier i SKIVA 1.`, } as AamosResult; } } private async analyzeStorageImage( taskType: "ANALYZE_FRIDGE_IMAGE" | "ANALYZE_PANTRY_IMAGE", input: TaskInput<"ANALYZE_FRIDGE_IMAGE">, options: AamosCallOptions, ): Promise> { const locale = options.localeContext ?? this.defaultLocale(); const started = Date.now(); const imageParts = await Promise.all( input.imageUrls.slice(0, 6).map(async (url) => { const res = await this.cfg.fetchImpl(url, { signal: AbortSignal.timeout(this.cfg.timeoutMs) }); if (!res.ok) throw new Error(`Kunde inte hämta bild: ${res.status} ${url}`); const buf = Buffer.from(await res.arrayBuffer()); return { inline_data: { mime_type: this.mimeType(buf), data: buf.toString("base64") } }; }), ); const estimatedCostUsd = imageParts.length * COST_ESTIMATE_PER_IMAGE_USD; if (await this.isOverBudget(estimatedCostUsd)) { return { status: "failed", output: null, error: "Global Gemini-dagsbudget är förbrukad.", }; } const prompt = this.buildFridgePrompt(taskType, locale, input.locationType); const payload = { contents: [{ parts: [{ text: prompt }, ...imageParts] }], generationConfig: { responseMimeType: "application/json", temperature: 0.2, }, }; const url = `${GEMINI_API_BASE}/models/${this.cfg.model}:generateContent?key=${this.cfg.apiKey}`; const res = await this.cfg.fetchImpl(url, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(payload), signal: AbortSignal.timeout(this.cfg.timeoutMs), }); if (!res.ok) { const body = await res.text(); return { status: "failed", output: null, error: `Gemini ${res.status}: ${body.slice(0, 500)}`, }; } const geminiBody = (await res.json()) as { candidates?: Array<{ content?: { parts?: Array<{ text?: string }> } }>; usageMetadata?: { promptTokenCount?: number; candidatesTokenCount?: number }; }; const text = geminiBody.candidates?.[0]?.content?.parts?.[0]?.text ?? ""; let parsed: z.infer; try { const json = JSON.parse(text); const safe = geminiFridgeResponseSchema.safeParse(json); if (!safe.success) { return { status: "failed", output: null, error: `Gemini-svar matchar inte schema: ${safe.error.message}` }; } parsed = safe.data; } catch { return { status: "failed", output: null, error: "Gemini-svar var inte giltig JSON." }; } const inputTokens = geminiBody.usageMetadata?.promptTokenCount ?? 0; const outputTokens = geminiBody.usageMetadata?.candidatesTokenCount ?? 0; const costUsd = this.estimateCostUsd(inputTokens, outputTokens); await this.recordSpend(costUsd); const latencyMs = Date.now() - started; const items = toDetectedItems(parsed); return { status: items.length > 0 ? "ok" : "uncertain", output: { items, imageQualityIssues: parsed.imageQualityIssues } as TaskOutput<"ANALYZE_FRIDGE_IMAGE">, modelVersion: this.cfg.model, promptVersion: this.cfg.promptVersion, latencyMs, costUsd, inputTokens, outputTokens, }; } private async generateRecipeCandidates( input: TaskInput<"GENERATE_RECIPE_CANDIDATES">, options: AamosCallOptions, ): Promise> { const locale = options.localeContext ?? this.defaultLocale(); const started = Date.now(); // Estimate cost: ~0.003 USD per candidate (text-only, pessimistic) const totalCandidates = input.targetMatrix.reduce((sum, t) => sum + t.count, 0); const estimatedCostUsd = totalCandidates * 0.003; if (await this.isOverBudget(estimatedCostUsd)) { return { status: "failed", output: null, error: "Global Gemini-dagsbudget är förbrukad.", }; } const prompt = this.buildRecipeGenerationPrompt(input, locale); const payload = { contents: [{ parts: [{ text: prompt }] }], generationConfig: { responseMimeType: "application/json", temperature: 0.3, }, }; const url = `${GEMINI_API_BASE}/models/${this.cfg.model}:generateContent?key=${this.cfg.apiKey}`; const res = await this.cfg.fetchImpl(url, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(payload), signal: AbortSignal.timeout(this.cfg.timeoutMs), }); if (!res.ok) { const body = await res.text(); return { status: "failed", output: null, error: `Gemini ${res.status}: ${body.slice(0, 500)}`, }; } const geminiBody = (await res.json()) as { candidates?: Array<{ content?: { parts?: Array<{ text?: string }> } }>; usageMetadata?: { promptTokenCount?: number; candidatesTokenCount?: number }; }; const text = geminiBody.candidates?.[0]?.content?.parts?.[0]?.text ?? ""; let parsed: z.infer; try { const json = JSON.parse(text); const safe = generateRecipeCandidatesOutput.safeParse(json); if (!safe.success) { return { status: "failed", output: null, error: `Gemini-svar matchar inte schema: ${safe.error.message}` }; } parsed = safe.data; } catch { return { status: "failed", output: null, error: "Gemini-svar var inte giltig JSON." }; } const inputTokens = geminiBody.usageMetadata?.promptTokenCount ?? 0; const outputTokens = geminiBody.usageMetadata?.candidatesTokenCount ?? 0; const costUsd = this.estimateCostUsd(inputTokens, outputTokens); await this.recordSpend(costUsd); // Normalisera enheter defensivt: Gemini kan ibland returnera "dl"/"msk"/"st". for (const c of parsed.candidates) { for (const ing of c.ingredients) { const normalized = normalizeUnit(String(ing.unit)); if (normalized) ing.unit = normalized as typeof ing.unit; } } const latencyMs = Date.now() - started; return { status: parsed.candidates.length > 0 ? "ok" : "uncertain", output: parsed as TaskOutput<"GENERATE_RECIPE_CANDIDATES">, modelVersion: this.cfg.model, promptVersion: "gemini-recipe-gen-v1", latencyMs, costUsd, inputTokens, outputTokens, }; } private buildRecipeGenerationPrompt( input: TaskInput<"GENERATE_RECIPE_CANDIDATES">, locale: LocaleContext, ): string { const lang = locale.languageTag.startsWith("en") ? "English" : "Swedish"; const catalog = input.canonicalIngredientsCatalog.map((i) => `- ${i.id} (${i.nameSv}, ${i.category}, enhet: ${i.defaultUnit}, vegan: ${i.isVegan}, veg: ${i.isVegetarian}, gluten: ${i.containsGluten}, laktos: ${i.containsLactose})` ).join("\n"); const targets = input.targetMatrix.map((t) => `- ${t.mealType} × ${t.mainIngredientId} × ${t.dietVariant}: ${t.count} st` ).join("\n"); const constraints = input.constraints; return `Du är en erfaren svensk matskribent som skriver vardagsrecept för svenska hushåll. VIKTIGAST: Använd ENDAST ingredienser från katalogen nedan. Hitta ALDRIG på ingredienser som inte finns i listan. Om ett recept behöver något som saknas, hoppa över det receptet och lägg till en notering i rejectedPrompts. Tillåtna ingredienser (canonical_ingredients): ${catalog} Önskade recept: ${targets} Begränsningar: - max förberedelsetid: ${constraints.maxPrepTimeMinutes ?? 60} min - max koktid: ${constraints.maxCookTimeMinutes ?? 45} min - portioner: ${constraints.portions ?? 4} - max kryddnivå: ${constraints.spiceLevelMax ?? 3} - undvik: ${constraints.avoidIngredients.join(", ") || "(ingen)"} Regler: - Recepten ska vara realistiska vardagsrätter för svenska hushåll. - Stegen ska vara tydliga, med rimliga tider och temperaturer. - Använd ENDAST dessa exakta enhetskoder: GRAM, KILOGRAM, MILLILITER, DECILITER, LITER, TEASPOON, TABLESPOON, CUP_US, COUNT, PORTION, PINCH, SLICE, CLOVE, CAN, PACKAGE. Aldrig "g", "dl", "msk", "tsk", "st". - Formuleringen ska vara mjölkprincips-vänlig: ingen svinnskam, inget "släng". - Varje ingrediens MÅSTE finnas i katalogen ovan. - Svara ENDAST med giltig JSON i exakt detta format: { "candidates": [ { "titleSv": "...", "descriptionSv": "...", "cuisine": "swedish", "mealTypes": ["dinner"], "prepTimeMinutes": 15, "cookTimeMinutes": 30, "portions": 4, "spiceLevel": 1, "ingredients": [ { "canonicalIngredientId": "kycklingfile", "displayNameSv": "kycklingfilé", "quantity": 500, "unit": "GRAM", "optional": false, "note": null } ], "steps": [ { "instructionSv": "...", "timerSeconds": null, "temperatureC": null, "tip": null } ], "storageGuidanceSv": "...", "mealPrepFriendly": false, "freezerFriendly": false, "confidence": 0.95 } ], "rejectedPrompts": [] } Språk: ${lang}.`; } private buildFridgePrompt( taskType: "ANALYZE_FRIDGE_IMAGE" | "ANALYZE_PANTRY_IMAGE", locale: LocaleContext, locationType: string, ): string { const place = taskType === "ANALYZE_FRIDGE_IMAGE" ? "kylskåpet" : "skafferiet"; const lang = locale.languageTag.startsWith("en") ? "English" : "Swedish"; return `Du är en noggrann livsmedelsassistent för svenska hushåll. Användaren har fotat ${place} (${locationType}). Identifiera varje livsmedelsprodukt du ser. För varje produkt, svara med: - produkt: produktens namn på svenska (t.ex. "Mellanmjölk", "Smör", "Vispgrädde") - varumarke: varumärke om det syns tydligt, annars null - kvantitet: siffra (t.ex. 1, 500, 1.5) - enhet: förkortning som l, g, ml, st, förpackning - kategori: t.ex. mejeri, frukt, grönt, kött, skafferi - bastaFore: bäst-före-datum om det syns (YYYY-MM-DD), annars null - sistaForbruk: sista förbrukningsdatum om det syns (YYYY-MM-DD), annars null - konfidens: 0.0–1.0 Svara ENDAST med giltig JSON i exakt detta format: { "items": [...], "imageQualityIssues": [] } imageQualityIssues kan innehålla någon av: "dark", "blurry", "occlusion", "too_far". Språk: ${lang}.`; } private defaultLocale(): LocaleContext { return { languageTag: "sv-SE", regionCode: "SE", timeZone: "Europe/Stockholm", measurementSystem: "METRIC", temperatureUnit: "CELSIUS", currencyCode: "SEK", }; } private mimeType(buf: Buffer): string { if (buf[0] === 0xff && buf[1] === 0xd8) return "image/jpeg"; if (buf.slice(0, 8).toString("hex") === "89504e470d0a1a0a") return "image/png"; if (buf.slice(0, 4).toString("ascii") === "RIFF") return "image/webp"; return "image/jpeg"; } private estimateCostUsd(inputTokens: number, outputTokens: number): number { // Gemini 2.5 Flash pricing (Aug 2025): ~$0.075/1M input, $0.30/1M output. return inputTokens * 0.075e-6 + outputTokens * 0.30e-6; } private async isOverBudget(estimateUsd: number): Promise { if (this.cfg.dailyBudgetUsd <= 0 || !this.cfg.budgetStore) return false; const current = await this.cfg.budgetStore.getDailySpendUsd(); return current + estimateUsd > this.cfg.dailyBudgetUsd; } private async recordSpend(costUsd: number): Promise { if (this.cfg.budgetStore && costUsd > 0) { await this.cfg.budgetStore.incrementDailySpendUsd(costUsd); } } }