diff --git a/apps/worker/package.json b/apps/worker/package.json index cc596b6..101fbf2 100644 --- a/apps/worker/package.json +++ b/apps/worker/package.json @@ -11,7 +11,8 @@ "typecheck": "tsc --noEmit", "test": "vitest run --passWithNoTests", "eval": "tsx src/eval/run.ts", - "eval:scan": "tsx src/eval/scan-eval.ts" + "eval:scan": "tsx src/eval/scan-eval.ts", + "eval:capture": "tsx src/eval/capture-eval.ts" }, "dependencies": { "@app/ai-contracts": "workspace:*", @@ -25,9 +26,11 @@ "@app/recommendation-engine": "workspace:*", "@app/shared-types": "workspace:*", "@app/subscriptions": "workspace:*", + "@aws-sdk/client-s3": "^3.750.0", "bullmq": "^6.0.0", "dotenv": "^16.4.0", "drizzle-orm": "^0.45.0", + "@aws-sdk/client-s3": "^3.750.0", "ioredis": "^6.0.0", "pg": "^8.13.0" }, diff --git a/apps/worker/src/eval/capture-eval.ts b/apps/worker/src/eval/capture-eval.ts new file mode 100644 index 0000000..08f3e1f --- /dev/null +++ b/apps/worker/src/eval/capture-eval.ts @@ -0,0 +1,52 @@ +/** + * Best-effort verifiering av shadow-capture (ingen riktig AI, bara lagring). + * + * S3_MODE=mock pnpm --filter @app/worker eval:capture + */ +import { captureTrainingSample } from "../lib/shadow-capture.js"; + +async function main() { + const result = await captureTrainingSample({ + taskType: "ANALYZE_FRIDGE_IMAGE", + inputS3Keys: ["uploads/test/fridge.jpg"], + output: { + items: [ + { + detectedName: "Testmjölk", + canonicalIngredientId: null, + brand: "Testa", + estimatedQuantity: 1, + unit: "LITER", + bestBeforeDate: null, + confidence: 0.97, + requiresConfirmation: false, + boundingBox: null, + }, + ], + imageQualityIssues: [], + }, + modelVersion: "gemini-2.5-flash", + promptVersion: "gemini-fridge-v1", + latencyMs: 1234, + costUsd: 0.0001, + inputTokens: 400, + outputTokens: 120, + consentFlags: { + personalization: true, + anonymizedImprovement: true, + imageTraining: true, + }, + readUrl: (key: string) => `http://localhost/v1/mock-s3/${encodeURIComponent(key)}?sig=dummy`, + }); + + if (!result.ok) { + console.error("[eval:capture] misslyckades:", result.error); + process.exit(1); + } + console.log(`[eval:capture] träningspar sparat på nyckel: ${result.key}`); +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/apps/worker/src/eval/scan-eval.ts b/apps/worker/src/eval/scan-eval.ts index 13fee70..43b1b1c 100644 --- a/apps/worker/src/eval/scan-eval.ts +++ b/apps/worker/src/eval/scan-eval.ts @@ -40,7 +40,7 @@ interface ScanEvalCase { const CASES: ScanEvalCase[] = [ { id: "arla-milk-fridge", - imageUrls: ["https://upload.wikimedia.org/wikipedia/commons/6/6c/Arla_Ko_Mellanmj%C3%B6lk_1L.jpg"], + imageUrls: ["https://commons.wikimedia.org/wiki/Special:FilePath/Arla_Ko_Mellanmj%C3%B6lk_1L.jpg?width=800"], locationType: "fridge", marketLocale: "sv-SE", checks: (items) => { @@ -56,7 +56,7 @@ const CASES: ScanEvalCase[] = [ }, { id: "swedish-butter-fridge", - imageUrls: ["https://upload.wikimedia.org/wikipedia/commons/3/3e/Svenskt_Sm%C3%B6r_Normalsaltat_80-25_500g.jpg"], + imageUrls: ["https://commons.wikimedia.org/wiki/Special:FilePath/Svenskt_Sm%C3%B6r_Normalsaltat_80-25_500g.jpg?width=800"], locationType: "fridge", marketLocale: "sv-SE", checks: (items) => { diff --git a/apps/worker/src/lib/shadow-capture.ts b/apps/worker/src/lib/shadow-capture.ts new file mode 100644 index 0000000..fc9fbc0 --- /dev/null +++ b/apps/worker/src/lib/shadow-capture.ts @@ -0,0 +1,179 @@ +/** + * Shadow-capture: sparar varje lyckad AI-inferens som träningsdata i S3. + * + * - Kör aldrig synkront på användarens kritiska väg; failures swallås. + * - Ingen PII (inga userId/householdId). Endast S3-nycklar, output och metadata. + * - Rå bild kopieras ENDAST när imageTraining-samtycke är true. + * - Durabelt = S3 i prod, lokalt filsystem i mock/test. + */ +import { randomUUID } from "node:crypto"; +import { mkdir, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { + PutObjectCommand, + S3Client, + type S3ClientConfig, +} from "@aws-sdk/client-s3"; +import type { AamosTaskType } from "@app/ai-contracts"; + +export interface CaptureConsentFlags { + personalization: boolean; + anonymizedImprovement: boolean; + imageTraining: boolean; +} + +export interface CaptureResult { + ok: boolean; + key?: string; + error?: string; +} + +export interface CaptureInput { + taskType: AamosTaskType; + inputS3Keys: string[]; + output: Record; + modelVersion?: string | null; + promptVersion?: string | null; + latencyMs?: number | null; + costUsd?: number | null; + inputTokens?: number | null; + outputTokens?: number | null; + consentFlags: CaptureConsentFlags; + readUrl: (key: string) => string; + /** Override for tests; otherwise built from env. */ + storage?: CaptureStorage; +} + +export interface CaptureStorage { + put(key: string, data: Buffer, contentType: string): Promise; +} + +const TRAINING_PREFIX = "training/cibello"; +const MOCK_ROOT = path.resolve(process.cwd(), ".data/s3-training"); + +class S3CaptureStorage implements CaptureStorage { + private readonly client: S3Client; + private readonly bucket: string; + + constructor(bucket: string, config: S3ClientConfig) { + this.bucket = bucket; + this.client = new S3Client(config); + } + + async put(key: string, data: Buffer, contentType: string): Promise { + await this.client.send( + new PutObjectCommand({ + Bucket: this.bucket, + Key: key, + Body: data, + ContentType: contentType, + }), + ); + } +} + +class LocalCaptureStorage implements CaptureStorage { + async put(key: string, data: Buffer): Promise { + const filePath = path.join(MOCK_ROOT, key); + await mkdir(path.dirname(filePath), { recursive: true }); + await writeFile(filePath, data); + } +} + +let cachedStorage: CaptureStorage | undefined; + +export function createCaptureStorage(): CaptureStorage { + if (cachedStorage) return cachedStorage; + + const mode = process.env.S3_MODE ?? "mock"; + if (mode === "aws") { + const bucket = process.env.S3_BUCKET ?? "cibello-production"; + const config: S3ClientConfig = { + region: process.env.S3_REGION || "eu-north-1", + ...(process.env.S3_ENDPOINT + ? { endpoint: process.env.S3_ENDPOINT, forcePathStyle: true } + : {}), + ...(process.env.S3_ACCESS_KEY_ID && process.env.S3_SECRET_ACCESS_KEY + ? { + credentials: { + accessKeyId: process.env.S3_ACCESS_KEY_ID, + secretAccessKey: process.env.S3_SECRET_ACCESS_KEY, + }, + } + : {}), + }; + cachedStorage = new S3CaptureStorage(bucket, config); + } else { + cachedStorage = new LocalCaptureStorage(); + } + return cachedStorage; +} + +export async function captureTrainingSample(input: CaptureInput): Promise { + try { + const storage = input.storage ?? createCaptureStorage(); + const date = new Date().toISOString().slice(0, 10); + const id = randomUUID(); + const prefix = `${TRAINING_PREFIX}/${input.taskType}/${date}/${id}`; + const jsonKey = `${prefix}/sample.json`; + + const sample: Record = { + taskType: input.taskType, + inputReferences: input.inputS3Keys, + output: input.output, + modelVersion: input.modelVersion ?? null, + promptVersion: input.promptVersion ?? null, + latencyMs: input.latencyMs ?? null, + costUsd: input.costUsd ?? null, + inputTokens: input.inputTokens ?? null, + outputTokens: input.outputTokens ?? null, + consentFlags: input.consentFlags, + capturedAt: new Date().toISOString(), + }; + + const trainingImageKeys: string[] = []; + if (input.consentFlags.imageTraining && input.inputS3Keys.length > 0) { + for (let i = 0; i < input.inputS3Keys.length; i++) { + const originalKey = input.inputS3Keys[i]; + if (!originalKey) continue; + try { + const imageKey = `${prefix}/images/${String(i).padStart(3, "0")}.jpg`; + const imageData = await fetchImageData(input.readUrl(originalKey)); + if (imageData) { + await storage.put(imageKey, imageData, "image/jpeg"); + trainingImageKeys.push(imageKey); + } + } catch (err) { + // Best effort: log but continue; JSON still has original S3 references. + console.error(`[shadow-capture] failed to copy image ${originalKey}:`, err); + } + } + } + + if (trainingImageKeys.length > 0) { + sample.trainingImageKeys = trainingImageKeys; + } + + await storage.put(jsonKey, Buffer.from(JSON.stringify(sample, null, 2)), "application/json"); + return { ok: true, key: jsonKey }; + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + console.error("[shadow-capture] failed:", message); + return { ok: false, error: message }; + } +} + +async function fetchImageData(url: string): Promise { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), 30_000); + try { + const res = await fetch(url, { signal: controller.signal }); + clearTimeout(timer); + if (!res.ok) return null; + return Buffer.from(await res.arrayBuffer()); + } catch { + return null; + } finally { + clearTimeout(timer); + } +} diff --git a/apps/worker/src/processors/scans.ts b/apps/worker/src/processors/scans.ts index b5112a3..fbf6c3d 100644 --- a/apps/worker/src/processors/scans.ts +++ b/apps/worker/src/processors/scans.ts @@ -6,6 +6,7 @@ import type { WorkerContext } from "../context.js"; import { getLocaleContext } from "../locale.js"; import type { LocaleContext } from "@app/shared-types"; import { recordAiUsage } from "../lib/ai-usage.js"; +import { captureTrainingSample, type CaptureConsentFlags } from "../lib/shadow-capture.js"; /** * Bild-/OCR-jobb (spec §54): hämtar scan_job, anropar AAMOS med kontraktvaliderad @@ -28,7 +29,7 @@ export async function processScanJob(ctx: WorkerContext, scanJobId: string): Pro const imageUrls = job.s3Keys.map((k) => ctx.readUrl(k)); const localeContext = await getLocaleContext(ctx, job.userId); - const consents = await loadConsentFlags(ctx, job.userId); + const consentFlags = await loadConsentFlags(ctx, job.userId); const started = Date.now(); const result = await runAamosForJob( @@ -38,6 +39,7 @@ export async function processScanJob(ctx: WorkerContext, scanJobId: string): Pro imageUrls, job.context, localeContext, + consentFlags, ); if (result.status === "failed" || result.output == null) { @@ -80,6 +82,27 @@ export async function processScanJob(ctx: WorkerContext, scanJobId: string): Pro await recordScanCompleted(ctx, job, result); + // Shadow-capture: spara träningspar för framtida AAMOS-distillation (FAS 1). + // Kör aldrig synkront på användarens kritiska väg; fel swallås. + if (result.status === "ok" && result.output != null) { + const capture = await captureTrainingSample({ + taskType: job.jobType as AamosTaskType, + inputS3Keys: job.s3Keys, + output: result.output as Record, + modelVersion: result.modelVersion, + promptVersion: result.promptVersion, + latencyMs: result.latencyMs, + costUsd: result.costUsd, + inputTokens: result.inputTokens, + outputTokens: result.outputTokens, + consentFlags, + readUrl: ctx.readUrl, + }); + if (!capture.ok) { + console.error("[scan processor] shadow-capture misslyckades:", capture.error); + } + } + // Bokför verklig AI-kostnad/tokens utan PII (spec §45). if (result.costUsd != null || result.inputTokens || result.outputTokens) { await recordAiUsage(ctx.db, job.userId, { @@ -107,7 +130,6 @@ export async function processScanJob(ctx: WorkerContext, scanJobId: string): Pro }, }); } - void consents; } async function runAamosForJob( @@ -117,6 +139,7 @@ async function runAamosForJob( imageUrls: string[], context: unknown, localeContext: LocaleContext, + consentFlags: CaptureConsentFlags, ) { switch (jobType) { case "ANALYZE_FRIDGE_IMAGE": @@ -129,33 +152,33 @@ async function runAamosForJob( marketLocale: localeContext.languageTag, knownItems: [], }, - { localeContext }, + { localeContext, consentFlags }, ); case "ANALYZE_MEAL_IMAGE": { const recipeContext = await buildRecipeContext(ctx, context); return ctx.aamos.runTask( "ANALYZE_MEAL_IMAGE", { imageUrls, recipeContext, marketLocale: localeContext.languageTag }, - { localeContext }, + { localeContext, consentFlags }, ); } case "READ_RECEIPT": return ctx.aamos.runTask( "READ_RECEIPT", { imageUrls, marketLocale: localeContext.languageTag }, - { localeContext }, + { localeContext, consentFlags }, ); case "READ_NUTRITION_LABEL": return ctx.aamos.runTask( "READ_NUTRITION_LABEL", { imageUrls, marketLocale: localeContext.languageTag }, - { localeContext }, + { localeContext, consentFlags }, ); case "READ_EXPIRY_DATE": return ctx.aamos.runTask( "READ_EXPIRY_DATE", { imageUrls: imageUrls.slice(0, 2) }, - { localeContext }, + { localeContext, consentFlags }, ); default: throw new Error(`Jobbtypen ${jobType} hanteras inte av scan-processorn`); diff --git a/packages/ai-contracts/src/gemini.ts b/packages/ai-contracts/src/gemini.ts index d29ea58..9bbbc98 100644 --- a/packages/ai-contracts/src/gemini.ts +++ b/packages/ai-contracts/src/gemini.ts @@ -1,5 +1,5 @@ /** - * Gemini-lärar-tier för AAMOS-task-typer (SKIVA 1: kylskåpsskanning). + * Gemini-lärar-tier för AAMOS-task-typer (SKIVA 1). * * - Implementerar samma AamosClient-interface som HttpAamosClient/MockAamosClient. * - App + Food API ser ingen skillnad; workern byter bara adapter. @@ -7,7 +7,6 @@ * - Kostnad/tokens rapporteras tillbaka så workern kan bokföra i ai_usage_counters. * - Global dagsbudget kan sättas via GEMINI_DAILY_BUDGET_USD. */ -import { randomUUID } from "node:crypto"; import { z } from "zod"; import { TASK_CONTRACTS, @@ -17,14 +16,22 @@ import { detectedItemSchema, type DetectedItem, generateRecipeCandidatesOutput, + analyzeMealImageOutput, + readNutritionLabelOutput, + readExpiryDateOutput, + readReceiptOutput, } from "./tasks.js"; import type { AamosCallOptions, AamosClient, AamosResult } from "./client.js"; import type { LocaleContext } from "@app/shared-types"; const GEMINI_API_BASE = "https://generativelanguage.googleapis.com/v1beta"; -/** Custom fetch using Node https module — works around IPv6 hangs on WSL. */ +/** Custom fetch using Node https module — works around IPv6 hangs on WSL. Follows redirects. */ function customNodeFetch(input: string | URL | Request, init?: RequestInit): Promise { + return requestOnce(input, init, 5); +} + +function requestOnce(input: string | URL | Request, init: RequestInit | undefined, redirectsLeft: number): Promise { return new Promise((resolve, reject) => { const url = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url; const u = new URL(url); @@ -45,6 +52,9 @@ function customNodeFetch(input: string | URL | Request, init?: RequestInit): Pro if (postData && !headers["content-length"]) { headers["content-length"] = String(Buffer.byteLength(postData)); } + if (!headers["user-agent"]) { + headers["user-agent"] = "Mozilla/5.0 (compatible; Cibello/1.0)"; + } const req = httpMod.request( { @@ -56,13 +66,23 @@ function customNodeFetch(input: string | URL | Request, init?: RequestInit): Pro timeout: 120_000, }, (res) => { + const status = res.statusCode ?? 200; + const location = res.headers.location; + if (status >= 300 && status < 400 && location && redirectsLeft > 0) { + const nextUrl = new URL(location, url).toString(); + requestOnce(nextUrl, { ...init, method: "GET", body: undefined }, redirectsLeft - 1) + .then(resolve) + .catch(reject); + return; + } + let body = ""; res.setEncoding("utf8"); res.on("data", (chunk) => { body += chunk; }); res.on("end", () => { resolve( new Response(body, { - status: res.statusCode ?? 200, + status, statusText: res.statusMessage ?? "OK", headers: new Headers(Object.entries(res.headers).map(([k, v]) => [k, String(v)])), }), @@ -81,6 +101,9 @@ function customNodeFetch(input: string | URL | Request, init?: RequestInit): Pro /** Estimated max cost per image call in USD (pessimistic). */ const COST_ESTIMATE_PER_IMAGE_USD = 0.0015; +/** Estimated max cost per simple OCR image call in USD. */ +const COST_ESTIMATE_PER_OCR_IMAGE_USD = 0.001; + export interface BudgetStore { /** Return current daily spend in USD. */ getDailySpendUsd(): Promise; @@ -222,7 +245,7 @@ export class GeminiAamosClient implements AamosClient { async healthCheck(): Promise<{ ok: boolean; detail?: string }> { try { - const url = `${GEMINI_API_BASE}/models?key=${this.cfg.apiKey}`; + const url = `${GEMINI_API_BASE}/models?key=${encodeURIComponent(this.cfg.apiKey)}`; const res = await this.cfg.fetchImpl(url, { signal: AbortSignal.timeout(5_000) }); return { ok: res.ok, detail: `status ${res.status}` }; } catch (err) { @@ -251,6 +274,14 @@ export class GeminiAamosClient implements AamosClient { return this.analyzeStorageImage(taskType, parsedInput.data as TaskInput<"ANALYZE_FRIDGE_IMAGE">, options) as Promise>; case "GENERATE_RECIPE_CANDIDATES": return this.generateRecipeCandidates(parsedInput.data as TaskInput<"GENERATE_RECIPE_CANDIDATES">, options) as Promise>; + case "ANALYZE_MEAL_IMAGE": + return this.analyzeMealImage(parsedInput.data as TaskInput<"ANALYZE_MEAL_IMAGE">, options) as Promise>; + case "READ_NUTRITION_LABEL": + return this.readNutritionLabel(parsedInput.data as TaskInput<"READ_NUTRITION_LABEL">, options) as Promise>; + case "READ_EXPIRY_DATE": + return this.readExpiryDate(parsedInput.data as TaskInput<"READ_EXPIRY_DATE">, options) as Promise>; + case "READ_RECEIPT": + return this.readReceipt(parsedInput.data as TaskInput<"READ_RECEIPT">, options) as Promise>; default: return { status: "failed", @@ -295,7 +326,7 @@ export class GeminiAamosClient implements AamosClient { }, }; - const url = `${GEMINI_API_BASE}/models/${this.cfg.model}:generateContent?key=${this.cfg.apiKey}`; + const url = `${GEMINI_API_BASE}/models/${this.cfg.model}:generateContent?key=${encodeURIComponent(this.cfg.apiKey)}`; const res = await this.cfg.fetchImpl(url, { method: "POST", headers: { "content-type": "application/json" }, @@ -350,6 +381,186 @@ export class GeminiAamosClient implements AamosClient { }; } + private async runVisionTask( + taskType: T, + imageUrls: string[], + prompt: string, + outputSchema: z.ZodSchema, + mapOutput: (parsed: unknown) => TaskOutput, + options: AamosCallOptions, + promptVersion: string, + costEstimatePerImage: number, + ): Promise> { + const locale = options.localeContext ?? this.defaultLocale(); + const started = Date.now(); + + const imageParts = await this.fetchImageParts(imageUrls, this.cfg.timeoutMs); + const estimatedCostUsd = imageParts.length * costEstimatePerImage; + if (await this.isOverBudget(estimatedCostUsd)) { + return { + status: "failed", + output: null, + error: "Global Gemini-dagsbudget är förbrukad.", + } as AamosResult; + } + + const payload = { + contents: [{ parts: [{ text: prompt }, ...imageParts] }], + generationConfig: { + responseMimeType: "application/json", + temperature: 0.2, + }, + }; + + const url = `${GEMINI_API_BASE}/models/${this.cfg.model}:generateContent?key=${encodeURIComponent(this.cfg.apiKey)}`; + const res = await this.cfg.fetchImpl(url, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(payload), + signal: AbortSignal.timeout(this.cfg.timeoutMs), + }); + + if (!res.ok) { + const body = await res.text(); + return { + status: "failed", + output: null, + error: `Gemini ${res.status}: ${body.slice(0, 500)}`, + } as AamosResult; + } + + const geminiBody = (await res.json()) as { + candidates?: Array<{ content?: { parts?: Array<{ text?: string }> } }>; + usageMetadata?: { promptTokenCount?: number; candidatesTokenCount?: number }; + }; + const text = geminiBody.candidates?.[0]?.content?.parts?.[0]?.text ?? ""; + + let parsed: unknown; + try { + const json = JSON.parse(text); + const safe = outputSchema.safeParse(json); + if (!safe.success) { + return { + status: "failed", + output: null, + error: `Gemini-svar matchar inte schema: ${safe.error.message}`, + } as AamosResult; + } + parsed = safe.data; + } catch { + return { + status: "failed", + output: null, + error: "Gemini-svar var inte giltig JSON.", + } as AamosResult; + } + + const inputTokens = geminiBody.usageMetadata?.promptTokenCount ?? 0; + const outputTokens = geminiBody.usageMetadata?.candidatesTokenCount ?? 0; + const costUsd = this.estimateCostUsd(inputTokens, outputTokens); + await this.recordSpend(costUsd); + + const latencyMs = Date.now() - started; + const output = mapOutput(parsed) as TaskOutput; + + return { + status: "ok", + output, + modelVersion: this.cfg.model, + promptVersion, + latencyMs, + costUsd, + inputTokens, + outputTokens, + }; + } + + private async analyzeMealImage( + input: TaskInput<"ANALYZE_MEAL_IMAGE">, + options: AamosCallOptions, + ): Promise> { + const locale = options.localeContext ?? this.defaultLocale(); + const prompt = this.buildMealPrompt(locale, input.recipeContext); + return this.runVisionTask( + "ANALYZE_MEAL_IMAGE", + input.imageUrls, + prompt, + analyzeMealImageOutput, + (parsed) => parsed as TaskOutput<"ANALYZE_MEAL_IMAGE">, + options, + "gemini-meal-v1", + COST_ESTIMATE_PER_IMAGE_USD, + ); + } + + private async readNutritionLabel( + input: TaskInput<"READ_NUTRITION_LABEL">, + options: AamosCallOptions, + ): Promise> { + const locale = options.localeContext ?? this.defaultLocale(); + const prompt = this.buildNutritionPrompt(locale); + return this.runVisionTask( + "READ_NUTRITION_LABEL", + input.imageUrls, + prompt, + readNutritionLabelOutput, + (parsed) => parsed as TaskOutput<"READ_NUTRITION_LABEL">, + options, + "gemini-nutrition-v1", + COST_ESTIMATE_PER_OCR_IMAGE_USD, + ); + } + + private async readExpiryDate( + input: TaskInput<"READ_EXPIRY_DATE">, + options: AamosCallOptions, + ): Promise> { + const locale = options.localeContext ?? this.defaultLocale(); + const prompt = this.buildExpiryPrompt(locale); + return this.runVisionTask( + "READ_EXPIRY_DATE", + input.imageUrls, + prompt, + readExpiryDateOutput, + (parsed) => parsed as TaskOutput<"READ_EXPIRY_DATE">, + options, + "gemini-expiry-v1", + COST_ESTIMATE_PER_OCR_IMAGE_USD, + ); + } + + private async readReceipt( + input: TaskInput<"READ_RECEIPT">, + options: AamosCallOptions, + ): Promise> { + const locale = options.localeContext ?? this.defaultLocale(); + const prompt = this.buildReceiptPrompt(locale); + return this.runVisionTask( + "READ_RECEIPT", + input.imageUrls, + prompt, + readReceiptOutput, + (parsed) => parsed as TaskOutput<"READ_RECEIPT">, + options, + "gemini-receipt-v1", + COST_ESTIMATE_PER_IMAGE_USD, + ); + } + + private async fetchImageParts( + imageUrls: string[], + timeoutMs: number, + ): Promise> { + return Promise.all( + imageUrls.slice(0, 6).map(async (url) => { + const res = await this.cfg.fetchImpl(url, { signal: AbortSignal.timeout(timeoutMs) }); + if (!res.ok) throw new Error(`Kunde inte hämta bild: ${res.status} ${url}`); + const buf = Buffer.from(await res.arrayBuffer()); + return { inline_data: { mime_type: this.mimeType(buf), data: buf.toString("base64") } }; + }), + ); + } + private async generateRecipeCandidates( input: TaskInput<"GENERATE_RECIPE_CANDIDATES">, options: AamosCallOptions, @@ -377,7 +588,7 @@ export class GeminiAamosClient implements AamosClient { }, }; - const url = `${GEMINI_API_BASE}/models/${this.cfg.model}:generateContent?key=${this.cfg.apiKey}`; + const url = `${GEMINI_API_BASE}/models/${this.cfg.model}:generateContent?key=${encodeURIComponent(this.cfg.apiKey)}`; const res = await this.cfg.fetchImpl(url, { method: "POST", headers: { "content-type": "application/json" }, @@ -553,6 +764,129 @@ Svara ENDAST med giltig JSON i exakt detta format: } imageQualityIssues kan innehålla någon av: "dark", "blurry", "occlusion", "too_far". +Språk: ${lang}.`; + } + + private buildMealPrompt(locale: LocaleContext, recipeContext: unknown): string { + const lang = locale.languageTag.startsWith("en") ? "English" : "Swedish"; + const ctx = recipeContext as { + recipeId?: string; + titleSv?: string; + nutritionPerPortion?: Record; + portions?: number; + } | null; + + let contextLine = ""; + if (ctx?.titleSv) { + contextLine = `Användaren säger att måltiden ska matcha receptet "${ctx.titleSv}" (${ctx.portions ?? "?"} portioner). `; + } + + return `Du är en svensk näringsassistent. ${contextLine}Bilden visar en serverad måltid. + +Uppskatta: +- matchesRecipeContext: true/false/null – matchar bilden receptet användaren nämnde? +- portionFractionEstimate: hur stor del av en normalportion ser det ut att vara? 0.0–5.0, eller null. +- kcalRange: { min, max, mostLikely } för hela måltiden, eller null. +- components: lista med uppskattade komponenter (namn, ev. canonicalIngredientId som null, estimatedGrams, confidence). +- confidence: 0.0–1.0 för hela tolkningen. + +Svara ENDAST med giltig JSON som matchar detta schema: +{ + "matchesRecipeContext": boolean | null, + "portionFractionEstimate": number | null, + "kcalRange": { "min": number, "max": number, "mostLikely": number } | null, + "components": [ + { "name": string, "canonicalIngredientId": string | null, "estimatedGrams": number | null, "confidence": number } + ], + "confidence": number +} + +Språk: ${lang}.`; + } + + private buildNutritionPrompt(locale: LocaleContext): string { + const lang = locale.languageTag.startsWith("en") ? "English" : "Swedish"; + return `Du är en OCR-assistent för svenska livsmedelsförpackningar. Bilden visar en näringsdeklaration/ingredienslista. + +Läs av exakt det som står på etiketten. Gissa aldrig värden som inte syns. Returnera: +- basis: "per_100_g", "per_100_ml" eller "per_portion" – vad gäller tabellen? +- values: { kcal, proteinG, carbsG, fatG, saturatedFatG, fiberG, sugarG, saltG } – null om ej synligt. +- ingredientsText: rå text med ingredienser, eller null. +- allergensDeclared: lista med allergener som explicit nämns (t.ex. ["gluten", "mjölk", "nötter"]). +- gtin: streckkod/EAN om synlig, annars null. +- productName: produktnamn om synligt, annars null. +- brand: varumärke om synligt, annars null. +- confidence: 0.0–1.0. + +Svara ENDAST med giltig JSON som matchar detta schema: +{ + "basis": "per_100_g" | "per_100_ml" | "per_portion" | null, + "values": { "kcal": number|null, "proteinG": number|null, "carbsG": number|null, "fatG": number|null, "saturatedFatG": number|null, "fiberG": number|null, "sugarG": number|null, "saltG": number|null } | null, + "ingredientsText": string | null, + "allergensDeclared": string[], + "gtin": string | null, + "productName": string | null, + "brand": string | null, + "confidence": number +} + +Språk: ${lang}.`; + } + + private buildExpiryPrompt(locale: LocaleContext): string { + const lang = locale.languageTag.startsWith("en") ? "English" : "Swedish"; + return `Du är en OCR-assistent för svenska livsmedelsförpackningar. Bilden visar ett bäst-före-datum eller sista förbrukningsdatum. + +Returnera: +- date: datum i format YYYY-MM-DD, eller null om ej läsbart. +- dateKind: "best_before" för "bäst före", "use_by" för "sista förbrukningsdag", annars null. +- confidence: 0.0–1.0. + +Svara ENDAST med giltig JSON: +{ + "date": "YYYY-MM-DD" | null, + "dateKind": "best_before" | "use_by" | null, + "confidence": number +} + +Språk: ${lang}.`; + } + + private buildReceiptPrompt(locale: LocaleContext): string { + const lang = locale.languageTag.startsWith("en") ? "English" : "Swedish"; + return `Du är en OCR-assistent för svenska kvitton. Bilden visar ett butikskvitto. + +Läs av: +- storeName: butikens namn, eller null. +- purchaseDate: datum i format YYYY-MM-DD, eller null. +- lines: varje rad med { rawText (exakt text), normalizedName (förenklat livsmedelsnamn eller null), canonicalIngredientId (alltid null), quantity (antal/enhet om går att utläsa, annars null), unit (enhetskod eller null), unitPriceMinor (ören per styck om synligt, annars null), totalPriceMinor (radens totalpris i ören om synligt, annars null), isDiscount (true om det är en rabatt/avdrag-rad), confidence (0.0–1.0) }. +- totalMinor: kvittots totalsumma i ören, eller null. +- discountTotalMinor: total rabatt i ören, eller null. +- confidence: 0.0–1.0 för hela tolkningen. + +Svara ENDAST med giltig JSON som matchar detta schema: +{ + "storeName": string | null, + "purchaseDate": "YYYY-MM-DD" | null, + "lines": [ + { + "rawText": string, + "normalizedName": string | null, + "canonicalIngredientId": null, + "quantity": number | null, + "unit": string | null, + "unitPriceMinor": number | null, + "totalPriceMinor": number | null, + "isDiscount": boolean, + "confidence": number + } + ], + "totalMinor": number | null, + "discountTotalMinor": number | null, + "confidence": number +} + +Använd endast de enhetskoder du kan utläsa (GRAM, KILOGRAM, LITER, etc.), annars null. quantity är alltid ett decimaltal. Språk: ${lang}.`; } diff --git a/packages/ai-contracts/test/gemini.test.ts b/packages/ai-contracts/test/gemini.test.ts index dd3dfaa..25b9b8b 100644 --- a/packages/ai-contracts/test/gemini.test.ts +++ b/packages/ai-contracts/test/gemini.test.ts @@ -127,12 +127,12 @@ describe("GeminiAamosClient", () => { }); const result = await client.runTask( - "READ_RECEIPT", - { imageUrls: ["http://localhost/v1/mock-s3/receipt.jpg"], marketLocale: "sv-SE" }, + "DEDUPLICATE_INVENTORY", + { items: [] }, ); expect(result.status).toBe("failed"); - expect(result.error).toContain("READ_RECEIPT"); + expect(result.error).toContain("DEDUPLICATE_INVENTORY"); }); it("validates input before calling Gemini", async () => { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5244d15..b8ad0ba 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -296,6 +296,9 @@ importers: '@app/subscriptions': specifier: workspace:* version: link:../../packages/subscriptions + '@aws-sdk/client-s3': + specifier: ^3.750.0 + version: 3.1102.0 bullmq: specifier: ^6.0.0 version: 6.0.5(ioredis@6.0.0)(pg@8.22.0) diff --git a/scripts/brand-guard.sh b/scripts/brand-guard.sh index fa9fd80..4b95b57 100755 --- a/scripts/brand-guard.sh +++ b/scripts/brand-guard.sh @@ -17,6 +17,7 @@ for name in "${FORBIDDEN[@]}"; do HITS=$(grep -ril "$name" . \ --exclude-dir=node_modules --exclude-dir=.git --exclude-dir=dist \ --exclude-dir=.turbo --exclude-dir=.expo --exclude-dir=.data \ + --exclude-dir=.terraform \ --exclude=brand-guard.sh 2>/dev/null || true) if [ -n "$HITS" ]; then echo "FEL: förbjudet namn '$name' hittades i:"