feat(worker/ai): Gemini-lärar-tier för scan/detect/allergen + shadow-capture

- Utöka GeminiAamosClient med ANALYZE_MEAL_IMAGE, READ_NUTRITION_LABEL,
  READ_EXPIRY_DATE och READ_RECEIPT (vision-prompts mot kontraktsscheman).
- Lägg till apps/worker/src/lib/shadow-capture.ts: S3/local fallback,
  imageTraining-gating, ingen PII, aldrig faila användarvägen.
- Integrera capture i scan-processorn efter lyckad runTask med consentFlags.
- Lägg till @aws-sdk/client-s3 i worker samt eval:capture-skript.
- Uppdatera eval:scan till Wikimedia Special:FilePath + redirect-following.
- Exkludera .terraform i brand-guard för att undvika false positives.
- Uppdatera gemini-test för DEDUPLICATE_INVENTORY som unsupported.
This commit is contained in:
Sven (AAMOS AI)
2026-08-12 02:51:05 +07:00
parent 515dd12cc7
commit 9a9f8a6a3c
9 changed files with 615 additions and 20 deletions
+4 -1
View File
@@ -11,7 +11,8 @@
"typecheck": "tsc --noEmit", "typecheck": "tsc --noEmit",
"test": "vitest run --passWithNoTests", "test": "vitest run --passWithNoTests",
"eval": "tsx src/eval/run.ts", "eval": "tsx src/eval/run.ts",
"eval:scan": "tsx src/eval/scan-eval.ts" "eval:scan": "tsx src/eval/scan-eval.ts",
"eval:capture": "tsx src/eval/capture-eval.ts"
}, },
"dependencies": { "dependencies": {
"@app/ai-contracts": "workspace:*", "@app/ai-contracts": "workspace:*",
@@ -25,9 +26,11 @@
"@app/recommendation-engine": "workspace:*", "@app/recommendation-engine": "workspace:*",
"@app/shared-types": "workspace:*", "@app/shared-types": "workspace:*",
"@app/subscriptions": "workspace:*", "@app/subscriptions": "workspace:*",
"@aws-sdk/client-s3": "^3.750.0",
"bullmq": "^6.0.0", "bullmq": "^6.0.0",
"dotenv": "^16.4.0", "dotenv": "^16.4.0",
"drizzle-orm": "^0.45.0", "drizzle-orm": "^0.45.0",
"@aws-sdk/client-s3": "^3.750.0",
"ioredis": "^6.0.0", "ioredis": "^6.0.0",
"pg": "^8.13.0" "pg": "^8.13.0"
}, },
+52
View File
@@ -0,0 +1,52 @@
/**
* Best-effort verifiering av shadow-capture (ingen riktig AI, bara lagring).
*
* S3_MODE=mock pnpm --filter @app/worker eval:capture
*/
import { captureTrainingSample } from "../lib/shadow-capture.js";
async function main() {
const result = await captureTrainingSample({
taskType: "ANALYZE_FRIDGE_IMAGE",
inputS3Keys: ["uploads/test/fridge.jpg"],
output: {
items: [
{
detectedName: "Testmjölk",
canonicalIngredientId: null,
brand: "Testa",
estimatedQuantity: 1,
unit: "LITER",
bestBeforeDate: null,
confidence: 0.97,
requiresConfirmation: false,
boundingBox: null,
},
],
imageQualityIssues: [],
},
modelVersion: "gemini-2.5-flash",
promptVersion: "gemini-fridge-v1",
latencyMs: 1234,
costUsd: 0.0001,
inputTokens: 400,
outputTokens: 120,
consentFlags: {
personalization: true,
anonymizedImprovement: true,
imageTraining: true,
},
readUrl: (key: string) => `http://localhost/v1/mock-s3/${encodeURIComponent(key)}?sig=dummy`,
});
if (!result.ok) {
console.error("[eval:capture] misslyckades:", result.error);
process.exit(1);
}
console.log(`[eval:capture] träningspar sparat på nyckel: ${result.key}`);
}
main().catch((err) => {
console.error(err);
process.exit(1);
});
+2 -2
View File
@@ -40,7 +40,7 @@ interface ScanEvalCase {
const CASES: ScanEvalCase[] = [ const CASES: ScanEvalCase[] = [
{ {
id: "arla-milk-fridge", id: "arla-milk-fridge",
imageUrls: ["https://upload.wikimedia.org/wikipedia/commons/6/6c/Arla_Ko_Mellanmj%C3%B6lk_1L.jpg"], imageUrls: ["https://commons.wikimedia.org/wiki/Special:FilePath/Arla_Ko_Mellanmj%C3%B6lk_1L.jpg?width=800"],
locationType: "fridge", locationType: "fridge",
marketLocale: "sv-SE", marketLocale: "sv-SE",
checks: (items) => { checks: (items) => {
@@ -56,7 +56,7 @@ const CASES: ScanEvalCase[] = [
}, },
{ {
id: "swedish-butter-fridge", id: "swedish-butter-fridge",
imageUrls: ["https://upload.wikimedia.org/wikipedia/commons/3/3e/Svenskt_Sm%C3%B6r_Normalsaltat_80-25_500g.jpg"], imageUrls: ["https://commons.wikimedia.org/wiki/Special:FilePath/Svenskt_Sm%C3%B6r_Normalsaltat_80-25_500g.jpg?width=800"],
locationType: "fridge", locationType: "fridge",
marketLocale: "sv-SE", marketLocale: "sv-SE",
checks: (items) => { checks: (items) => {
+179
View File
@@ -0,0 +1,179 @@
/**
* Shadow-capture: sparar varje lyckad AI-inferens som träningsdata i S3.
*
* - Kör aldrig synkront på användarens kritiska väg; failures swallås.
* - Ingen PII (inga userId/householdId). Endast S3-nycklar, output och metadata.
* - Rå bild kopieras ENDAST när imageTraining-samtycke är true.
* - Durabelt = S3 i prod, lokalt filsystem i mock/test.
*/
import { randomUUID } from "node:crypto";
import { mkdir, writeFile } from "node:fs/promises";
import path from "node:path";
import {
PutObjectCommand,
S3Client,
type S3ClientConfig,
} from "@aws-sdk/client-s3";
import type { AamosTaskType } from "@app/ai-contracts";
export interface CaptureConsentFlags {
personalization: boolean;
anonymizedImprovement: boolean;
imageTraining: boolean;
}
export interface CaptureResult {
ok: boolean;
key?: string;
error?: string;
}
export interface CaptureInput {
taskType: AamosTaskType;
inputS3Keys: string[];
output: Record<string, unknown>;
modelVersion?: string | null;
promptVersion?: string | null;
latencyMs?: number | null;
costUsd?: number | null;
inputTokens?: number | null;
outputTokens?: number | null;
consentFlags: CaptureConsentFlags;
readUrl: (key: string) => string;
/** Override for tests; otherwise built from env. */
storage?: CaptureStorage;
}
export interface CaptureStorage {
put(key: string, data: Buffer, contentType: string): Promise<void>;
}
const TRAINING_PREFIX = "training/cibello";
const MOCK_ROOT = path.resolve(process.cwd(), ".data/s3-training");
class S3CaptureStorage implements CaptureStorage {
private readonly client: S3Client;
private readonly bucket: string;
constructor(bucket: string, config: S3ClientConfig) {
this.bucket = bucket;
this.client = new S3Client(config);
}
async put(key: string, data: Buffer, contentType: string): Promise<void> {
await this.client.send(
new PutObjectCommand({
Bucket: this.bucket,
Key: key,
Body: data,
ContentType: contentType,
}),
);
}
}
class LocalCaptureStorage implements CaptureStorage {
async put(key: string, data: Buffer): Promise<void> {
const filePath = path.join(MOCK_ROOT, key);
await mkdir(path.dirname(filePath), { recursive: true });
await writeFile(filePath, data);
}
}
let cachedStorage: CaptureStorage | undefined;
export function createCaptureStorage(): CaptureStorage {
if (cachedStorage) return cachedStorage;
const mode = process.env.S3_MODE ?? "mock";
if (mode === "aws") {
const bucket = process.env.S3_BUCKET ?? "cibello-production";
const config: S3ClientConfig = {
region: process.env.S3_REGION || "eu-north-1",
...(process.env.S3_ENDPOINT
? { endpoint: process.env.S3_ENDPOINT, forcePathStyle: true }
: {}),
...(process.env.S3_ACCESS_KEY_ID && process.env.S3_SECRET_ACCESS_KEY
? {
credentials: {
accessKeyId: process.env.S3_ACCESS_KEY_ID,
secretAccessKey: process.env.S3_SECRET_ACCESS_KEY,
},
}
: {}),
};
cachedStorage = new S3CaptureStorage(bucket, config);
} else {
cachedStorage = new LocalCaptureStorage();
}
return cachedStorage;
}
export async function captureTrainingSample(input: CaptureInput): Promise<CaptureResult> {
try {
const storage = input.storage ?? createCaptureStorage();
const date = new Date().toISOString().slice(0, 10);
const id = randomUUID();
const prefix = `${TRAINING_PREFIX}/${input.taskType}/${date}/${id}`;
const jsonKey = `${prefix}/sample.json`;
const sample: Record<string, unknown> = {
taskType: input.taskType,
inputReferences: input.inputS3Keys,
output: input.output,
modelVersion: input.modelVersion ?? null,
promptVersion: input.promptVersion ?? null,
latencyMs: input.latencyMs ?? null,
costUsd: input.costUsd ?? null,
inputTokens: input.inputTokens ?? null,
outputTokens: input.outputTokens ?? null,
consentFlags: input.consentFlags,
capturedAt: new Date().toISOString(),
};
const trainingImageKeys: string[] = [];
if (input.consentFlags.imageTraining && input.inputS3Keys.length > 0) {
for (let i = 0; i < input.inputS3Keys.length; i++) {
const originalKey = input.inputS3Keys[i];
if (!originalKey) continue;
try {
const imageKey = `${prefix}/images/${String(i).padStart(3, "0")}.jpg`;
const imageData = await fetchImageData(input.readUrl(originalKey));
if (imageData) {
await storage.put(imageKey, imageData, "image/jpeg");
trainingImageKeys.push(imageKey);
}
} catch (err) {
// Best effort: log but continue; JSON still has original S3 references.
console.error(`[shadow-capture] failed to copy image ${originalKey}:`, err);
}
}
}
if (trainingImageKeys.length > 0) {
sample.trainingImageKeys = trainingImageKeys;
}
await storage.put(jsonKey, Buffer.from(JSON.stringify(sample, null, 2)), "application/json");
return { ok: true, key: jsonKey };
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
console.error("[shadow-capture] failed:", message);
return { ok: false, error: message };
}
}
async function fetchImageData(url: string): Promise<Buffer | null> {
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), 30_000);
try {
const res = await fetch(url, { signal: controller.signal });
clearTimeout(timer);
if (!res.ok) return null;
return Buffer.from(await res.arrayBuffer());
} catch {
return null;
} finally {
clearTimeout(timer);
}
}
+30 -7
View File
@@ -6,6 +6,7 @@ import type { WorkerContext } from "../context.js";
import { getLocaleContext } from "../locale.js"; import { getLocaleContext } from "../locale.js";
import type { LocaleContext } from "@app/shared-types"; import type { LocaleContext } from "@app/shared-types";
import { recordAiUsage } from "../lib/ai-usage.js"; import { recordAiUsage } from "../lib/ai-usage.js";
import { captureTrainingSample, type CaptureConsentFlags } from "../lib/shadow-capture.js";
/** /**
* Bild-/OCR-jobb (spec §54): hämtar scan_job, anropar AAMOS med kontraktvaliderad * Bild-/OCR-jobb (spec §54): hämtar scan_job, anropar AAMOS med kontraktvaliderad
@@ -28,7 +29,7 @@ export async function processScanJob(ctx: WorkerContext, scanJobId: string): Pro
const imageUrls = job.s3Keys.map((k) => ctx.readUrl(k)); const imageUrls = job.s3Keys.map((k) => ctx.readUrl(k));
const localeContext = await getLocaleContext(ctx, job.userId); const localeContext = await getLocaleContext(ctx, job.userId);
const consents = await loadConsentFlags(ctx, job.userId); const consentFlags = await loadConsentFlags(ctx, job.userId);
const started = Date.now(); const started = Date.now();
const result = await runAamosForJob( const result = await runAamosForJob(
@@ -38,6 +39,7 @@ export async function processScanJob(ctx: WorkerContext, scanJobId: string): Pro
imageUrls, imageUrls,
job.context, job.context,
localeContext, localeContext,
consentFlags,
); );
if (result.status === "failed" || result.output == null) { if (result.status === "failed" || result.output == null) {
@@ -80,6 +82,27 @@ export async function processScanJob(ctx: WorkerContext, scanJobId: string): Pro
await recordScanCompleted(ctx, job, result); await recordScanCompleted(ctx, job, result);
// Shadow-capture: spara träningspar för framtida AAMOS-distillation (FAS 1).
// Kör aldrig synkront på användarens kritiska väg; fel swallås.
if (result.status === "ok" && result.output != null) {
const capture = await captureTrainingSample({
taskType: job.jobType as AamosTaskType,
inputS3Keys: job.s3Keys,
output: result.output as Record<string, unknown>,
modelVersion: result.modelVersion,
promptVersion: result.promptVersion,
latencyMs: result.latencyMs,
costUsd: result.costUsd,
inputTokens: result.inputTokens,
outputTokens: result.outputTokens,
consentFlags,
readUrl: ctx.readUrl,
});
if (!capture.ok) {
console.error("[scan processor] shadow-capture misslyckades:", capture.error);
}
}
// Bokför verklig AI-kostnad/tokens utan PII (spec §45). // Bokför verklig AI-kostnad/tokens utan PII (spec §45).
if (result.costUsd != null || result.inputTokens || result.outputTokens) { if (result.costUsd != null || result.inputTokens || result.outputTokens) {
await recordAiUsage(ctx.db, job.userId, { await recordAiUsage(ctx.db, job.userId, {
@@ -107,7 +130,6 @@ export async function processScanJob(ctx: WorkerContext, scanJobId: string): Pro
}, },
}); });
} }
void consents;
} }
async function runAamosForJob( async function runAamosForJob(
@@ -117,6 +139,7 @@ async function runAamosForJob(
imageUrls: string[], imageUrls: string[],
context: unknown, context: unknown,
localeContext: LocaleContext, localeContext: LocaleContext,
consentFlags: CaptureConsentFlags,
) { ) {
switch (jobType) { switch (jobType) {
case "ANALYZE_FRIDGE_IMAGE": case "ANALYZE_FRIDGE_IMAGE":
@@ -129,33 +152,33 @@ async function runAamosForJob(
marketLocale: localeContext.languageTag, marketLocale: localeContext.languageTag,
knownItems: [], knownItems: [],
}, },
{ localeContext }, { localeContext, consentFlags },
); );
case "ANALYZE_MEAL_IMAGE": { case "ANALYZE_MEAL_IMAGE": {
const recipeContext = await buildRecipeContext(ctx, context); const recipeContext = await buildRecipeContext(ctx, context);
return ctx.aamos.runTask( return ctx.aamos.runTask(
"ANALYZE_MEAL_IMAGE", "ANALYZE_MEAL_IMAGE",
{ imageUrls, recipeContext, marketLocale: localeContext.languageTag }, { imageUrls, recipeContext, marketLocale: localeContext.languageTag },
{ localeContext }, { localeContext, consentFlags },
); );
} }
case "READ_RECEIPT": case "READ_RECEIPT":
return ctx.aamos.runTask( return ctx.aamos.runTask(
"READ_RECEIPT", "READ_RECEIPT",
{ imageUrls, marketLocale: localeContext.languageTag }, { imageUrls, marketLocale: localeContext.languageTag },
{ localeContext }, { localeContext, consentFlags },
); );
case "READ_NUTRITION_LABEL": case "READ_NUTRITION_LABEL":
return ctx.aamos.runTask( return ctx.aamos.runTask(
"READ_NUTRITION_LABEL", "READ_NUTRITION_LABEL",
{ imageUrls, marketLocale: localeContext.languageTag }, { imageUrls, marketLocale: localeContext.languageTag },
{ localeContext }, { localeContext, consentFlags },
); );
case "READ_EXPIRY_DATE": case "READ_EXPIRY_DATE":
return ctx.aamos.runTask( return ctx.aamos.runTask(
"READ_EXPIRY_DATE", "READ_EXPIRY_DATE",
{ imageUrls: imageUrls.slice(0, 2) }, { imageUrls: imageUrls.slice(0, 2) },
{ localeContext }, { localeContext, consentFlags },
); );
default: default:
throw new Error(`Jobbtypen ${jobType} hanteras inte av scan-processorn`); throw new Error(`Jobbtypen ${jobType} hanteras inte av scan-processorn`);
+341 -7
View File
@@ -1,5 +1,5 @@
/** /**
* Gemini-lärar-tier för AAMOS-task-typer (SKIVA 1: kylskåpsskanning). * Gemini-lärar-tier för AAMOS-task-typer (SKIVA 1).
* *
* - Implementerar samma AamosClient-interface som HttpAamosClient/MockAamosClient. * - Implementerar samma AamosClient-interface som HttpAamosClient/MockAamosClient.
* - App + Food API ser ingen skillnad; workern byter bara adapter. * - App + Food API ser ingen skillnad; workern byter bara adapter.
@@ -7,7 +7,6 @@
* - Kostnad/tokens rapporteras tillbaka så workern kan bokföra i ai_usage_counters. * - Kostnad/tokens rapporteras tillbaka så workern kan bokföra i ai_usage_counters.
* - Global dagsbudget kan sättas via GEMINI_DAILY_BUDGET_USD. * - Global dagsbudget kan sättas via GEMINI_DAILY_BUDGET_USD.
*/ */
import { randomUUID } from "node:crypto";
import { z } from "zod"; import { z } from "zod";
import { import {
TASK_CONTRACTS, TASK_CONTRACTS,
@@ -17,14 +16,22 @@ import {
detectedItemSchema, detectedItemSchema,
type DetectedItem, type DetectedItem,
generateRecipeCandidatesOutput, generateRecipeCandidatesOutput,
analyzeMealImageOutput,
readNutritionLabelOutput,
readExpiryDateOutput,
readReceiptOutput,
} from "./tasks.js"; } from "./tasks.js";
import type { AamosCallOptions, AamosClient, AamosResult } from "./client.js"; import type { AamosCallOptions, AamosClient, AamosResult } from "./client.js";
import type { LocaleContext } from "@app/shared-types"; import type { LocaleContext } from "@app/shared-types";
const GEMINI_API_BASE = "https://generativelanguage.googleapis.com/v1beta"; const GEMINI_API_BASE = "https://generativelanguage.googleapis.com/v1beta";
/** Custom fetch using Node https module — works around IPv6 hangs on WSL. */ /** Custom fetch using Node https module — works around IPv6 hangs on WSL. Follows redirects. */
function customNodeFetch(input: string | URL | Request, init?: RequestInit): Promise<Response> { function customNodeFetch(input: string | URL | Request, init?: RequestInit): Promise<Response> {
return requestOnce(input, init, 5);
}
function requestOnce(input: string | URL | Request, init: RequestInit | undefined, redirectsLeft: number): Promise<Response> {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
const url = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url; const url = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url;
const u = new URL(url); const u = new URL(url);
@@ -45,6 +52,9 @@ function customNodeFetch(input: string | URL | Request, init?: RequestInit): Pro
if (postData && !headers["content-length"]) { if (postData && !headers["content-length"]) {
headers["content-length"] = String(Buffer.byteLength(postData)); headers["content-length"] = String(Buffer.byteLength(postData));
} }
if (!headers["user-agent"]) {
headers["user-agent"] = "Mozilla/5.0 (compatible; Cibello/1.0)";
}
const req = httpMod.request( const req = httpMod.request(
{ {
@@ -56,13 +66,23 @@ function customNodeFetch(input: string | URL | Request, init?: RequestInit): Pro
timeout: 120_000, timeout: 120_000,
}, },
(res) => { (res) => {
const status = res.statusCode ?? 200;
const location = res.headers.location;
if (status >= 300 && status < 400 && location && redirectsLeft > 0) {
const nextUrl = new URL(location, url).toString();
requestOnce(nextUrl, { ...init, method: "GET", body: undefined }, redirectsLeft - 1)
.then(resolve)
.catch(reject);
return;
}
let body = ""; let body = "";
res.setEncoding("utf8"); res.setEncoding("utf8");
res.on("data", (chunk) => { body += chunk; }); res.on("data", (chunk) => { body += chunk; });
res.on("end", () => { res.on("end", () => {
resolve( resolve(
new Response(body, { new Response(body, {
status: res.statusCode ?? 200, status,
statusText: res.statusMessage ?? "OK", statusText: res.statusMessage ?? "OK",
headers: new Headers(Object.entries(res.headers).map(([k, v]) => [k, String(v)])), headers: new Headers(Object.entries(res.headers).map(([k, v]) => [k, String(v)])),
}), }),
@@ -81,6 +101,9 @@ function customNodeFetch(input: string | URL | Request, init?: RequestInit): Pro
/** Estimated max cost per image call in USD (pessimistic). */ /** Estimated max cost per image call in USD (pessimistic). */
const COST_ESTIMATE_PER_IMAGE_USD = 0.0015; const COST_ESTIMATE_PER_IMAGE_USD = 0.0015;
/** Estimated max cost per simple OCR image call in USD. */
const COST_ESTIMATE_PER_OCR_IMAGE_USD = 0.001;
export interface BudgetStore { export interface BudgetStore {
/** Return current daily spend in USD. */ /** Return current daily spend in USD. */
getDailySpendUsd(): Promise<number>; getDailySpendUsd(): Promise<number>;
@@ -222,7 +245,7 @@ export class GeminiAamosClient implements AamosClient {
async healthCheck(): Promise<{ ok: boolean; detail?: string }> { async healthCheck(): Promise<{ ok: boolean; detail?: string }> {
try { try {
const url = `${GEMINI_API_BASE}/models?key=${this.cfg.apiKey}`; const url = `${GEMINI_API_BASE}/models?key=${encodeURIComponent(this.cfg.apiKey)}`;
const res = await this.cfg.fetchImpl(url, { signal: AbortSignal.timeout(5_000) }); const res = await this.cfg.fetchImpl(url, { signal: AbortSignal.timeout(5_000) });
return { ok: res.ok, detail: `status ${res.status}` }; return { ok: res.ok, detail: `status ${res.status}` };
} catch (err) { } catch (err) {
@@ -251,6 +274,14 @@ export class GeminiAamosClient implements AamosClient {
return this.analyzeStorageImage(taskType, parsedInput.data as TaskInput<"ANALYZE_FRIDGE_IMAGE">, options) as Promise<AamosResult<T>>; return this.analyzeStorageImage(taskType, parsedInput.data as TaskInput<"ANALYZE_FRIDGE_IMAGE">, options) as Promise<AamosResult<T>>;
case "GENERATE_RECIPE_CANDIDATES": case "GENERATE_RECIPE_CANDIDATES":
return this.generateRecipeCandidates(parsedInput.data as TaskInput<"GENERATE_RECIPE_CANDIDATES">, options) as Promise<AamosResult<T>>; return this.generateRecipeCandidates(parsedInput.data as TaskInput<"GENERATE_RECIPE_CANDIDATES">, options) as Promise<AamosResult<T>>;
case "ANALYZE_MEAL_IMAGE":
return this.analyzeMealImage(parsedInput.data as TaskInput<"ANALYZE_MEAL_IMAGE">, options) as Promise<AamosResult<T>>;
case "READ_NUTRITION_LABEL":
return this.readNutritionLabel(parsedInput.data as TaskInput<"READ_NUTRITION_LABEL">, options) as Promise<AamosResult<T>>;
case "READ_EXPIRY_DATE":
return this.readExpiryDate(parsedInput.data as TaskInput<"READ_EXPIRY_DATE">, options) as Promise<AamosResult<T>>;
case "READ_RECEIPT":
return this.readReceipt(parsedInput.data as TaskInput<"READ_RECEIPT">, options) as Promise<AamosResult<T>>;
default: default:
return { return {
status: "failed", status: "failed",
@@ -295,7 +326,7 @@ export class GeminiAamosClient implements AamosClient {
}, },
}; };
const url = `${GEMINI_API_BASE}/models/${this.cfg.model}:generateContent?key=${this.cfg.apiKey}`; const url = `${GEMINI_API_BASE}/models/${this.cfg.model}:generateContent?key=${encodeURIComponent(this.cfg.apiKey)}`;
const res = await this.cfg.fetchImpl(url, { const res = await this.cfg.fetchImpl(url, {
method: "POST", method: "POST",
headers: { "content-type": "application/json" }, headers: { "content-type": "application/json" },
@@ -350,6 +381,186 @@ export class GeminiAamosClient implements AamosClient {
}; };
} }
private async runVisionTask<T extends AamosTaskType>(
taskType: T,
imageUrls: string[],
prompt: string,
outputSchema: z.ZodSchema<unknown>,
mapOutput: (parsed: unknown) => TaskOutput<T>,
options: AamosCallOptions,
promptVersion: string,
costEstimatePerImage: number,
): Promise<AamosResult<T>> {
const locale = options.localeContext ?? this.defaultLocale();
const started = Date.now();
const imageParts = await this.fetchImageParts(imageUrls, this.cfg.timeoutMs);
const estimatedCostUsd = imageParts.length * costEstimatePerImage;
if (await this.isOverBudget(estimatedCostUsd)) {
return {
status: "failed",
output: null,
error: "Global Gemini-dagsbudget är förbrukad.",
} as AamosResult<T>;
}
const payload = {
contents: [{ parts: [{ text: prompt }, ...imageParts] }],
generationConfig: {
responseMimeType: "application/json",
temperature: 0.2,
},
};
const url = `${GEMINI_API_BASE}/models/${this.cfg.model}:generateContent?key=${encodeURIComponent(this.cfg.apiKey)}`;
const res = await this.cfg.fetchImpl(url, {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify(payload),
signal: AbortSignal.timeout(this.cfg.timeoutMs),
});
if (!res.ok) {
const body = await res.text();
return {
status: "failed",
output: null,
error: `Gemini ${res.status}: ${body.slice(0, 500)}`,
} as AamosResult<T>;
}
const geminiBody = (await res.json()) as {
candidates?: Array<{ content?: { parts?: Array<{ text?: string }> } }>;
usageMetadata?: { promptTokenCount?: number; candidatesTokenCount?: number };
};
const text = geminiBody.candidates?.[0]?.content?.parts?.[0]?.text ?? "";
let parsed: unknown;
try {
const json = JSON.parse(text);
const safe = outputSchema.safeParse(json);
if (!safe.success) {
return {
status: "failed",
output: null,
error: `Gemini-svar matchar inte schema: ${safe.error.message}`,
} as AamosResult<T>;
}
parsed = safe.data;
} catch {
return {
status: "failed",
output: null,
error: "Gemini-svar var inte giltig JSON.",
} as AamosResult<T>;
}
const inputTokens = geminiBody.usageMetadata?.promptTokenCount ?? 0;
const outputTokens = geminiBody.usageMetadata?.candidatesTokenCount ?? 0;
const costUsd = this.estimateCostUsd(inputTokens, outputTokens);
await this.recordSpend(costUsd);
const latencyMs = Date.now() - started;
const output = mapOutput(parsed) as TaskOutput<T>;
return {
status: "ok",
output,
modelVersion: this.cfg.model,
promptVersion,
latencyMs,
costUsd,
inputTokens,
outputTokens,
};
}
private async analyzeMealImage(
input: TaskInput<"ANALYZE_MEAL_IMAGE">,
options: AamosCallOptions,
): Promise<AamosResult<"ANALYZE_MEAL_IMAGE">> {
const locale = options.localeContext ?? this.defaultLocale();
const prompt = this.buildMealPrompt(locale, input.recipeContext);
return this.runVisionTask(
"ANALYZE_MEAL_IMAGE",
input.imageUrls,
prompt,
analyzeMealImageOutput,
(parsed) => parsed as TaskOutput<"ANALYZE_MEAL_IMAGE">,
options,
"gemini-meal-v1",
COST_ESTIMATE_PER_IMAGE_USD,
);
}
private async readNutritionLabel(
input: TaskInput<"READ_NUTRITION_LABEL">,
options: AamosCallOptions,
): Promise<AamosResult<"READ_NUTRITION_LABEL">> {
const locale = options.localeContext ?? this.defaultLocale();
const prompt = this.buildNutritionPrompt(locale);
return this.runVisionTask(
"READ_NUTRITION_LABEL",
input.imageUrls,
prompt,
readNutritionLabelOutput,
(parsed) => parsed as TaskOutput<"READ_NUTRITION_LABEL">,
options,
"gemini-nutrition-v1",
COST_ESTIMATE_PER_OCR_IMAGE_USD,
);
}
private async readExpiryDate(
input: TaskInput<"READ_EXPIRY_DATE">,
options: AamosCallOptions,
): Promise<AamosResult<"READ_EXPIRY_DATE">> {
const locale = options.localeContext ?? this.defaultLocale();
const prompt = this.buildExpiryPrompt(locale);
return this.runVisionTask(
"READ_EXPIRY_DATE",
input.imageUrls,
prompt,
readExpiryDateOutput,
(parsed) => parsed as TaskOutput<"READ_EXPIRY_DATE">,
options,
"gemini-expiry-v1",
COST_ESTIMATE_PER_OCR_IMAGE_USD,
);
}
private async readReceipt(
input: TaskInput<"READ_RECEIPT">,
options: AamosCallOptions,
): Promise<AamosResult<"READ_RECEIPT">> {
const locale = options.localeContext ?? this.defaultLocale();
const prompt = this.buildReceiptPrompt(locale);
return this.runVisionTask(
"READ_RECEIPT",
input.imageUrls,
prompt,
readReceiptOutput,
(parsed) => parsed as TaskOutput<"READ_RECEIPT">,
options,
"gemini-receipt-v1",
COST_ESTIMATE_PER_IMAGE_USD,
);
}
private async fetchImageParts(
imageUrls: string[],
timeoutMs: number,
): Promise<Array<{ inline_data: { mime_type: string; data: string } }>> {
return Promise.all(
imageUrls.slice(0, 6).map(async (url) => {
const res = await this.cfg.fetchImpl(url, { signal: AbortSignal.timeout(timeoutMs) });
if (!res.ok) throw new Error(`Kunde inte hämta bild: ${res.status} ${url}`);
const buf = Buffer.from(await res.arrayBuffer());
return { inline_data: { mime_type: this.mimeType(buf), data: buf.toString("base64") } };
}),
);
}
private async generateRecipeCandidates( private async generateRecipeCandidates(
input: TaskInput<"GENERATE_RECIPE_CANDIDATES">, input: TaskInput<"GENERATE_RECIPE_CANDIDATES">,
options: AamosCallOptions, options: AamosCallOptions,
@@ -377,7 +588,7 @@ export class GeminiAamosClient implements AamosClient {
}, },
}; };
const url = `${GEMINI_API_BASE}/models/${this.cfg.model}:generateContent?key=${this.cfg.apiKey}`; const url = `${GEMINI_API_BASE}/models/${this.cfg.model}:generateContent?key=${encodeURIComponent(this.cfg.apiKey)}`;
const res = await this.cfg.fetchImpl(url, { const res = await this.cfg.fetchImpl(url, {
method: "POST", method: "POST",
headers: { "content-type": "application/json" }, headers: { "content-type": "application/json" },
@@ -553,6 +764,129 @@ Svara ENDAST med giltig JSON i exakt detta format:
} }
imageQualityIssues kan innehålla någon av: "dark", "blurry", "occlusion", "too_far". imageQualityIssues kan innehålla någon av: "dark", "blurry", "occlusion", "too_far".
Språk: ${lang}.`;
}
private buildMealPrompt(locale: LocaleContext, recipeContext: unknown): string {
const lang = locale.languageTag.startsWith("en") ? "English" : "Swedish";
const ctx = recipeContext as {
recipeId?: string;
titleSv?: string;
nutritionPerPortion?: Record<string, number>;
portions?: number;
} | null;
let contextLine = "";
if (ctx?.titleSv) {
contextLine = `Användaren säger att måltiden ska matcha receptet "${ctx.titleSv}" (${ctx.portions ?? "?"} portioner). `;
}
return `Du är en svensk näringsassistent. ${contextLine}Bilden visar en serverad måltid.
Uppskatta:
- matchesRecipeContext: true/false/null matchar bilden receptet användaren nämnde?
- portionFractionEstimate: hur stor del av en normalportion ser det ut att vara? 0.05.0, eller null.
- kcalRange: { min, max, mostLikely } för hela måltiden, eller null.
- components: lista med uppskattade komponenter (namn, ev. canonicalIngredientId som null, estimatedGrams, confidence).
- confidence: 0.01.0 för hela tolkningen.
Svara ENDAST med giltig JSON som matchar detta schema:
{
"matchesRecipeContext": boolean | null,
"portionFractionEstimate": number | null,
"kcalRange": { "min": number, "max": number, "mostLikely": number } | null,
"components": [
{ "name": string, "canonicalIngredientId": string | null, "estimatedGrams": number | null, "confidence": number }
],
"confidence": number
}
Språk: ${lang}.`;
}
private buildNutritionPrompt(locale: LocaleContext): string {
const lang = locale.languageTag.startsWith("en") ? "English" : "Swedish";
return `Du är en OCR-assistent för svenska livsmedelsförpackningar. Bilden visar en näringsdeklaration/ingredienslista.
Läs av exakt det som står på etiketten. Gissa aldrig värden som inte syns. Returnera:
- basis: "per_100_g", "per_100_ml" eller "per_portion" vad gäller tabellen?
- values: { kcal, proteinG, carbsG, fatG, saturatedFatG, fiberG, sugarG, saltG } null om ej synligt.
- ingredientsText: rå text med ingredienser, eller null.
- allergensDeclared: lista med allergener som explicit nämns (t.ex. ["gluten", "mjölk", "nötter"]).
- gtin: streckkod/EAN om synlig, annars null.
- productName: produktnamn om synligt, annars null.
- brand: varumärke om synligt, annars null.
- confidence: 0.01.0.
Svara ENDAST med giltig JSON som matchar detta schema:
{
"basis": "per_100_g" | "per_100_ml" | "per_portion" | null,
"values": { "kcal": number|null, "proteinG": number|null, "carbsG": number|null, "fatG": number|null, "saturatedFatG": number|null, "fiberG": number|null, "sugarG": number|null, "saltG": number|null } | null,
"ingredientsText": string | null,
"allergensDeclared": string[],
"gtin": string | null,
"productName": string | null,
"brand": string | null,
"confidence": number
}
Språk: ${lang}.`;
}
private buildExpiryPrompt(locale: LocaleContext): string {
const lang = locale.languageTag.startsWith("en") ? "English" : "Swedish";
return `Du är en OCR-assistent för svenska livsmedelsförpackningar. Bilden visar ett bäst-före-datum eller sista förbrukningsdatum.
Returnera:
- date: datum i format YYYY-MM-DD, eller null om ej läsbart.
- dateKind: "best_before" för "bäst före", "use_by" för "sista förbrukningsdag", annars null.
- confidence: 0.01.0.
Svara ENDAST med giltig JSON:
{
"date": "YYYY-MM-DD" | null,
"dateKind": "best_before" | "use_by" | null,
"confidence": number
}
Språk: ${lang}.`;
}
private buildReceiptPrompt(locale: LocaleContext): string {
const lang = locale.languageTag.startsWith("en") ? "English" : "Swedish";
return `Du är en OCR-assistent för svenska kvitton. Bilden visar ett butikskvitto.
Läs av:
- storeName: butikens namn, eller null.
- purchaseDate: datum i format YYYY-MM-DD, eller null.
- lines: varje rad med { rawText (exakt text), normalizedName (förenklat livsmedelsnamn eller null), canonicalIngredientId (alltid null), quantity (antal/enhet om går att utläsa, annars null), unit (enhetskod eller null), unitPriceMinor (ören per styck om synligt, annars null), totalPriceMinor (radens totalpris i ören om synligt, annars null), isDiscount (true om det är en rabatt/avdrag-rad), confidence (0.01.0) }.
- totalMinor: kvittots totalsumma i ören, eller null.
- discountTotalMinor: total rabatt i ören, eller null.
- confidence: 0.01.0 för hela tolkningen.
Svara ENDAST med giltig JSON som matchar detta schema:
{
"storeName": string | null,
"purchaseDate": "YYYY-MM-DD" | null,
"lines": [
{
"rawText": string,
"normalizedName": string | null,
"canonicalIngredientId": null,
"quantity": number | null,
"unit": string | null,
"unitPriceMinor": number | null,
"totalPriceMinor": number | null,
"isDiscount": boolean,
"confidence": number
}
],
"totalMinor": number | null,
"discountTotalMinor": number | null,
"confidence": number
}
Använd endast de enhetskoder du kan utläsa (GRAM, KILOGRAM, LITER, etc.), annars null. quantity är alltid ett decimaltal.
Språk: ${lang}.`; Språk: ${lang}.`;
} }
+3 -3
View File
@@ -127,12 +127,12 @@ describe("GeminiAamosClient", () => {
}); });
const result = await client.runTask( const result = await client.runTask(
"READ_RECEIPT", "DEDUPLICATE_INVENTORY",
{ imageUrls: ["http://localhost/v1/mock-s3/receipt.jpg"], marketLocale: "sv-SE" }, { items: [] },
); );
expect(result.status).toBe("failed"); expect(result.status).toBe("failed");
expect(result.error).toContain("READ_RECEIPT"); expect(result.error).toContain("DEDUPLICATE_INVENTORY");
}); });
it("validates input before calling Gemini", async () => { it("validates input before calling Gemini", async () => {
+3
View File
@@ -296,6 +296,9 @@ importers:
'@app/subscriptions': '@app/subscriptions':
specifier: workspace:* specifier: workspace:*
version: link:../../packages/subscriptions version: link:../../packages/subscriptions
'@aws-sdk/client-s3':
specifier: ^3.750.0
version: 3.1102.0
bullmq: bullmq:
specifier: ^6.0.0 specifier: ^6.0.0
version: 6.0.5(ioredis@6.0.0)(pg@8.22.0) version: 6.0.5(ioredis@6.0.0)(pg@8.22.0)
+1
View File
@@ -17,6 +17,7 @@ for name in "${FORBIDDEN[@]}"; do
HITS=$(grep -ril "$name" . \ HITS=$(grep -ril "$name" . \
--exclude-dir=node_modules --exclude-dir=.git --exclude-dir=dist \ --exclude-dir=node_modules --exclude-dir=.git --exclude-dir=dist \
--exclude-dir=.turbo --exclude-dir=.expo --exclude-dir=.data \ --exclude-dir=.turbo --exclude-dir=.expo --exclude-dir=.data \
--exclude-dir=.terraform \
--exclude=brand-guard.sh 2>/dev/null || true) --exclude=brand-guard.sh 2>/dev/null || true)
if [ -n "$HITS" ]; then if [ -n "$HITS" ]; then
echo "FEL: förbjudet namn '$name' hittades i:" echo "FEL: förbjudet namn '$name' hittades i:"