feat(worker/ai): Gemini-lärar-tier för scan/detect/allergen + shadow-capture
- Utöka GeminiAamosClient med ANALYZE_MEAL_IMAGE, READ_NUTRITION_LABEL, READ_EXPIRY_DATE och READ_RECEIPT (vision-prompts mot kontraktsscheman). - Lägg till apps/worker/src/lib/shadow-capture.ts: S3/local fallback, imageTraining-gating, ingen PII, aldrig faila användarvägen. - Integrera capture i scan-processorn efter lyckad runTask med consentFlags. - Lägg till @aws-sdk/client-s3 i worker samt eval:capture-skript. - Uppdatera eval:scan till Wikimedia Special:FilePath + redirect-following. - Exkludera .terraform i brand-guard för att undvika false positives. - Uppdatera gemini-test för DEDUPLICATE_INVENTORY som unsupported.
This commit is contained in:
@@ -11,7 +11,8 @@
|
|||||||
"typecheck": "tsc --noEmit",
|
"typecheck": "tsc --noEmit",
|
||||||
"test": "vitest run --passWithNoTests",
|
"test": "vitest run --passWithNoTests",
|
||||||
"eval": "tsx src/eval/run.ts",
|
"eval": "tsx src/eval/run.ts",
|
||||||
"eval:scan": "tsx src/eval/scan-eval.ts"
|
"eval:scan": "tsx src/eval/scan-eval.ts",
|
||||||
|
"eval:capture": "tsx src/eval/capture-eval.ts"
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@app/ai-contracts": "workspace:*",
|
"@app/ai-contracts": "workspace:*",
|
||||||
@@ -25,9 +26,11 @@
|
|||||||
"@app/recommendation-engine": "workspace:*",
|
"@app/recommendation-engine": "workspace:*",
|
||||||
"@app/shared-types": "workspace:*",
|
"@app/shared-types": "workspace:*",
|
||||||
"@app/subscriptions": "workspace:*",
|
"@app/subscriptions": "workspace:*",
|
||||||
|
"@aws-sdk/client-s3": "^3.750.0",
|
||||||
"bullmq": "^6.0.0",
|
"bullmq": "^6.0.0",
|
||||||
"dotenv": "^16.4.0",
|
"dotenv": "^16.4.0",
|
||||||
"drizzle-orm": "^0.45.0",
|
"drizzle-orm": "^0.45.0",
|
||||||
|
"@aws-sdk/client-s3": "^3.750.0",
|
||||||
"ioredis": "^6.0.0",
|
"ioredis": "^6.0.0",
|
||||||
"pg": "^8.13.0"
|
"pg": "^8.13.0"
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -0,0 +1,52 @@
|
|||||||
|
/**
|
||||||
|
* Best-effort verifiering av shadow-capture (ingen riktig AI, bara lagring).
|
||||||
|
*
|
||||||
|
* S3_MODE=mock pnpm --filter @app/worker eval:capture
|
||||||
|
*/
|
||||||
|
import { captureTrainingSample } from "../lib/shadow-capture.js";
|
||||||
|
|
||||||
|
async function main() {
|
||||||
|
const result = await captureTrainingSample({
|
||||||
|
taskType: "ANALYZE_FRIDGE_IMAGE",
|
||||||
|
inputS3Keys: ["uploads/test/fridge.jpg"],
|
||||||
|
output: {
|
||||||
|
items: [
|
||||||
|
{
|
||||||
|
detectedName: "Testmjölk",
|
||||||
|
canonicalIngredientId: null,
|
||||||
|
brand: "Testa",
|
||||||
|
estimatedQuantity: 1,
|
||||||
|
unit: "LITER",
|
||||||
|
bestBeforeDate: null,
|
||||||
|
confidence: 0.97,
|
||||||
|
requiresConfirmation: false,
|
||||||
|
boundingBox: null,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
imageQualityIssues: [],
|
||||||
|
},
|
||||||
|
modelVersion: "gemini-2.5-flash",
|
||||||
|
promptVersion: "gemini-fridge-v1",
|
||||||
|
latencyMs: 1234,
|
||||||
|
costUsd: 0.0001,
|
||||||
|
inputTokens: 400,
|
||||||
|
outputTokens: 120,
|
||||||
|
consentFlags: {
|
||||||
|
personalization: true,
|
||||||
|
anonymizedImprovement: true,
|
||||||
|
imageTraining: true,
|
||||||
|
},
|
||||||
|
readUrl: (key: string) => `http://localhost/v1/mock-s3/${encodeURIComponent(key)}?sig=dummy`,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!result.ok) {
|
||||||
|
console.error("[eval:capture] misslyckades:", result.error);
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
console.log(`[eval:capture] träningspar sparat på nyckel: ${result.key}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
main().catch((err) => {
|
||||||
|
console.error(err);
|
||||||
|
process.exit(1);
|
||||||
|
});
|
||||||
@@ -40,7 +40,7 @@ interface ScanEvalCase {
|
|||||||
const CASES: ScanEvalCase[] = [
|
const CASES: ScanEvalCase[] = [
|
||||||
{
|
{
|
||||||
id: "arla-milk-fridge",
|
id: "arla-milk-fridge",
|
||||||
imageUrls: ["https://upload.wikimedia.org/wikipedia/commons/6/6c/Arla_Ko_Mellanmj%C3%B6lk_1L.jpg"],
|
imageUrls: ["https://commons.wikimedia.org/wiki/Special:FilePath/Arla_Ko_Mellanmj%C3%B6lk_1L.jpg?width=800"],
|
||||||
locationType: "fridge",
|
locationType: "fridge",
|
||||||
marketLocale: "sv-SE",
|
marketLocale: "sv-SE",
|
||||||
checks: (items) => {
|
checks: (items) => {
|
||||||
@@ -56,7 +56,7 @@ const CASES: ScanEvalCase[] = [
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
id: "swedish-butter-fridge",
|
id: "swedish-butter-fridge",
|
||||||
imageUrls: ["https://upload.wikimedia.org/wikipedia/commons/3/3e/Svenskt_Sm%C3%B6r_Normalsaltat_80-25_500g.jpg"],
|
imageUrls: ["https://commons.wikimedia.org/wiki/Special:FilePath/Svenskt_Sm%C3%B6r_Normalsaltat_80-25_500g.jpg?width=800"],
|
||||||
locationType: "fridge",
|
locationType: "fridge",
|
||||||
marketLocale: "sv-SE",
|
marketLocale: "sv-SE",
|
||||||
checks: (items) => {
|
checks: (items) => {
|
||||||
|
|||||||
@@ -0,0 +1,179 @@
|
|||||||
|
/**
|
||||||
|
* Shadow-capture: sparar varje lyckad AI-inferens som träningsdata i S3.
|
||||||
|
*
|
||||||
|
* - Kör aldrig synkront på användarens kritiska väg; failures swallås.
|
||||||
|
* - Ingen PII (inga userId/householdId). Endast S3-nycklar, output och metadata.
|
||||||
|
* - Rå bild kopieras ENDAST när imageTraining-samtycke är true.
|
||||||
|
* - Durabelt = S3 i prod, lokalt filsystem i mock/test.
|
||||||
|
*/
|
||||||
|
import { randomUUID } from "node:crypto";
|
||||||
|
import { mkdir, writeFile } from "node:fs/promises";
|
||||||
|
import path from "node:path";
|
||||||
|
import {
|
||||||
|
PutObjectCommand,
|
||||||
|
S3Client,
|
||||||
|
type S3ClientConfig,
|
||||||
|
} from "@aws-sdk/client-s3";
|
||||||
|
import type { AamosTaskType } from "@app/ai-contracts";
|
||||||
|
|
||||||
|
export interface CaptureConsentFlags {
|
||||||
|
personalization: boolean;
|
||||||
|
anonymizedImprovement: boolean;
|
||||||
|
imageTraining: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface CaptureResult {
|
||||||
|
ok: boolean;
|
||||||
|
key?: string;
|
||||||
|
error?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface CaptureInput {
|
||||||
|
taskType: AamosTaskType;
|
||||||
|
inputS3Keys: string[];
|
||||||
|
output: Record<string, unknown>;
|
||||||
|
modelVersion?: string | null;
|
||||||
|
promptVersion?: string | null;
|
||||||
|
latencyMs?: number | null;
|
||||||
|
costUsd?: number | null;
|
||||||
|
inputTokens?: number | null;
|
||||||
|
outputTokens?: number | null;
|
||||||
|
consentFlags: CaptureConsentFlags;
|
||||||
|
readUrl: (key: string) => string;
|
||||||
|
/** Override for tests; otherwise built from env. */
|
||||||
|
storage?: CaptureStorage;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface CaptureStorage {
|
||||||
|
put(key: string, data: Buffer, contentType: string): Promise<void>;
|
||||||
|
}
|
||||||
|
|
||||||
|
const TRAINING_PREFIX = "training/cibello";
|
||||||
|
const MOCK_ROOT = path.resolve(process.cwd(), ".data/s3-training");
|
||||||
|
|
||||||
|
class S3CaptureStorage implements CaptureStorage {
|
||||||
|
private readonly client: S3Client;
|
||||||
|
private readonly bucket: string;
|
||||||
|
|
||||||
|
constructor(bucket: string, config: S3ClientConfig) {
|
||||||
|
this.bucket = bucket;
|
||||||
|
this.client = new S3Client(config);
|
||||||
|
}
|
||||||
|
|
||||||
|
async put(key: string, data: Buffer, contentType: string): Promise<void> {
|
||||||
|
await this.client.send(
|
||||||
|
new PutObjectCommand({
|
||||||
|
Bucket: this.bucket,
|
||||||
|
Key: key,
|
||||||
|
Body: data,
|
||||||
|
ContentType: contentType,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
class LocalCaptureStorage implements CaptureStorage {
|
||||||
|
async put(key: string, data: Buffer): Promise<void> {
|
||||||
|
const filePath = path.join(MOCK_ROOT, key);
|
||||||
|
await mkdir(path.dirname(filePath), { recursive: true });
|
||||||
|
await writeFile(filePath, data);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let cachedStorage: CaptureStorage | undefined;
|
||||||
|
|
||||||
|
export function createCaptureStorage(): CaptureStorage {
|
||||||
|
if (cachedStorage) return cachedStorage;
|
||||||
|
|
||||||
|
const mode = process.env.S3_MODE ?? "mock";
|
||||||
|
if (mode === "aws") {
|
||||||
|
const bucket = process.env.S3_BUCKET ?? "cibello-production";
|
||||||
|
const config: S3ClientConfig = {
|
||||||
|
region: process.env.S3_REGION || "eu-north-1",
|
||||||
|
...(process.env.S3_ENDPOINT
|
||||||
|
? { endpoint: process.env.S3_ENDPOINT, forcePathStyle: true }
|
||||||
|
: {}),
|
||||||
|
...(process.env.S3_ACCESS_KEY_ID && process.env.S3_SECRET_ACCESS_KEY
|
||||||
|
? {
|
||||||
|
credentials: {
|
||||||
|
accessKeyId: process.env.S3_ACCESS_KEY_ID,
|
||||||
|
secretAccessKey: process.env.S3_SECRET_ACCESS_KEY,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
: {}),
|
||||||
|
};
|
||||||
|
cachedStorage = new S3CaptureStorage(bucket, config);
|
||||||
|
} else {
|
||||||
|
cachedStorage = new LocalCaptureStorage();
|
||||||
|
}
|
||||||
|
return cachedStorage;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function captureTrainingSample(input: CaptureInput): Promise<CaptureResult> {
|
||||||
|
try {
|
||||||
|
const storage = input.storage ?? createCaptureStorage();
|
||||||
|
const date = new Date().toISOString().slice(0, 10);
|
||||||
|
const id = randomUUID();
|
||||||
|
const prefix = `${TRAINING_PREFIX}/${input.taskType}/${date}/${id}`;
|
||||||
|
const jsonKey = `${prefix}/sample.json`;
|
||||||
|
|
||||||
|
const sample: Record<string, unknown> = {
|
||||||
|
taskType: input.taskType,
|
||||||
|
inputReferences: input.inputS3Keys,
|
||||||
|
output: input.output,
|
||||||
|
modelVersion: input.modelVersion ?? null,
|
||||||
|
promptVersion: input.promptVersion ?? null,
|
||||||
|
latencyMs: input.latencyMs ?? null,
|
||||||
|
costUsd: input.costUsd ?? null,
|
||||||
|
inputTokens: input.inputTokens ?? null,
|
||||||
|
outputTokens: input.outputTokens ?? null,
|
||||||
|
consentFlags: input.consentFlags,
|
||||||
|
capturedAt: new Date().toISOString(),
|
||||||
|
};
|
||||||
|
|
||||||
|
const trainingImageKeys: string[] = [];
|
||||||
|
if (input.consentFlags.imageTraining && input.inputS3Keys.length > 0) {
|
||||||
|
for (let i = 0; i < input.inputS3Keys.length; i++) {
|
||||||
|
const originalKey = input.inputS3Keys[i];
|
||||||
|
if (!originalKey) continue;
|
||||||
|
try {
|
||||||
|
const imageKey = `${prefix}/images/${String(i).padStart(3, "0")}.jpg`;
|
||||||
|
const imageData = await fetchImageData(input.readUrl(originalKey));
|
||||||
|
if (imageData) {
|
||||||
|
await storage.put(imageKey, imageData, "image/jpeg");
|
||||||
|
trainingImageKeys.push(imageKey);
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
// Best effort: log but continue; JSON still has original S3 references.
|
||||||
|
console.error(`[shadow-capture] failed to copy image ${originalKey}:`, err);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (trainingImageKeys.length > 0) {
|
||||||
|
sample.trainingImageKeys = trainingImageKeys;
|
||||||
|
}
|
||||||
|
|
||||||
|
await storage.put(jsonKey, Buffer.from(JSON.stringify(sample, null, 2)), "application/json");
|
||||||
|
return { ok: true, key: jsonKey };
|
||||||
|
} catch (err) {
|
||||||
|
const message = err instanceof Error ? err.message : String(err);
|
||||||
|
console.error("[shadow-capture] failed:", message);
|
||||||
|
return { ok: false, error: message };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function fetchImageData(url: string): Promise<Buffer | null> {
|
||||||
|
const controller = new AbortController();
|
||||||
|
const timer = setTimeout(() => controller.abort(), 30_000);
|
||||||
|
try {
|
||||||
|
const res = await fetch(url, { signal: controller.signal });
|
||||||
|
clearTimeout(timer);
|
||||||
|
if (!res.ok) return null;
|
||||||
|
return Buffer.from(await res.arrayBuffer());
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
} finally {
|
||||||
|
clearTimeout(timer);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -6,6 +6,7 @@ import type { WorkerContext } from "../context.js";
|
|||||||
import { getLocaleContext } from "../locale.js";
|
import { getLocaleContext } from "../locale.js";
|
||||||
import type { LocaleContext } from "@app/shared-types";
|
import type { LocaleContext } from "@app/shared-types";
|
||||||
import { recordAiUsage } from "../lib/ai-usage.js";
|
import { recordAiUsage } from "../lib/ai-usage.js";
|
||||||
|
import { captureTrainingSample, type CaptureConsentFlags } from "../lib/shadow-capture.js";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Bild-/OCR-jobb (spec §54): hämtar scan_job, anropar AAMOS med kontraktvaliderad
|
* Bild-/OCR-jobb (spec §54): hämtar scan_job, anropar AAMOS med kontraktvaliderad
|
||||||
@@ -28,7 +29,7 @@ export async function processScanJob(ctx: WorkerContext, scanJobId: string): Pro
|
|||||||
|
|
||||||
const imageUrls = job.s3Keys.map((k) => ctx.readUrl(k));
|
const imageUrls = job.s3Keys.map((k) => ctx.readUrl(k));
|
||||||
const localeContext = await getLocaleContext(ctx, job.userId);
|
const localeContext = await getLocaleContext(ctx, job.userId);
|
||||||
const consents = await loadConsentFlags(ctx, job.userId);
|
const consentFlags = await loadConsentFlags(ctx, job.userId);
|
||||||
|
|
||||||
const started = Date.now();
|
const started = Date.now();
|
||||||
const result = await runAamosForJob(
|
const result = await runAamosForJob(
|
||||||
@@ -38,6 +39,7 @@ export async function processScanJob(ctx: WorkerContext, scanJobId: string): Pro
|
|||||||
imageUrls,
|
imageUrls,
|
||||||
job.context,
|
job.context,
|
||||||
localeContext,
|
localeContext,
|
||||||
|
consentFlags,
|
||||||
);
|
);
|
||||||
|
|
||||||
if (result.status === "failed" || result.output == null) {
|
if (result.status === "failed" || result.output == null) {
|
||||||
@@ -80,6 +82,27 @@ export async function processScanJob(ctx: WorkerContext, scanJobId: string): Pro
|
|||||||
|
|
||||||
await recordScanCompleted(ctx, job, result);
|
await recordScanCompleted(ctx, job, result);
|
||||||
|
|
||||||
|
// Shadow-capture: spara träningspar för framtida AAMOS-distillation (FAS 1).
|
||||||
|
// Kör aldrig synkront på användarens kritiska väg; fel swallås.
|
||||||
|
if (result.status === "ok" && result.output != null) {
|
||||||
|
const capture = await captureTrainingSample({
|
||||||
|
taskType: job.jobType as AamosTaskType,
|
||||||
|
inputS3Keys: job.s3Keys,
|
||||||
|
output: result.output as Record<string, unknown>,
|
||||||
|
modelVersion: result.modelVersion,
|
||||||
|
promptVersion: result.promptVersion,
|
||||||
|
latencyMs: result.latencyMs,
|
||||||
|
costUsd: result.costUsd,
|
||||||
|
inputTokens: result.inputTokens,
|
||||||
|
outputTokens: result.outputTokens,
|
||||||
|
consentFlags,
|
||||||
|
readUrl: ctx.readUrl,
|
||||||
|
});
|
||||||
|
if (!capture.ok) {
|
||||||
|
console.error("[scan processor] shadow-capture misslyckades:", capture.error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Bokför verklig AI-kostnad/tokens utan PII (spec §45).
|
// Bokför verklig AI-kostnad/tokens utan PII (spec §45).
|
||||||
if (result.costUsd != null || result.inputTokens || result.outputTokens) {
|
if (result.costUsd != null || result.inputTokens || result.outputTokens) {
|
||||||
await recordAiUsage(ctx.db, job.userId, {
|
await recordAiUsage(ctx.db, job.userId, {
|
||||||
@@ -107,7 +130,6 @@ export async function processScanJob(ctx: WorkerContext, scanJobId: string): Pro
|
|||||||
},
|
},
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
void consents;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async function runAamosForJob(
|
async function runAamosForJob(
|
||||||
@@ -117,6 +139,7 @@ async function runAamosForJob(
|
|||||||
imageUrls: string[],
|
imageUrls: string[],
|
||||||
context: unknown,
|
context: unknown,
|
||||||
localeContext: LocaleContext,
|
localeContext: LocaleContext,
|
||||||
|
consentFlags: CaptureConsentFlags,
|
||||||
) {
|
) {
|
||||||
switch (jobType) {
|
switch (jobType) {
|
||||||
case "ANALYZE_FRIDGE_IMAGE":
|
case "ANALYZE_FRIDGE_IMAGE":
|
||||||
@@ -129,33 +152,33 @@ async function runAamosForJob(
|
|||||||
marketLocale: localeContext.languageTag,
|
marketLocale: localeContext.languageTag,
|
||||||
knownItems: [],
|
knownItems: [],
|
||||||
},
|
},
|
||||||
{ localeContext },
|
{ localeContext, consentFlags },
|
||||||
);
|
);
|
||||||
case "ANALYZE_MEAL_IMAGE": {
|
case "ANALYZE_MEAL_IMAGE": {
|
||||||
const recipeContext = await buildRecipeContext(ctx, context);
|
const recipeContext = await buildRecipeContext(ctx, context);
|
||||||
return ctx.aamos.runTask(
|
return ctx.aamos.runTask(
|
||||||
"ANALYZE_MEAL_IMAGE",
|
"ANALYZE_MEAL_IMAGE",
|
||||||
{ imageUrls, recipeContext, marketLocale: localeContext.languageTag },
|
{ imageUrls, recipeContext, marketLocale: localeContext.languageTag },
|
||||||
{ localeContext },
|
{ localeContext, consentFlags },
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
case "READ_RECEIPT":
|
case "READ_RECEIPT":
|
||||||
return ctx.aamos.runTask(
|
return ctx.aamos.runTask(
|
||||||
"READ_RECEIPT",
|
"READ_RECEIPT",
|
||||||
{ imageUrls, marketLocale: localeContext.languageTag },
|
{ imageUrls, marketLocale: localeContext.languageTag },
|
||||||
{ localeContext },
|
{ localeContext, consentFlags },
|
||||||
);
|
);
|
||||||
case "READ_NUTRITION_LABEL":
|
case "READ_NUTRITION_LABEL":
|
||||||
return ctx.aamos.runTask(
|
return ctx.aamos.runTask(
|
||||||
"READ_NUTRITION_LABEL",
|
"READ_NUTRITION_LABEL",
|
||||||
{ imageUrls, marketLocale: localeContext.languageTag },
|
{ imageUrls, marketLocale: localeContext.languageTag },
|
||||||
{ localeContext },
|
{ localeContext, consentFlags },
|
||||||
);
|
);
|
||||||
case "READ_EXPIRY_DATE":
|
case "READ_EXPIRY_DATE":
|
||||||
return ctx.aamos.runTask(
|
return ctx.aamos.runTask(
|
||||||
"READ_EXPIRY_DATE",
|
"READ_EXPIRY_DATE",
|
||||||
{ imageUrls: imageUrls.slice(0, 2) },
|
{ imageUrls: imageUrls.slice(0, 2) },
|
||||||
{ localeContext },
|
{ localeContext, consentFlags },
|
||||||
);
|
);
|
||||||
default:
|
default:
|
||||||
throw new Error(`Jobbtypen ${jobType} hanteras inte av scan-processorn`);
|
throw new Error(`Jobbtypen ${jobType} hanteras inte av scan-processorn`);
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
/**
|
/**
|
||||||
* Gemini-lärar-tier för AAMOS-task-typer (SKIVA 1: kylskåpsskanning).
|
* Gemini-lärar-tier för AAMOS-task-typer (SKIVA 1).
|
||||||
*
|
*
|
||||||
* - Implementerar samma AamosClient-interface som HttpAamosClient/MockAamosClient.
|
* - Implementerar samma AamosClient-interface som HttpAamosClient/MockAamosClient.
|
||||||
* - App + Food API ser ingen skillnad; workern byter bara adapter.
|
* - App + Food API ser ingen skillnad; workern byter bara adapter.
|
||||||
@@ -7,7 +7,6 @@
|
|||||||
* - Kostnad/tokens rapporteras tillbaka så workern kan bokföra i ai_usage_counters.
|
* - Kostnad/tokens rapporteras tillbaka så workern kan bokföra i ai_usage_counters.
|
||||||
* - Global dagsbudget kan sättas via GEMINI_DAILY_BUDGET_USD.
|
* - Global dagsbudget kan sättas via GEMINI_DAILY_BUDGET_USD.
|
||||||
*/
|
*/
|
||||||
import { randomUUID } from "node:crypto";
|
|
||||||
import { z } from "zod";
|
import { z } from "zod";
|
||||||
import {
|
import {
|
||||||
TASK_CONTRACTS,
|
TASK_CONTRACTS,
|
||||||
@@ -17,14 +16,22 @@ import {
|
|||||||
detectedItemSchema,
|
detectedItemSchema,
|
||||||
type DetectedItem,
|
type DetectedItem,
|
||||||
generateRecipeCandidatesOutput,
|
generateRecipeCandidatesOutput,
|
||||||
|
analyzeMealImageOutput,
|
||||||
|
readNutritionLabelOutput,
|
||||||
|
readExpiryDateOutput,
|
||||||
|
readReceiptOutput,
|
||||||
} from "./tasks.js";
|
} from "./tasks.js";
|
||||||
import type { AamosCallOptions, AamosClient, AamosResult } from "./client.js";
|
import type { AamosCallOptions, AamosClient, AamosResult } from "./client.js";
|
||||||
import type { LocaleContext } from "@app/shared-types";
|
import type { LocaleContext } from "@app/shared-types";
|
||||||
|
|
||||||
const GEMINI_API_BASE = "https://generativelanguage.googleapis.com/v1beta";
|
const GEMINI_API_BASE = "https://generativelanguage.googleapis.com/v1beta";
|
||||||
|
|
||||||
/** Custom fetch using Node https module — works around IPv6 hangs on WSL. */
|
/** Custom fetch using Node https module — works around IPv6 hangs on WSL. Follows redirects. */
|
||||||
function customNodeFetch(input: string | URL | Request, init?: RequestInit): Promise<Response> {
|
function customNodeFetch(input: string | URL | Request, init?: RequestInit): Promise<Response> {
|
||||||
|
return requestOnce(input, init, 5);
|
||||||
|
}
|
||||||
|
|
||||||
|
function requestOnce(input: string | URL | Request, init: RequestInit | undefined, redirectsLeft: number): Promise<Response> {
|
||||||
return new Promise((resolve, reject) => {
|
return new Promise((resolve, reject) => {
|
||||||
const url = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url;
|
const url = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url;
|
||||||
const u = new URL(url);
|
const u = new URL(url);
|
||||||
@@ -45,6 +52,9 @@ function customNodeFetch(input: string | URL | Request, init?: RequestInit): Pro
|
|||||||
if (postData && !headers["content-length"]) {
|
if (postData && !headers["content-length"]) {
|
||||||
headers["content-length"] = String(Buffer.byteLength(postData));
|
headers["content-length"] = String(Buffer.byteLength(postData));
|
||||||
}
|
}
|
||||||
|
if (!headers["user-agent"]) {
|
||||||
|
headers["user-agent"] = "Mozilla/5.0 (compatible; Cibello/1.0)";
|
||||||
|
}
|
||||||
|
|
||||||
const req = httpMod.request(
|
const req = httpMod.request(
|
||||||
{
|
{
|
||||||
@@ -56,13 +66,23 @@ function customNodeFetch(input: string | URL | Request, init?: RequestInit): Pro
|
|||||||
timeout: 120_000,
|
timeout: 120_000,
|
||||||
},
|
},
|
||||||
(res) => {
|
(res) => {
|
||||||
|
const status = res.statusCode ?? 200;
|
||||||
|
const location = res.headers.location;
|
||||||
|
if (status >= 300 && status < 400 && location && redirectsLeft > 0) {
|
||||||
|
const nextUrl = new URL(location, url).toString();
|
||||||
|
requestOnce(nextUrl, { ...init, method: "GET", body: undefined }, redirectsLeft - 1)
|
||||||
|
.then(resolve)
|
||||||
|
.catch(reject);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
let body = "";
|
let body = "";
|
||||||
res.setEncoding("utf8");
|
res.setEncoding("utf8");
|
||||||
res.on("data", (chunk) => { body += chunk; });
|
res.on("data", (chunk) => { body += chunk; });
|
||||||
res.on("end", () => {
|
res.on("end", () => {
|
||||||
resolve(
|
resolve(
|
||||||
new Response(body, {
|
new Response(body, {
|
||||||
status: res.statusCode ?? 200,
|
status,
|
||||||
statusText: res.statusMessage ?? "OK",
|
statusText: res.statusMessage ?? "OK",
|
||||||
headers: new Headers(Object.entries(res.headers).map(([k, v]) => [k, String(v)])),
|
headers: new Headers(Object.entries(res.headers).map(([k, v]) => [k, String(v)])),
|
||||||
}),
|
}),
|
||||||
@@ -81,6 +101,9 @@ function customNodeFetch(input: string | URL | Request, init?: RequestInit): Pro
|
|||||||
/** Estimated max cost per image call in USD (pessimistic). */
|
/** Estimated max cost per image call in USD (pessimistic). */
|
||||||
const COST_ESTIMATE_PER_IMAGE_USD = 0.0015;
|
const COST_ESTIMATE_PER_IMAGE_USD = 0.0015;
|
||||||
|
|
||||||
|
/** Estimated max cost per simple OCR image call in USD. */
|
||||||
|
const COST_ESTIMATE_PER_OCR_IMAGE_USD = 0.001;
|
||||||
|
|
||||||
export interface BudgetStore {
|
export interface BudgetStore {
|
||||||
/** Return current daily spend in USD. */
|
/** Return current daily spend in USD. */
|
||||||
getDailySpendUsd(): Promise<number>;
|
getDailySpendUsd(): Promise<number>;
|
||||||
@@ -222,7 +245,7 @@ export class GeminiAamosClient implements AamosClient {
|
|||||||
|
|
||||||
async healthCheck(): Promise<{ ok: boolean; detail?: string }> {
|
async healthCheck(): Promise<{ ok: boolean; detail?: string }> {
|
||||||
try {
|
try {
|
||||||
const url = `${GEMINI_API_BASE}/models?key=${this.cfg.apiKey}`;
|
const url = `${GEMINI_API_BASE}/models?key=${encodeURIComponent(this.cfg.apiKey)}`;
|
||||||
const res = await this.cfg.fetchImpl(url, { signal: AbortSignal.timeout(5_000) });
|
const res = await this.cfg.fetchImpl(url, { signal: AbortSignal.timeout(5_000) });
|
||||||
return { ok: res.ok, detail: `status ${res.status}` };
|
return { ok: res.ok, detail: `status ${res.status}` };
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
@@ -251,6 +274,14 @@ export class GeminiAamosClient implements AamosClient {
|
|||||||
return this.analyzeStorageImage(taskType, parsedInput.data as TaskInput<"ANALYZE_FRIDGE_IMAGE">, options) as Promise<AamosResult<T>>;
|
return this.analyzeStorageImage(taskType, parsedInput.data as TaskInput<"ANALYZE_FRIDGE_IMAGE">, options) as Promise<AamosResult<T>>;
|
||||||
case "GENERATE_RECIPE_CANDIDATES":
|
case "GENERATE_RECIPE_CANDIDATES":
|
||||||
return this.generateRecipeCandidates(parsedInput.data as TaskInput<"GENERATE_RECIPE_CANDIDATES">, options) as Promise<AamosResult<T>>;
|
return this.generateRecipeCandidates(parsedInput.data as TaskInput<"GENERATE_RECIPE_CANDIDATES">, options) as Promise<AamosResult<T>>;
|
||||||
|
case "ANALYZE_MEAL_IMAGE":
|
||||||
|
return this.analyzeMealImage(parsedInput.data as TaskInput<"ANALYZE_MEAL_IMAGE">, options) as Promise<AamosResult<T>>;
|
||||||
|
case "READ_NUTRITION_LABEL":
|
||||||
|
return this.readNutritionLabel(parsedInput.data as TaskInput<"READ_NUTRITION_LABEL">, options) as Promise<AamosResult<T>>;
|
||||||
|
case "READ_EXPIRY_DATE":
|
||||||
|
return this.readExpiryDate(parsedInput.data as TaskInput<"READ_EXPIRY_DATE">, options) as Promise<AamosResult<T>>;
|
||||||
|
case "READ_RECEIPT":
|
||||||
|
return this.readReceipt(parsedInput.data as TaskInput<"READ_RECEIPT">, options) as Promise<AamosResult<T>>;
|
||||||
default:
|
default:
|
||||||
return {
|
return {
|
||||||
status: "failed",
|
status: "failed",
|
||||||
@@ -295,7 +326,7 @@ export class GeminiAamosClient implements AamosClient {
|
|||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
const url = `${GEMINI_API_BASE}/models/${this.cfg.model}:generateContent?key=${this.cfg.apiKey}`;
|
const url = `${GEMINI_API_BASE}/models/${this.cfg.model}:generateContent?key=${encodeURIComponent(this.cfg.apiKey)}`;
|
||||||
const res = await this.cfg.fetchImpl(url, {
|
const res = await this.cfg.fetchImpl(url, {
|
||||||
method: "POST",
|
method: "POST",
|
||||||
headers: { "content-type": "application/json" },
|
headers: { "content-type": "application/json" },
|
||||||
@@ -350,6 +381,186 @@ export class GeminiAamosClient implements AamosClient {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private async runVisionTask<T extends AamosTaskType>(
|
||||||
|
taskType: T,
|
||||||
|
imageUrls: string[],
|
||||||
|
prompt: string,
|
||||||
|
outputSchema: z.ZodSchema<unknown>,
|
||||||
|
mapOutput: (parsed: unknown) => TaskOutput<T>,
|
||||||
|
options: AamosCallOptions,
|
||||||
|
promptVersion: string,
|
||||||
|
costEstimatePerImage: number,
|
||||||
|
): Promise<AamosResult<T>> {
|
||||||
|
const locale = options.localeContext ?? this.defaultLocale();
|
||||||
|
const started = Date.now();
|
||||||
|
|
||||||
|
const imageParts = await this.fetchImageParts(imageUrls, this.cfg.timeoutMs);
|
||||||
|
const estimatedCostUsd = imageParts.length * costEstimatePerImage;
|
||||||
|
if (await this.isOverBudget(estimatedCostUsd)) {
|
||||||
|
return {
|
||||||
|
status: "failed",
|
||||||
|
output: null,
|
||||||
|
error: "Global Gemini-dagsbudget är förbrukad.",
|
||||||
|
} as AamosResult<T>;
|
||||||
|
}
|
||||||
|
|
||||||
|
const payload = {
|
||||||
|
contents: [{ parts: [{ text: prompt }, ...imageParts] }],
|
||||||
|
generationConfig: {
|
||||||
|
responseMimeType: "application/json",
|
||||||
|
temperature: 0.2,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
const url = `${GEMINI_API_BASE}/models/${this.cfg.model}:generateContent?key=${encodeURIComponent(this.cfg.apiKey)}`;
|
||||||
|
const res = await this.cfg.fetchImpl(url, {
|
||||||
|
method: "POST",
|
||||||
|
headers: { "content-type": "application/json" },
|
||||||
|
body: JSON.stringify(payload),
|
||||||
|
signal: AbortSignal.timeout(this.cfg.timeoutMs),
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!res.ok) {
|
||||||
|
const body = await res.text();
|
||||||
|
return {
|
||||||
|
status: "failed",
|
||||||
|
output: null,
|
||||||
|
error: `Gemini ${res.status}: ${body.slice(0, 500)}`,
|
||||||
|
} as AamosResult<T>;
|
||||||
|
}
|
||||||
|
|
||||||
|
const geminiBody = (await res.json()) as {
|
||||||
|
candidates?: Array<{ content?: { parts?: Array<{ text?: string }> } }>;
|
||||||
|
usageMetadata?: { promptTokenCount?: number; candidatesTokenCount?: number };
|
||||||
|
};
|
||||||
|
const text = geminiBody.candidates?.[0]?.content?.parts?.[0]?.text ?? "";
|
||||||
|
|
||||||
|
let parsed: unknown;
|
||||||
|
try {
|
||||||
|
const json = JSON.parse(text);
|
||||||
|
const safe = outputSchema.safeParse(json);
|
||||||
|
if (!safe.success) {
|
||||||
|
return {
|
||||||
|
status: "failed",
|
||||||
|
output: null,
|
||||||
|
error: `Gemini-svar matchar inte schema: ${safe.error.message}`,
|
||||||
|
} as AamosResult<T>;
|
||||||
|
}
|
||||||
|
parsed = safe.data;
|
||||||
|
} catch {
|
||||||
|
return {
|
||||||
|
status: "failed",
|
||||||
|
output: null,
|
||||||
|
error: "Gemini-svar var inte giltig JSON.",
|
||||||
|
} as AamosResult<T>;
|
||||||
|
}
|
||||||
|
|
||||||
|
const inputTokens = geminiBody.usageMetadata?.promptTokenCount ?? 0;
|
||||||
|
const outputTokens = geminiBody.usageMetadata?.candidatesTokenCount ?? 0;
|
||||||
|
const costUsd = this.estimateCostUsd(inputTokens, outputTokens);
|
||||||
|
await this.recordSpend(costUsd);
|
||||||
|
|
||||||
|
const latencyMs = Date.now() - started;
|
||||||
|
const output = mapOutput(parsed) as TaskOutput<T>;
|
||||||
|
|
||||||
|
return {
|
||||||
|
status: "ok",
|
||||||
|
output,
|
||||||
|
modelVersion: this.cfg.model,
|
||||||
|
promptVersion,
|
||||||
|
latencyMs,
|
||||||
|
costUsd,
|
||||||
|
inputTokens,
|
||||||
|
outputTokens,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private async analyzeMealImage(
|
||||||
|
input: TaskInput<"ANALYZE_MEAL_IMAGE">,
|
||||||
|
options: AamosCallOptions,
|
||||||
|
): Promise<AamosResult<"ANALYZE_MEAL_IMAGE">> {
|
||||||
|
const locale = options.localeContext ?? this.defaultLocale();
|
||||||
|
const prompt = this.buildMealPrompt(locale, input.recipeContext);
|
||||||
|
return this.runVisionTask(
|
||||||
|
"ANALYZE_MEAL_IMAGE",
|
||||||
|
input.imageUrls,
|
||||||
|
prompt,
|
||||||
|
analyzeMealImageOutput,
|
||||||
|
(parsed) => parsed as TaskOutput<"ANALYZE_MEAL_IMAGE">,
|
||||||
|
options,
|
||||||
|
"gemini-meal-v1",
|
||||||
|
COST_ESTIMATE_PER_IMAGE_USD,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async readNutritionLabel(
|
||||||
|
input: TaskInput<"READ_NUTRITION_LABEL">,
|
||||||
|
options: AamosCallOptions,
|
||||||
|
): Promise<AamosResult<"READ_NUTRITION_LABEL">> {
|
||||||
|
const locale = options.localeContext ?? this.defaultLocale();
|
||||||
|
const prompt = this.buildNutritionPrompt(locale);
|
||||||
|
return this.runVisionTask(
|
||||||
|
"READ_NUTRITION_LABEL",
|
||||||
|
input.imageUrls,
|
||||||
|
prompt,
|
||||||
|
readNutritionLabelOutput,
|
||||||
|
(parsed) => parsed as TaskOutput<"READ_NUTRITION_LABEL">,
|
||||||
|
options,
|
||||||
|
"gemini-nutrition-v1",
|
||||||
|
COST_ESTIMATE_PER_OCR_IMAGE_USD,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async readExpiryDate(
|
||||||
|
input: TaskInput<"READ_EXPIRY_DATE">,
|
||||||
|
options: AamosCallOptions,
|
||||||
|
): Promise<AamosResult<"READ_EXPIRY_DATE">> {
|
||||||
|
const locale = options.localeContext ?? this.defaultLocale();
|
||||||
|
const prompt = this.buildExpiryPrompt(locale);
|
||||||
|
return this.runVisionTask(
|
||||||
|
"READ_EXPIRY_DATE",
|
||||||
|
input.imageUrls,
|
||||||
|
prompt,
|
||||||
|
readExpiryDateOutput,
|
||||||
|
(parsed) => parsed as TaskOutput<"READ_EXPIRY_DATE">,
|
||||||
|
options,
|
||||||
|
"gemini-expiry-v1",
|
||||||
|
COST_ESTIMATE_PER_OCR_IMAGE_USD,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async readReceipt(
|
||||||
|
input: TaskInput<"READ_RECEIPT">,
|
||||||
|
options: AamosCallOptions,
|
||||||
|
): Promise<AamosResult<"READ_RECEIPT">> {
|
||||||
|
const locale = options.localeContext ?? this.defaultLocale();
|
||||||
|
const prompt = this.buildReceiptPrompt(locale);
|
||||||
|
return this.runVisionTask(
|
||||||
|
"READ_RECEIPT",
|
||||||
|
input.imageUrls,
|
||||||
|
prompt,
|
||||||
|
readReceiptOutput,
|
||||||
|
(parsed) => parsed as TaskOutput<"READ_RECEIPT">,
|
||||||
|
options,
|
||||||
|
"gemini-receipt-v1",
|
||||||
|
COST_ESTIMATE_PER_IMAGE_USD,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async fetchImageParts(
|
||||||
|
imageUrls: string[],
|
||||||
|
timeoutMs: number,
|
||||||
|
): Promise<Array<{ inline_data: { mime_type: string; data: string } }>> {
|
||||||
|
return Promise.all(
|
||||||
|
imageUrls.slice(0, 6).map(async (url) => {
|
||||||
|
const res = await this.cfg.fetchImpl(url, { signal: AbortSignal.timeout(timeoutMs) });
|
||||||
|
if (!res.ok) throw new Error(`Kunde inte hämta bild: ${res.status} ${url}`);
|
||||||
|
const buf = Buffer.from(await res.arrayBuffer());
|
||||||
|
return { inline_data: { mime_type: this.mimeType(buf), data: buf.toString("base64") } };
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
private async generateRecipeCandidates(
|
private async generateRecipeCandidates(
|
||||||
input: TaskInput<"GENERATE_RECIPE_CANDIDATES">,
|
input: TaskInput<"GENERATE_RECIPE_CANDIDATES">,
|
||||||
options: AamosCallOptions,
|
options: AamosCallOptions,
|
||||||
@@ -377,7 +588,7 @@ export class GeminiAamosClient implements AamosClient {
|
|||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
const url = `${GEMINI_API_BASE}/models/${this.cfg.model}:generateContent?key=${this.cfg.apiKey}`;
|
const url = `${GEMINI_API_BASE}/models/${this.cfg.model}:generateContent?key=${encodeURIComponent(this.cfg.apiKey)}`;
|
||||||
const res = await this.cfg.fetchImpl(url, {
|
const res = await this.cfg.fetchImpl(url, {
|
||||||
method: "POST",
|
method: "POST",
|
||||||
headers: { "content-type": "application/json" },
|
headers: { "content-type": "application/json" },
|
||||||
@@ -553,6 +764,129 @@ Svara ENDAST med giltig JSON i exakt detta format:
|
|||||||
}
|
}
|
||||||
imageQualityIssues kan innehålla någon av: "dark", "blurry", "occlusion", "too_far".
|
imageQualityIssues kan innehålla någon av: "dark", "blurry", "occlusion", "too_far".
|
||||||
|
|
||||||
|
Språk: ${lang}.`;
|
||||||
|
}
|
||||||
|
|
||||||
|
private buildMealPrompt(locale: LocaleContext, recipeContext: unknown): string {
|
||||||
|
const lang = locale.languageTag.startsWith("en") ? "English" : "Swedish";
|
||||||
|
const ctx = recipeContext as {
|
||||||
|
recipeId?: string;
|
||||||
|
titleSv?: string;
|
||||||
|
nutritionPerPortion?: Record<string, number>;
|
||||||
|
portions?: number;
|
||||||
|
} | null;
|
||||||
|
|
||||||
|
let contextLine = "";
|
||||||
|
if (ctx?.titleSv) {
|
||||||
|
contextLine = `Användaren säger att måltiden ska matcha receptet "${ctx.titleSv}" (${ctx.portions ?? "?"} portioner). `;
|
||||||
|
}
|
||||||
|
|
||||||
|
return `Du är en svensk näringsassistent. ${contextLine}Bilden visar en serverad måltid.
|
||||||
|
|
||||||
|
Uppskatta:
|
||||||
|
- matchesRecipeContext: true/false/null – matchar bilden receptet användaren nämnde?
|
||||||
|
- portionFractionEstimate: hur stor del av en normalportion ser det ut att vara? 0.0–5.0, eller null.
|
||||||
|
- kcalRange: { min, max, mostLikely } för hela måltiden, eller null.
|
||||||
|
- components: lista med uppskattade komponenter (namn, ev. canonicalIngredientId som null, estimatedGrams, confidence).
|
||||||
|
- confidence: 0.0–1.0 för hela tolkningen.
|
||||||
|
|
||||||
|
Svara ENDAST med giltig JSON som matchar detta schema:
|
||||||
|
{
|
||||||
|
"matchesRecipeContext": boolean | null,
|
||||||
|
"portionFractionEstimate": number | null,
|
||||||
|
"kcalRange": { "min": number, "max": number, "mostLikely": number } | null,
|
||||||
|
"components": [
|
||||||
|
{ "name": string, "canonicalIngredientId": string | null, "estimatedGrams": number | null, "confidence": number }
|
||||||
|
],
|
||||||
|
"confidence": number
|
||||||
|
}
|
||||||
|
|
||||||
|
Språk: ${lang}.`;
|
||||||
|
}
|
||||||
|
|
||||||
|
private buildNutritionPrompt(locale: LocaleContext): string {
|
||||||
|
const lang = locale.languageTag.startsWith("en") ? "English" : "Swedish";
|
||||||
|
return `Du är en OCR-assistent för svenska livsmedelsförpackningar. Bilden visar en näringsdeklaration/ingredienslista.
|
||||||
|
|
||||||
|
Läs av exakt det som står på etiketten. Gissa aldrig värden som inte syns. Returnera:
|
||||||
|
- basis: "per_100_g", "per_100_ml" eller "per_portion" – vad gäller tabellen?
|
||||||
|
- values: { kcal, proteinG, carbsG, fatG, saturatedFatG, fiberG, sugarG, saltG } – null om ej synligt.
|
||||||
|
- ingredientsText: rå text med ingredienser, eller null.
|
||||||
|
- allergensDeclared: lista med allergener som explicit nämns (t.ex. ["gluten", "mjölk", "nötter"]).
|
||||||
|
- gtin: streckkod/EAN om synlig, annars null.
|
||||||
|
- productName: produktnamn om synligt, annars null.
|
||||||
|
- brand: varumärke om synligt, annars null.
|
||||||
|
- confidence: 0.0–1.0.
|
||||||
|
|
||||||
|
Svara ENDAST med giltig JSON som matchar detta schema:
|
||||||
|
{
|
||||||
|
"basis": "per_100_g" | "per_100_ml" | "per_portion" | null,
|
||||||
|
"values": { "kcal": number|null, "proteinG": number|null, "carbsG": number|null, "fatG": number|null, "saturatedFatG": number|null, "fiberG": number|null, "sugarG": number|null, "saltG": number|null } | null,
|
||||||
|
"ingredientsText": string | null,
|
||||||
|
"allergensDeclared": string[],
|
||||||
|
"gtin": string | null,
|
||||||
|
"productName": string | null,
|
||||||
|
"brand": string | null,
|
||||||
|
"confidence": number
|
||||||
|
}
|
||||||
|
|
||||||
|
Språk: ${lang}.`;
|
||||||
|
}
|
||||||
|
|
||||||
|
private buildExpiryPrompt(locale: LocaleContext): string {
|
||||||
|
const lang = locale.languageTag.startsWith("en") ? "English" : "Swedish";
|
||||||
|
return `Du är en OCR-assistent för svenska livsmedelsförpackningar. Bilden visar ett bäst-före-datum eller sista förbrukningsdatum.
|
||||||
|
|
||||||
|
Returnera:
|
||||||
|
- date: datum i format YYYY-MM-DD, eller null om ej läsbart.
|
||||||
|
- dateKind: "best_before" för "bäst före", "use_by" för "sista förbrukningsdag", annars null.
|
||||||
|
- confidence: 0.0–1.0.
|
||||||
|
|
||||||
|
Svara ENDAST med giltig JSON:
|
||||||
|
{
|
||||||
|
"date": "YYYY-MM-DD" | null,
|
||||||
|
"dateKind": "best_before" | "use_by" | null,
|
||||||
|
"confidence": number
|
||||||
|
}
|
||||||
|
|
||||||
|
Språk: ${lang}.`;
|
||||||
|
}
|
||||||
|
|
||||||
|
private buildReceiptPrompt(locale: LocaleContext): string {
|
||||||
|
const lang = locale.languageTag.startsWith("en") ? "English" : "Swedish";
|
||||||
|
return `Du är en OCR-assistent för svenska kvitton. Bilden visar ett butikskvitto.
|
||||||
|
|
||||||
|
Läs av:
|
||||||
|
- storeName: butikens namn, eller null.
|
||||||
|
- purchaseDate: datum i format YYYY-MM-DD, eller null.
|
||||||
|
- lines: varje rad med { rawText (exakt text), normalizedName (förenklat livsmedelsnamn eller null), canonicalIngredientId (alltid null), quantity (antal/enhet om går att utläsa, annars null), unit (enhetskod eller null), unitPriceMinor (ören per styck om synligt, annars null), totalPriceMinor (radens totalpris i ören om synligt, annars null), isDiscount (true om det är en rabatt/avdrag-rad), confidence (0.0–1.0) }.
|
||||||
|
- totalMinor: kvittots totalsumma i ören, eller null.
|
||||||
|
- discountTotalMinor: total rabatt i ören, eller null.
|
||||||
|
- confidence: 0.0–1.0 för hela tolkningen.
|
||||||
|
|
||||||
|
Svara ENDAST med giltig JSON som matchar detta schema:
|
||||||
|
{
|
||||||
|
"storeName": string | null,
|
||||||
|
"purchaseDate": "YYYY-MM-DD" | null,
|
||||||
|
"lines": [
|
||||||
|
{
|
||||||
|
"rawText": string,
|
||||||
|
"normalizedName": string | null,
|
||||||
|
"canonicalIngredientId": null,
|
||||||
|
"quantity": number | null,
|
||||||
|
"unit": string | null,
|
||||||
|
"unitPriceMinor": number | null,
|
||||||
|
"totalPriceMinor": number | null,
|
||||||
|
"isDiscount": boolean,
|
||||||
|
"confidence": number
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"totalMinor": number | null,
|
||||||
|
"discountTotalMinor": number | null,
|
||||||
|
"confidence": number
|
||||||
|
}
|
||||||
|
|
||||||
|
Använd endast de enhetskoder du kan utläsa (GRAM, KILOGRAM, LITER, etc.), annars null. quantity är alltid ett decimaltal.
|
||||||
Språk: ${lang}.`;
|
Språk: ${lang}.`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -127,12 +127,12 @@ describe("GeminiAamosClient", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
const result = await client.runTask(
|
const result = await client.runTask(
|
||||||
"READ_RECEIPT",
|
"DEDUPLICATE_INVENTORY",
|
||||||
{ imageUrls: ["http://localhost/v1/mock-s3/receipt.jpg"], marketLocale: "sv-SE" },
|
{ items: [] },
|
||||||
);
|
);
|
||||||
|
|
||||||
expect(result.status).toBe("failed");
|
expect(result.status).toBe("failed");
|
||||||
expect(result.error).toContain("READ_RECEIPT");
|
expect(result.error).toContain("DEDUPLICATE_INVENTORY");
|
||||||
});
|
});
|
||||||
|
|
||||||
it("validates input before calling Gemini", async () => {
|
it("validates input before calling Gemini", async () => {
|
||||||
|
|||||||
Generated
+3
@@ -296,6 +296,9 @@ importers:
|
|||||||
'@app/subscriptions':
|
'@app/subscriptions':
|
||||||
specifier: workspace:*
|
specifier: workspace:*
|
||||||
version: link:../../packages/subscriptions
|
version: link:../../packages/subscriptions
|
||||||
|
'@aws-sdk/client-s3':
|
||||||
|
specifier: ^3.750.0
|
||||||
|
version: 3.1102.0
|
||||||
bullmq:
|
bullmq:
|
||||||
specifier: ^6.0.0
|
specifier: ^6.0.0
|
||||||
version: 6.0.5(ioredis@6.0.0)(pg@8.22.0)
|
version: 6.0.5(ioredis@6.0.0)(pg@8.22.0)
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ for name in "${FORBIDDEN[@]}"; do
|
|||||||
HITS=$(grep -ril "$name" . \
|
HITS=$(grep -ril "$name" . \
|
||||||
--exclude-dir=node_modules --exclude-dir=.git --exclude-dir=dist \
|
--exclude-dir=node_modules --exclude-dir=.git --exclude-dir=dist \
|
||||||
--exclude-dir=.turbo --exclude-dir=.expo --exclude-dir=.data \
|
--exclude-dir=.turbo --exclude-dir=.expo --exclude-dir=.data \
|
||||||
|
--exclude-dir=.terraform \
|
||||||
--exclude=brand-guard.sh 2>/dev/null || true)
|
--exclude=brand-guard.sh 2>/dev/null || true)
|
||||||
if [ -n "$HITS" ]; then
|
if [ -n "$HITS" ]; then
|
||||||
echo "FEL: förbjudet namn '$name' hittades i:"
|
echo "FEL: förbjudet namn '$name' hittades i:"
|
||||||
|
|||||||
Reference in New Issue
Block a user