feat(worker/ai): Gemini-lärar-tier för scan/detect/allergen + shadow-capture
- Utöka GeminiAamosClient med ANALYZE_MEAL_IMAGE, READ_NUTRITION_LABEL, READ_EXPIRY_DATE och READ_RECEIPT (vision-prompts mot kontraktsscheman). - Lägg till apps/worker/src/lib/shadow-capture.ts: S3/local fallback, imageTraining-gating, ingen PII, aldrig faila användarvägen. - Integrera capture i scan-processorn efter lyckad runTask med consentFlags. - Lägg till @aws-sdk/client-s3 i worker samt eval:capture-skript. - Uppdatera eval:scan till Wikimedia Special:FilePath + redirect-following. - Exkludera .terraform i brand-guard för att undvika false positives. - Uppdatera gemini-test för DEDUPLICATE_INVENTORY som unsupported.
This commit is contained in:
Sven (AAMOS AI)
@@ -11,7 +11,8 @@
|
||||
"typecheck": "tsc --noEmit",
|
||||
"test": "vitest run --passWithNoTests",
|
||||
"eval": "tsx src/eval/run.ts",
|
||||
"eval:scan": "tsx src/eval/scan-eval.ts"
|
||||
"eval:scan": "tsx src/eval/scan-eval.ts",
|
||||
"eval:capture": "tsx src/eval/capture-eval.ts"
|
||||
},
|
||||
"dependencies": {
|
||||
"@app/ai-contracts": "workspace:*",
|
||||
@@ -25,9 +26,11 @@
|
||||
"@app/recommendation-engine": "workspace:*",
|
||||
"@app/shared-types": "workspace:*",
|
||||
"@app/subscriptions": "workspace:*",
|
||||
"@aws-sdk/client-s3": "^3.750.0",
|
||||
"bullmq": "^6.0.0",
|
||||
"dotenv": "^16.4.0",
|
||||
"drizzle-orm": "^0.45.0",
|
||||
"@aws-sdk/client-s3": "^3.750.0",
|
||||
"ioredis": "^6.0.0",
|
||||
"pg": "^8.13.0"
|
||||
},
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
/**
|
||||
* Best-effort verifiering av shadow-capture (ingen riktig AI, bara lagring).
|
||||
*
|
||||
* S3_MODE=mock pnpm --filter @app/worker eval:capture
|
||||
*/
|
||||
import { captureTrainingSample } from "../lib/shadow-capture.js";
|
||||
|
||||
async function main() {
|
||||
const result = await captureTrainingSample({
|
||||
taskType: "ANALYZE_FRIDGE_IMAGE",
|
||||
inputS3Keys: ["uploads/test/fridge.jpg"],
|
||||
output: {
|
||||
items: [
|
||||
{
|
||||
detectedName: "Testmjölk",
|
||||
canonicalIngredientId: null,
|
||||
brand: "Testa",
|
||||
estimatedQuantity: 1,
|
||||
unit: "LITER",
|
||||
bestBeforeDate: null,
|
||||
confidence: 0.97,
|
||||
requiresConfirmation: false,
|
||||
boundingBox: null,
|
||||
},
|
||||
],
|
||||
imageQualityIssues: [],
|
||||
},
|
||||
modelVersion: "gemini-2.5-flash",
|
||||
promptVersion: "gemini-fridge-v1",
|
||||
latencyMs: 1234,
|
||||
costUsd: 0.0001,
|
||||
inputTokens: 400,
|
||||
outputTokens: 120,
|
||||
consentFlags: {
|
||||
personalization: true,
|
||||
anonymizedImprovement: true,
|
||||
imageTraining: true,
|
||||
},
|
||||
readUrl: (key: string) => `http://localhost/v1/mock-s3/${encodeURIComponent(key)}?sig=dummy`,
|
||||
});
|
||||
|
||||
if (!result.ok) {
|
||||
console.error("[eval:capture] misslyckades:", result.error);
|
||||
process.exit(1);
|
||||
}
|
||||
console.log(`[eval:capture] träningspar sparat på nyckel: ${result.key}`);
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -40,7 +40,7 @@ interface ScanEvalCase {
|
||||
const CASES: ScanEvalCase[] = [
|
||||
{
|
||||
id: "arla-milk-fridge",
|
||||
imageUrls: ["https://upload.wikimedia.org/wikipedia/commons/6/6c/Arla_Ko_Mellanmj%C3%B6lk_1L.jpg"],
|
||||
imageUrls: ["https://commons.wikimedia.org/wiki/Special:FilePath/Arla_Ko_Mellanmj%C3%B6lk_1L.jpg?width=800"],
|
||||
locationType: "fridge",
|
||||
marketLocale: "sv-SE",
|
||||
checks: (items) => {
|
||||
@@ -56,7 +56,7 @@ const CASES: ScanEvalCase[] = [
|
||||
},
|
||||
{
|
||||
id: "swedish-butter-fridge",
|
||||
imageUrls: ["https://upload.wikimedia.org/wikipedia/commons/3/3e/Svenskt_Sm%C3%B6r_Normalsaltat_80-25_500g.jpg"],
|
||||
imageUrls: ["https://commons.wikimedia.org/wiki/Special:FilePath/Svenskt_Sm%C3%B6r_Normalsaltat_80-25_500g.jpg?width=800"],
|
||||
locationType: "fridge",
|
||||
marketLocale: "sv-SE",
|
||||
checks: (items) => {
|
||||
|
||||
@@ -0,0 +1,179 @@
|
||||
/**
|
||||
* Shadow-capture: sparar varje lyckad AI-inferens som träningsdata i S3.
|
||||
*
|
||||
* - Kör aldrig synkront på användarens kritiska väg; failures swallås.
|
||||
* - Ingen PII (inga userId/householdId). Endast S3-nycklar, output och metadata.
|
||||
* - Rå bild kopieras ENDAST när imageTraining-samtycke är true.
|
||||
* - Durabelt = S3 i prod, lokalt filsystem i mock/test.
|
||||
*/
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { mkdir, writeFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import {
|
||||
PutObjectCommand,
|
||||
S3Client,
|
||||
type S3ClientConfig,
|
||||
} from "@aws-sdk/client-s3";
|
||||
import type { AamosTaskType } from "@app/ai-contracts";
|
||||
|
||||
export interface CaptureConsentFlags {
|
||||
personalization: boolean;
|
||||
anonymizedImprovement: boolean;
|
||||
imageTraining: boolean;
|
||||
}
|
||||
|
||||
export interface CaptureResult {
|
||||
ok: boolean;
|
||||
key?: string;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export interface CaptureInput {
|
||||
taskType: AamosTaskType;
|
||||
inputS3Keys: string[];
|
||||
output: Record<string, unknown>;
|
||||
modelVersion?: string | null;
|
||||
promptVersion?: string | null;
|
||||
latencyMs?: number | null;
|
||||
costUsd?: number | null;
|
||||
inputTokens?: number | null;
|
||||
outputTokens?: number | null;
|
||||
consentFlags: CaptureConsentFlags;
|
||||
readUrl: (key: string) => string;
|
||||
/** Override for tests; otherwise built from env. */
|
||||
storage?: CaptureStorage;
|
||||
}
|
||||
|
||||
export interface CaptureStorage {
|
||||
put(key: string, data: Buffer, contentType: string): Promise<void>;
|
||||
}
|
||||
|
||||
const TRAINING_PREFIX = "training/cibello";
|
||||
const MOCK_ROOT = path.resolve(process.cwd(), ".data/s3-training");
|
||||
|
||||
class S3CaptureStorage implements CaptureStorage {
|
||||
private readonly client: S3Client;
|
||||
private readonly bucket: string;
|
||||
|
||||
constructor(bucket: string, config: S3ClientConfig) {
|
||||
this.bucket = bucket;
|
||||
this.client = new S3Client(config);
|
||||
}
|
||||
|
||||
async put(key: string, data: Buffer, contentType: string): Promise<void> {
|
||||
await this.client.send(
|
||||
new PutObjectCommand({
|
||||
Bucket: this.bucket,
|
||||
Key: key,
|
||||
Body: data,
|
||||
ContentType: contentType,
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
class LocalCaptureStorage implements CaptureStorage {
|
||||
async put(key: string, data: Buffer): Promise<void> {
|
||||
const filePath = path.join(MOCK_ROOT, key);
|
||||
await mkdir(path.dirname(filePath), { recursive: true });
|
||||
await writeFile(filePath, data);
|
||||
}
|
||||
}
|
||||
|
||||
let cachedStorage: CaptureStorage | undefined;
|
||||
|
||||
export function createCaptureStorage(): CaptureStorage {
|
||||
if (cachedStorage) return cachedStorage;
|
||||
|
||||
const mode = process.env.S3_MODE ?? "mock";
|
||||
if (mode === "aws") {
|
||||
const bucket = process.env.S3_BUCKET ?? "cibello-production";
|
||||
const config: S3ClientConfig = {
|
||||
region: process.env.S3_REGION || "eu-north-1",
|
||||
...(process.env.S3_ENDPOINT
|
||||
? { endpoint: process.env.S3_ENDPOINT, forcePathStyle: true }
|
||||
: {}),
|
||||
...(process.env.S3_ACCESS_KEY_ID && process.env.S3_SECRET_ACCESS_KEY
|
||||
? {
|
||||
credentials: {
|
||||
accessKeyId: process.env.S3_ACCESS_KEY_ID,
|
||||
secretAccessKey: process.env.S3_SECRET_ACCESS_KEY,
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
};
|
||||
cachedStorage = new S3CaptureStorage(bucket, config);
|
||||
} else {
|
||||
cachedStorage = new LocalCaptureStorage();
|
||||
}
|
||||
return cachedStorage;
|
||||
}
|
||||
|
||||
export async function captureTrainingSample(input: CaptureInput): Promise<CaptureResult> {
|
||||
try {
|
||||
const storage = input.storage ?? createCaptureStorage();
|
||||
const date = new Date().toISOString().slice(0, 10);
|
||||
const id = randomUUID();
|
||||
const prefix = `${TRAINING_PREFIX}/${input.taskType}/${date}/${id}`;
|
||||
const jsonKey = `${prefix}/sample.json`;
|
||||
|
||||
const sample: Record<string, unknown> = {
|
||||
taskType: input.taskType,
|
||||
inputReferences: input.inputS3Keys,
|
||||
output: input.output,
|
||||
modelVersion: input.modelVersion ?? null,
|
||||
promptVersion: input.promptVersion ?? null,
|
||||
latencyMs: input.latencyMs ?? null,
|
||||
costUsd: input.costUsd ?? null,
|
||||
inputTokens: input.inputTokens ?? null,
|
||||
outputTokens: input.outputTokens ?? null,
|
||||
consentFlags: input.consentFlags,
|
||||
capturedAt: new Date().toISOString(),
|
||||
};
|
||||
|
||||
const trainingImageKeys: string[] = [];
|
||||
if (input.consentFlags.imageTraining && input.inputS3Keys.length > 0) {
|
||||
for (let i = 0; i < input.inputS3Keys.length; i++) {
|
||||
const originalKey = input.inputS3Keys[i];
|
||||
if (!originalKey) continue;
|
||||
try {
|
||||
const imageKey = `${prefix}/images/${String(i).padStart(3, "0")}.jpg`;
|
||||
const imageData = await fetchImageData(input.readUrl(originalKey));
|
||||
if (imageData) {
|
||||
await storage.put(imageKey, imageData, "image/jpeg");
|
||||
trainingImageKeys.push(imageKey);
|
||||
}
|
||||
} catch (err) {
|
||||
// Best effort: log but continue; JSON still has original S3 references.
|
||||
console.error(`[shadow-capture] failed to copy image ${originalKey}:`, err);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (trainingImageKeys.length > 0) {
|
||||
sample.trainingImageKeys = trainingImageKeys;
|
||||
}
|
||||
|
||||
await storage.put(jsonKey, Buffer.from(JSON.stringify(sample, null, 2)), "application/json");
|
||||
return { ok: true, key: jsonKey };
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
console.error("[shadow-capture] failed:", message);
|
||||
return { ok: false, error: message };
|
||||
}
|
||||
}
|
||||
|
||||
async function fetchImageData(url: string): Promise<Buffer | null> {
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), 30_000);
|
||||
try {
|
||||
const res = await fetch(url, { signal: controller.signal });
|
||||
clearTimeout(timer);
|
||||
if (!res.ok) return null;
|
||||
return Buffer.from(await res.arrayBuffer());
|
||||
} catch {
|
||||
return null;
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
@@ -6,6 +6,7 @@ import type { WorkerContext } from "../context.js";
|
||||
import { getLocaleContext } from "../locale.js";
|
||||
import type { LocaleContext } from "@app/shared-types";
|
||||
import { recordAiUsage } from "../lib/ai-usage.js";
|
||||
import { captureTrainingSample, type CaptureConsentFlags } from "../lib/shadow-capture.js";
|
||||
|
||||
/**
|
||||
* Bild-/OCR-jobb (spec §54): hämtar scan_job, anropar AAMOS med kontraktvaliderad
|
||||
@@ -28,7 +29,7 @@ export async function processScanJob(ctx: WorkerContext, scanJobId: string): Pro
|
||||
|
||||
const imageUrls = job.s3Keys.map((k) => ctx.readUrl(k));
|
||||
const localeContext = await getLocaleContext(ctx, job.userId);
|
||||
const consents = await loadConsentFlags(ctx, job.userId);
|
||||
const consentFlags = await loadConsentFlags(ctx, job.userId);
|
||||
|
||||
const started = Date.now();
|
||||
const result = await runAamosForJob(
|
||||
@@ -38,6 +39,7 @@ export async function processScanJob(ctx: WorkerContext, scanJobId: string): Pro
|
||||
imageUrls,
|
||||
job.context,
|
||||
localeContext,
|
||||
consentFlags,
|
||||
);
|
||||
|
||||
if (result.status === "failed" || result.output == null) {
|
||||
@@ -80,6 +82,27 @@ export async function processScanJob(ctx: WorkerContext, scanJobId: string): Pro
|
||||
|
||||
await recordScanCompleted(ctx, job, result);
|
||||
|
||||
// Shadow-capture: spara träningspar för framtida AAMOS-distillation (FAS 1).
|
||||
// Kör aldrig synkront på användarens kritiska väg; fel swallås.
|
||||
if (result.status === "ok" && result.output != null) {
|
||||
const capture = await captureTrainingSample({
|
||||
taskType: job.jobType as AamosTaskType,
|
||||
inputS3Keys: job.s3Keys,
|
||||
output: result.output as Record<string, unknown>,
|
||||
modelVersion: result.modelVersion,
|
||||
promptVersion: result.promptVersion,
|
||||
latencyMs: result.latencyMs,
|
||||
costUsd: result.costUsd,
|
||||
inputTokens: result.inputTokens,
|
||||
outputTokens: result.outputTokens,
|
||||
consentFlags,
|
||||
readUrl: ctx.readUrl,
|
||||
});
|
||||
if (!capture.ok) {
|
||||
console.error("[scan processor] shadow-capture misslyckades:", capture.error);
|
||||
}
|
||||
}
|
||||
|
||||
// Bokför verklig AI-kostnad/tokens utan PII (spec §45).
|
||||
if (result.costUsd != null || result.inputTokens || result.outputTokens) {
|
||||
await recordAiUsage(ctx.db, job.userId, {
|
||||
@@ -107,7 +130,6 @@ export async function processScanJob(ctx: WorkerContext, scanJobId: string): Pro
|
||||
},
|
||||
});
|
||||
}
|
||||
void consents;
|
||||
}
|
||||
|
||||
async function runAamosForJob(
|
||||
@@ -117,6 +139,7 @@ async function runAamosForJob(
|
||||
imageUrls: string[],
|
||||
context: unknown,
|
||||
localeContext: LocaleContext,
|
||||
consentFlags: CaptureConsentFlags,
|
||||
) {
|
||||
switch (jobType) {
|
||||
case "ANALYZE_FRIDGE_IMAGE":
|
||||
@@ -129,33 +152,33 @@ async function runAamosForJob(
|
||||
marketLocale: localeContext.languageTag,
|
||||
knownItems: [],
|
||||
},
|
||||
{ localeContext },
|
||||
{ localeContext, consentFlags },
|
||||
);
|
||||
case "ANALYZE_MEAL_IMAGE": {
|
||||
const recipeContext = await buildRecipeContext(ctx, context);
|
||||
return ctx.aamos.runTask(
|
||||
"ANALYZE_MEAL_IMAGE",
|
||||
{ imageUrls, recipeContext, marketLocale: localeContext.languageTag },
|
||||
{ localeContext },
|
||||
{ localeContext, consentFlags },
|
||||
);
|
||||
}
|
||||
case "READ_RECEIPT":
|
||||
return ctx.aamos.runTask(
|
||||
"READ_RECEIPT",
|
||||
{ imageUrls, marketLocale: localeContext.languageTag },
|
||||
{ localeContext },
|
||||
{ localeContext, consentFlags },
|
||||
);
|
||||
case "READ_NUTRITION_LABEL":
|
||||
return ctx.aamos.runTask(
|
||||
"READ_NUTRITION_LABEL",
|
||||
{ imageUrls, marketLocale: localeContext.languageTag },
|
||||
{ localeContext },
|
||||
{ localeContext, consentFlags },
|
||||
);
|
||||
case "READ_EXPIRY_DATE":
|
||||
return ctx.aamos.runTask(
|
||||
"READ_EXPIRY_DATE",
|
||||
{ imageUrls: imageUrls.slice(0, 2) },
|
||||
{ localeContext },
|
||||
{ localeContext, consentFlags },
|
||||
);
|
||||
default:
|
||||
throw new Error(`Jobbtypen ${jobType} hanteras inte av scan-processorn`);
|
||||
|
||||
Reference in New Issue
Block a user