From c2cc3878dd55bf85244a278408cdc5c94dd3b9cf Mon Sep 17 00:00:00 2001 From: "Sven (AAMOS AI)" Date: Sat, 8 Aug 2026 04:00:49 +0700 Subject: [PATCH] =?UTF-8?q?fix(skiva-1):=20spara=20accept,=20bildref,=20lo?= =?UTF-8?q?kal=20tr=C3=A4ningsbank,=20l=C3=A4rande-loop-dok?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - confirm-loop sparar nu även action=accept som positivt exempel i ai_corrections - imageS3Key sparas vid image_training-samtycke, annars null - ai_corrections.proposal lagrar det specifika AI-förslaget per item - BUILD_TRAINING_SAMPLE bankar lokalt till ai_training_bank (ej externt runTask) - Jobbet kastar aldrig i AAMOS_MODE=gemini - Migration 0020: ai_corrections.image_s3_key/proposal + ai_training_bank - docs/28-lärande-loop.md: datakontrakt, samtycke, retention, GDPR-radering - Tester: accept + bildref (ja/nej) + lokal bank i gemini-läge - REQUIRE_REAL=1 är AI-fokuserat i gemini-läge; staging mail/S3 får vara mock --- apps/api/src/routes/scans.ts | 87 +++++-- apps/api/test/scans.test.ts | 238 ++++++++++++++++++ apps/worker/src/processors/maintenance.ts | 52 ++-- apps/worker/test/setup-env.ts | 12 + apps/worker/test/training-export.test.ts | 100 ++++++++ docs/28-lärande-loop.md | 122 +++++++++ .../0020_ai_corrections_training_bank.sql | 27 ++ infrastructure/migrations/meta/_journal.json | 7 + packages/database/src/schema/scans.ts | 39 +++ turbo.json | 5 + 10 files changed, 646 insertions(+), 43 deletions(-) create mode 100644 apps/api/test/scans.test.ts create mode 100644 apps/worker/test/setup-env.ts create mode 100644 apps/worker/test/training-export.test.ts create mode 100644 docs/28-lärande-loop.md create mode 100644 infrastructure/migrations/0020_ai_corrections_training_bank.sql diff --git a/apps/api/src/routes/scans.ts b/apps/api/src/routes/scans.ts index 25b986c..970e90a 100644 --- a/apps/api/src/routes/scans.ts +++ b/apps/api/src/routes/scans.ts @@ -129,17 +129,12 @@ export async function scanRoutes(app: FastifyInstance) { input.storageLocationId ?? (await defaultLocation(app, householdId, job.scanType)); const created: string[] = []; + const proposals = extractProposals(job); for (const item of input.items) { - if (item.action === "reject") { - await recordCorrection(app, job, item.tempId ?? null, { action: "reject" }); - continue; - } - if (item.action === "edit" || item.action === "add") { - await recordCorrection(app, job, item.tempId ?? null, { - action: item.action, - corrected: { name: item.displayName, quantity: item.quantity, unit: item.unit }, - }); - } + const proposal = findProposal(proposals, item.tempId); + await recordCorrection(app, job, item, proposal); + if (item.action === "reject") continue; + const locationId = item.storageLocationId ?? fallbackLocation; if (!locationId) throw errors.badRequest("storageLocationId saknas och ingen standardplats finns."); @@ -334,6 +329,39 @@ async function defaultLocation(app: FastifyInstance, householdId: string, scanTy return loc?.id ?? null; } +type ProposalItem = { + tempId?: string; + detectedName?: string; + canonicalIngredientId?: string | null; + brand?: string | null; + estimatedQuantity?: number | null; + unit?: string | null; + bestBeforeDate?: string | null; + confidence?: number; + requiresConfirmation?: boolean; +}; + +function extractProposals(job: { result: unknown }): ProposalItem[] { + const result = job.result as Record | null; + if (!result || !Array.isArray(result.items)) return []; + return result.items.map((it, idx) => ({ + tempId: String(it.tempId ?? idx), + detectedName: String(it.detectedName ?? ""), + canonicalIngredientId: it.canonicalIngredientId ?? null, + brand: it.brand ?? null, + estimatedQuantity: it.estimatedQuantity ?? null, + unit: it.unit ?? null, + bestBeforeDate: it.bestBeforeDate ?? null, + confidence: typeof it.confidence === "number" ? it.confidence : null, + requiresConfirmation: typeof it.requiresConfirmation === "boolean" ? it.requiresConfirmation : null, + })); +} + +function findProposal(proposals: ProposalItem[], tempId: string | undefined): ProposalItem | null { + if (tempId == null) return null; + return proposals.find((p) => p.tempId === tempId) ?? null; +} + async function recordCorrection( app: FastifyInstance, job: { @@ -341,33 +369,62 @@ async function recordCorrection( userId: string; jobType: string; result: unknown; + s3Keys: string[]; modelVersion: string | null; promptVersion: string | null; }, - tempId: string | null, - correction: Record, + item: { + tempId?: string; + action: "accept" | "edit" | "reject" | "add"; + displayName: string; + quantity: number; + unit: string; + brand?: string; + canonicalIngredientId?: string; + bestBeforeDate?: string; + useByDate?: string; + }, + proposal: ProposalItem | null, ) { const consents = await app.db .select() .from(schema.userConsents) .where(eq(schema.userConsents.userId, job.userId)); const snapshot = Object.fromEntries(consents.map((c) => [c.kind, c.status])); + const hasImageConsent = snapshot.image_training === "granted"; + + const userCorrection: Record = { action: item.action }; + if (item.action !== "reject") { + userCorrection.corrected = { + displayName: item.displayName, + canonicalIngredientId: item.canonicalIngredientId ?? null, + brand: item.brand ?? null, + quantity: item.quantity, + unit: item.unit, + bestBeforeDate: item.bestBeforeDate ?? null, + useByDate: item.useByDate ?? null, + }; + } + await app.db.insert(schema.aiCorrections).values({ scanJobId: job.id, userId: job.userId, taskType: job.jobType, - aiOutput: { tempId, raw: job.result }, - userCorrection: correction, + aiOutput: { raw: job.result }, + proposal: proposal, + userCorrection, + imageS3Key: hasImageConsent && job.s3Keys.length > 0 ? job.s3Keys[0] : null, modelVersion: job.modelVersion, promptVersion: job.promptVersion, consentSnapshot: snapshot, }); + await emitEvent(app.db, { type: "AI_CORRECTED", payload: { scanJobId: job.id, taskType: job.jobType, - field: String(correction.action ?? "unknown"), + field: item.action, }, userId: job.userId, }); diff --git a/apps/api/test/scans.test.ts b/apps/api/test/scans.test.ts new file mode 100644 index 0000000..7e79ea5 --- /dev/null +++ b/apps/api/test/scans.test.ts @@ -0,0 +1,238 @@ +import "./setup-env.js"; +import { describe, expect, it, beforeAll, afterAll } from "vitest"; +import { eq, inArray } from "drizzle-orm"; +import { buildServer } from "../src/server.js"; +import { loadConfig } from "../src/config.js"; +import { createDatabase, closeDatabase, schema } from "@app/database"; + +describe("scan confirmation → ai_corrections", () => { + const testDb = createDatabase(process.env.TEST_DATABASE_URL!); + const config = loadConfig(); + let app: Awaited>; + let token: string; + let userId: string; + let householdId: string; + let locationId: string; + const email = "scan-confirm-test@example.invalid"; + + async function cleanup() { + const existing = await testDb.db + .select({ id: schema.users.id }) + .from(schema.users) + .where(inArray(schema.users.email, [email])); + for (const u of existing) { + await testDb.db.delete(schema.aiCorrections).where(eq(schema.aiCorrections.userId, u.id)); + await testDb.db.delete(schema.idempotencyKeys).where(eq(schema.idempotencyKeys.userId, u.id)); + const memberships = await testDb.db + .select({ householdId: schema.householdMembers.householdId }) + .from(schema.householdMembers) + .where(eq(schema.householdMembers.userId, u.id)); + for (const m of memberships) { + const items = await testDb.db + .select({ id: schema.inventoryItems.id }) + .from(schema.inventoryItems) + .where(eq(schema.inventoryItems.householdId, m.householdId)); + for (const it of items) { + await testDb.db.delete(schema.inventoryTransactions).where(eq(schema.inventoryTransactions.inventoryItemId, it.id)); + await testDb.db.delete(schema.inventoryConflicts).where(eq(schema.inventoryConflicts.inventoryItemId, it.id)); + } + await testDb.db.delete(schema.inventoryItems).where(eq(schema.inventoryItems.householdId, m.householdId)); + await testDb.db.delete(schema.inventoryConflicts).where(eq(schema.inventoryConflicts.householdId, m.householdId)); + await testDb.db.delete(schema.storageLocations).where(eq(schema.storageLocations.householdId, m.householdId)); + await testDb.db.delete(schema.householdMembers).where(eq(schema.householdMembers.householdId, m.householdId)); + await testDb.db.delete(schema.households).where(eq(schema.households.id, m.householdId)); + } + await testDb.db.delete(schema.scanJobs).where(eq(schema.scanJobs.userId, u.id)); + await testDb.db.delete(schema.userConsents).where(eq(schema.userConsents.userId, u.id)); + await testDb.db.delete(schema.userPreferences).where(eq(schema.userPreferences.userId, u.id)); + await testDb.db.delete(schema.users).where(eq(schema.users.id, u.id)); + } + } + + async function setConsent(imageTraining: boolean) { + const kinds = ["personalization", "anonymized_improvement"] as const; + for (const kind of kinds) { + await testDb.db + .insert(schema.userConsents) + .values({ userId, kind, status: "granted" as const }) + .onConflictDoUpdate({ + target: [schema.userConsents.userId, schema.userConsents.kind], + set: { status: "granted" as const }, + }); + } + await testDb.db + .insert(schema.userConsents) + .values({ userId, kind: "image_training" as const, status: (imageTraining ? "granted" : "denied") as "granted" | "denied" }) + .onConflictDoUpdate({ + target: [schema.userConsents.userId, schema.userConsents.kind], + set: { status: (imageTraining ? "granted" : "denied") as "granted" | "denied" }, + }); + } + + async function createScanJob() { + const [job] = await testDb.db + .insert(schema.scanJobs) + .values({ + userId, + householdId, + scanType: "fridge", + jobType: "ANALYZE_FRIDGE_IMAGE", + status: "awaiting_confirmation", + s3Keys: ["fridge-scans/test-image.jpg"], + result: { + items: [ + { + tempId: "item-1", + detectedName: "Mellanmjölk", + canonicalIngredientId: "milk_1_5", + brand: "Arla", + estimatedQuantity: 1, + unit: "LITER", + confidence: 0.98, + requiresConfirmation: false, + }, + ], + }, + modelVersion: "gemini-2.5-flash", + promptVersion: "gemini-fridge-v1", + }) + .returning(); + return job!.id; + } + + beforeAll(async () => { + await cleanup(); + app = await buildServer(config); + await app.ready(); + + const res = await app.inject({ + method: "POST", + url: "/v1/auth/register", + payload: { email, password: "Password123!", displayName: "Scan Confirm Test" }, + }); + const body = JSON.parse(res.body) as { accessToken: string }; + token = body.accessToken; + userId = (JSON.parse(atob(token.split(".")[1]!)) as { sub: string }).sub; + + const quick = await app.inject({ + method: "POST", + url: "/v1/onboarding/quick-start", + headers: { authorization: `Bearer ${token}` }, + payload: { goals: ["less_waste"], precisionMode: "simple" }, + }); + householdId = (JSON.parse(quick.body) as { householdId: string }).householdId; + + const [location] = await testDb.db + .select({ id: schema.storageLocations.id }) + .from(schema.storageLocations) + .where(eq(schema.storageLocations.householdId, householdId)) + .limit(1); + locationId = location!.id; + }); + + afterAll(async () => { + await cleanup(); + await closeDatabase(); + await app.close(); + }); + + it("accept action writes a positive row in ai_corrections", async () => { + await setConsent(false); + const scanJobId = await createScanJob(); + + const res = await app.inject({ + method: "POST", + url: `/v1/scans/${scanJobId}/confirm`, + headers: { authorization: `Bearer ${token}` }, + payload: { + items: [ + { + tempId: "item-1", + action: "accept", + displayName: "Mellanmjölk", + canonicalIngredientId: "milk_1_5", + brand: "Arla", + quantity: 1, + unit: "LITER", + storageLocationId: locationId, + }, + ], + }, + }); + expect(res.statusCode).toBe(200); + + const corrections = await testDb.db + .select() + .from(schema.aiCorrections) + .where(eq(schema.aiCorrections.scanJobId, scanJobId)); + expect(corrections).toHaveLength(1); + expect((corrections[0]!.userCorrection as Record).action).toBe("accept"); + expect((corrections[0]!.proposal as Record).detectedName).toBe("Mellanmjölk"); + expect((corrections[0]!.userCorrection as Record>).corrected).toMatchObject({ + displayName: "Mellanmjölk", + quantity: 1, + unit: "LITER", + }); + }); + + it("saves image reference when image_training consent is granted", async () => { + await setConsent(true); + const scanJobId = await createScanJob(); + + const res = await app.inject({ + method: "POST", + url: `/v1/scans/${scanJobId}/confirm`, + headers: { authorization: `Bearer ${token}` }, + payload: { + items: [ + { + tempId: "item-1", + action: "accept", + displayName: "Mellanmjölk", + quantity: 1, + unit: "LITER", + storageLocationId: locationId, + }, + ], + }, + }); + expect(res.statusCode).toBe(200); + + const corrections = await testDb.db + .select() + .from(schema.aiCorrections) + .where(eq(schema.aiCorrections.scanJobId, scanJobId)); + expect(corrections[0]!.imageS3Key).toBe("fridge-scans/test-image.jpg"); + expect((corrections[0]!.consentSnapshot as Record).image_training).toBe("granted"); + }); + + it("does not save image reference when image_training consent is denied", async () => { + await setConsent(false); + const scanJobId = await createScanJob(); + + const res = await app.inject({ + method: "POST", + url: `/v1/scans/${scanJobId}/confirm`, + headers: { authorization: `Bearer ${token}` }, + payload: { + items: [ + { + tempId: "item-1", + action: "accept", + displayName: "Mellanmjölk", + quantity: 1, + unit: "LITER", + storageLocationId: locationId, + }, + ], + }, + }); + expect(res.statusCode).toBe(200); + + const corrections = await testDb.db + .select() + .from(schema.aiCorrections) + .where(eq(schema.aiCorrections.scanJobId, scanJobId)); + expect(corrections[0]!.imageS3Key).toBeNull(); + }); +}); diff --git a/apps/worker/src/processors/maintenance.ts b/apps/worker/src/processors/maintenance.ts index b798ed0..5d77c9c 100644 --- a/apps/worker/src/processors/maintenance.ts +++ b/apps/worker/src/processors/maintenance.ts @@ -333,7 +333,10 @@ export async function processMemorySync(ctx: WorkerContext): Promise { return updates; } -/** BUILD_TRAINING_SAMPLE (spec §33): exportera korrigeringar MED samtycke till AAMOS. */ +/** BUILD_TRAINING_SAMPLE (spec §33): bank eligible corrections to an app-owned, + * versioned training dataset. In Skiva 1 this is local storage, not an external + * AAMOS/Gemini runTask call, so the job never throws when AAMOS_MODE=gemini. + */ export async function processTrainingExport(ctx: WorkerContext): Promise { const corrections = await ctx.db .select() @@ -348,37 +351,30 @@ export async function processTrainingExport(ctx: WorkerContext): Promise if (eligible.length === 0) return 0; - const result = await ctx.aamos.runTask( - "EXPORT_TRAINING_SAMPLE", - { - marketLocale: "sv-SE", - samples: eligible.map((c) => ({ - taskType: c.taskType, - aiOutput: c.aiOutput as Record, - userCorrection: c.userCorrection as Record, - modelVersion: c.modelVersion ?? null, - promptVersion: c.promptVersion ?? null, - })), - }, - { - correlationId: `training-export-${Date.now()}`, - consentFlags: { - personalization: false, - anonymizedImprovement: true, - imageTraining: false, - }, - }, - ); + const version = "v1"; + const batchId = `cibello-local-${version}-${Date.now()}`; + const exportedAt = new Date(); - if (result.status !== "ok" || !result.output) { - throw new Error(`AAMOS training export failed: ${result.error ?? "unknown"}`); - } - - const batchId = result.output.batchId; for (const correction of eligible) { + const userCorrection = correction.userCorrection as Record; + await ctx.db.insert(schema.aiTrainingBank).values({ + correctionId: correction.id, + scanJobId: correction.scanJobId, + version, + taskType: correction.taskType, + imageS3Key: correction.imageS3Key, + proposal: (correction.proposal ?? {}) as Record, + action: String(userCorrection.action ?? "unknown"), + corrected: (userCorrection.corrected ?? null) as Record | null, + modelVersion: correction.modelVersion, + promptVersion: correction.promptVersion, + consentSnapshot: correction.consentSnapshot, + exportedAt, + }); + await ctx.db .update(schema.aiCorrections) - .set({ exportedToTraining: new Date(), trainingBatchId: batchId }) + .set({ exportedToTraining: exportedAt, trainingBatchId: batchId }) .where(eq(schema.aiCorrections.id, correction.id)); } diff --git a/apps/worker/test/setup-env.ts b/apps/worker/test/setup-env.ts new file mode 100644 index 0000000..6162443 --- /dev/null +++ b/apps/worker/test/setup-env.ts @@ -0,0 +1,12 @@ +/** + * Hermetic test environment for worker integration tests. + * Must run BEFORE any application module is imported. + */ +process.env.NODE_ENV = "test"; +process.env.AAMOS_MODE = "mock"; +process.env.EMAIL_MODE = "log"; +process.env.S3_MODE = "mock"; +process.env.LOG_LEVEL = "error"; + +process.env.TEST_DATABASE_URL ||= "postgres://app_user:app_dev_password@localhost:5432/cibello_test"; +process.env.DATABASE_URL = process.env.TEST_DATABASE_URL; diff --git a/apps/worker/test/training-export.test.ts b/apps/worker/test/training-export.test.ts new file mode 100644 index 0000000..36f20bb --- /dev/null +++ b/apps/worker/test/training-export.test.ts @@ -0,0 +1,100 @@ +import "./setup-env.js"; +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { eq } from "drizzle-orm"; +import { createDatabase, closeDatabase, schema } from "@app/database"; +import { processTrainingExport } from "../src/processors/maintenance.js"; + +describe("BUILD_TRAINING_SAMPLE banks locally", () => { + const testDb = createDatabase(process.env.TEST_DATABASE_URL!); + const email = "training-export-test@example.invalid"; + + async function cleanup() { + const existing = await testDb.db.select({ id: schema.users.id }).from(schema.users).where(eq(schema.users.email, email)); + for (const u of existing) { + await testDb.db.delete(schema.aiCorrections).where(eq(schema.aiCorrections.userId, u.id)); + await testDb.db.delete(schema.userConsents).where(eq(schema.userConsents.userId, u.id)); + await testDb.db.delete(schema.scanJobs).where(eq(schema.scanJobs.userId, u.id)); + await testDb.db.delete(schema.users).where(eq(schema.users.id, u.id)); + } + } + + beforeAll(async () => { + await cleanup(); + }); + + afterAll(async () => { + await cleanup(); + await closeDatabase(); + }); + + it("banks eligible corrections locally and does not throw in gemini mode", async () => { + const [user] = await testDb.db + .insert(schema.users) + .values({ + email: "training-export-test@example.invalid", + passwordHash: "not-used", + displayName: "Training Export Test", + }) + .returning(); + const userId = user!.id; + + await testDb.db.insert(schema.userConsents).values([ + { userId, kind: "anonymized_improvement", status: "granted" }, + { userId, kind: "image_training", status: "granted" }, + ]); + + const [job] = await testDb.db + .insert(schema.scanJobs) + .values({ + userId, + scanType: "fridge", + jobType: "ANALYZE_FRIDGE_IMAGE", + status: "completed", + s3Keys: ["fridge-scans/train.jpg"], + result: { items: [] }, + }) + .returning(); + + const [correction] = await testDb.db + .insert(schema.aiCorrections) + .values({ + scanJobId: job!.id, + userId, + taskType: "ANALYZE_FRIDGE_IMAGE", + aiOutput: { raw: { items: [] } }, + proposal: { detectedName: "Mellanmjölk", confidence: 0.98 }, + userCorrection: { action: "accept", corrected: { displayName: "Mellanmjölk", quantity: 1, unit: "LITER" } }, + imageS3Key: "fridge-scans/train.jpg", + modelVersion: "gemini-2.5-flash", + promptVersion: "gemini-fridge-v1", + consentSnapshot: { anonymized_improvement: "granted", image_training: "granted" }, + }) + .returning(); + + // Force gemini mode in environment so we prove the job does not call AAMOS. + const previousMode = process.env.AAMOS_MODE; + process.env.AAMOS_MODE = "gemini"; + + const exported = await processTrainingExport({ db: testDb.db } as never); + + process.env.AAMOS_MODE = previousMode; + + expect(exported).toBe(1); + + const banked = await testDb.db + .select() + .from(schema.aiTrainingBank) + .where(eq(schema.aiTrainingBank.correctionId, correction!.id)); + expect(banked).toHaveLength(1); + expect(banked[0]!.version).toBe("v1"); + expect(banked[0]!.imageS3Key).toBe("fridge-scans/train.jpg"); + expect(banked[0]!.action).toBe("accept"); + + const updated = await testDb.db + .select() + .from(schema.aiCorrections) + .where(eq(schema.aiCorrections.id, correction!.id)); + expect(updated[0]!.exportedToTraining).not.toBeNull(); + expect(updated[0]!.trainingBatchId).toMatch(/^cibello-local-v1-/); + }); +}); diff --git a/docs/28-lärande-loop.md b/docs/28-lärande-loop.md new file mode 100644 index 0000000..b9e43da --- /dev/null +++ b/docs/28-lärande-loop.md @@ -0,0 +1,122 @@ +# Del 28 – Lärande-loop (ai_corrections → ai_training_bank) + +> Ingen AI är facit. Varje skanning där användaren granskar förslag blir ett +> träningsexempel — om hon samtycker. Detta dokument beskriver datakontraktet, +> samtyckesgätning och GDPR-radering. + +## Översikt + +``` +App → skanna → worker → AI-förslag → app (awaiting_confirmation) + ↓ + användaren granskar varje item + ↓ + POST /v1/scans/:id/confirm + ↓ + ai_corrections (ett rad per item) + ↓ + BUILD_TRAINING_SAMPLE (scheduler, 1×/vecka) + ↓ + ai_training_bank (cibello-ägt dataset) +``` + +`ai_training_bank` är den consenteda platsen för rikare data +(bildreferens, förslag, korrigering). Analytics får aldrig innehålla PII +eller råa bilder (se §56/§58). + +## Datakontrakt per skanning + +Varje rad i `ai_corrections` representerar **ett item** från en skanning: + +| Fält | Innehåll | +|---|---| +| `scanJobId` | Källskanningen (`scan_jobs.id`). | +| `taskType` | T.ex. `ANALYZE_FRIDGE_IMAGE`, `READ_RECEIPT`. | +| `aiOutput` | Hela AI-raw-resultatet från `scan_jobs.result`. | +| `proposal` | Det specifika AI-förslag item:et kom från (`detectedName`, `canonicalIngredientId`, `brand`, `estimatedQuantity`, `unit`, `bestBeforeDate`, `confidence`, `requiresConfirmation`). | +| `userCorrection` | `{ action: "accept" \| "edit" \| "reject" \| "add", corrected?: {...} }` | +| `corrected` (inbäddad) | Användarens slutgiltiga värden vid accept/edit/add: `displayName`, `canonicalIngredientId`, `brand`, `quantity`, `unit`, `bestBeforeDate`, `useByDate`. | +| `imageS3Key` | Första lagrade bildnyckeln från skanningen, **endast om** `image_training`-samtycke fanns vid bekräftelsen. Annars `null`. | +| `modelVersion` / `promptVersion` | Vilken modell och prompt som producerade förslaget. | +| `consentSnapshot` | `{ anonymized_improvement: "granted"\|"denied", image_training: "granted"\|"denied", ... }` som JSON vid bekräftelsetillfället. | +| `createdAt` | Tidsstämpel för bekräftelsen. | + +### Åtgärder som sparas + +- **`accept`** — positivt exempel. AI-förslaget var korrekt nog att användaren accepterade det oförändrat. +- **`edit`** — användaren ändrade något (namn, kvantitet, enhet, datum …). +- **`add`** — AI missade item:et helt; användaren lade till det manuellt. +- **`reject`** — AI hittade något som inte finns; användaren kastade det. + +Alla fyra åtgärder sparas. Bara accept/edit/add leder till att ett +`inventory_items`-rad skapas; reject gör det inte. + +## ai_training_bank + +När `BUILD_TRAINING_SAMPLE` kör (veckoschema i worker) bankas rader med +`anonymized_improvement = granted` till `ai_training_bank`: + +| Fält | Innehåll | +|---|---| +| `correctionId` | Referens till `ai_corrections.id` (cascade delete). | +| `scanJobId` | Källskanningen. | +| `version` | Dataset-version, t.ex. `v1`. Bumpar när formatet ändras. | +| `taskType` | Samma som källan. | +| `imageS3Key` | Kopierad från `ai_corrections.image_s3_key` (kan vara `null`). | +| `proposal` | AI-förslaget för just det item:et. | +| `action` | Användarens åtgärd. | +| `corrected` | Slutgiltiga värden, eller `null` vid reject. | +| `modelVersion` / `promptVersion` | Spårbarhet till modell/prompt. | +| `consentSnapshot` | Kopia av samtyckesläget. | +| `exportedAt` | När raden bankades. | + +Banken ägs av cibello och är versionerad. Ingen extern leverantör +anropas under exporten — jobbet får aldrig kasta på grund av att +`AAMOS_MODE=gemini` saknar `EXPORT_TRAINING_SAMPLE`-stöd. + +## Samtycke + +Två separata samtycken styr vad som sparas och var: + +1. **`anonymized_improvement`** — krävs för att överhuvudtaget banka till + `ai_training_bank`. Utan detta lämnas `ai_corrections` kvar men raderna + exporteras inte. +2. **`image_training`** — krävs för att `imageS3Key` ska sparas. Utan + samtycke sparas endast textparet (`proposal`, `corrected`) och + `imageS3Key` är `null`. + +Samtyckessnapshoten sparas per rad så att framtida ändringar av +användarens samtycke inte påverkar redan bankade data. + +## Retention + +- `ai_corrections`: behålls så länge användarkontot finns. Underlättar + support och debugging. +- `ai_training_bank`: behålls så länge användarkontot finns, om inte + användaren återkallar samtycke — då raderas endast rader där + `consentSnapshot.anonymized_improvement = "denied"` (i praktiken + exporteras de aldrig). +- Bilder i lagring: följer samma regler som `imageS3Key` — sparas så + länge kontot finns, raderas vid kontoradering. + +## GDPR / kontoradering + +Vid kontoradering (eller rätten att bli glömd): + +- `users` → cascade delete → `ai_corrections` försvinner (FK `ON DELETE CASCADE`). +- `ai_corrections` → cascade delete → `ai_training_bank` försvinner (FK `ON DELETE CASCADE`). +- Bilder som refereras av `ai_corrections.imageS3Key` och + `ai_training_bank.imageS3Key` måste raderas från lagring. Detta görs av + en GDPR-raderingsprocessor (se Del 12) som läser bildnycklarna innan + användarposten tas bort. + +Verifiera alltid att kontoraderingstestet kontrollerar både +`ai_corrections`, `ai_training_bank` och att inga överblivna bildnycklar +finns kvar i S3/mock-lagringen. + +## Inget PII i analytics + +Träningsdatan (rikare bild+förslag+korrigering) finns endast i +`ai_corrections`/`ai_training_bank` under samtycke. Analytics-events som +`AI_CORRECTED` innehåller endast `scanJobId`, `taskType` och `field` +(åtgärd), aldrig bilder, namn eller detaljerade värden. diff --git a/infrastructure/migrations/0020_ai_corrections_training_bank.sql b/infrastructure/migrations/0020_ai_corrections_training_bank.sql new file mode 100644 index 0000000..2d04ef7 --- /dev/null +++ b/infrastructure/migrations/0020_ai_corrections_training_bank.sql @@ -0,0 +1,27 @@ +-- Extend ai_corrections with image reference and add a local, versioned training bank. +-- BUILD_TRAINING_SAMPLE banks here instead of calling external AAMOS/Gemini runTask. + +ALTER TABLE ai_corrections + ADD COLUMN image_s3_key TEXT, + ADD COLUMN proposal JSONB; + +CREATE TABLE ai_training_bank ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + correction_id UUID NOT NULL REFERENCES ai_corrections(id) ON DELETE CASCADE, + scan_job_id UUID REFERENCES scan_jobs(id) ON DELETE SET NULL, + version TEXT NOT NULL DEFAULT 'v1', + task_type TEXT NOT NULL, + image_s3_key TEXT, + proposal JSONB NOT NULL, + action TEXT NOT NULL, + corrected JSONB, + model_version TEXT, + prompt_version TEXT, + consent_snapshot JSONB NOT NULL DEFAULT '{}', + exported_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW() +); + +CREATE INDEX ai_training_bank_version_idx ON ai_training_bank(version, task_type); +CREATE INDEX ai_training_bank_scan_job_idx ON ai_training_bank(scan_job_id); +CREATE INDEX ai_training_bank_created_at_idx ON ai_training_bank(created_at); diff --git a/infrastructure/migrations/meta/_journal.json b/infrastructure/migrations/meta/_journal.json index 1bbe277..7773279 100644 --- a/infrastructure/migrations/meta/_journal.json +++ b/infrastructure/migrations/meta/_journal.json @@ -134,6 +134,13 @@ "when": 1786141200000, "tag": "0019_ai_usage_cost_usd", "breakpoints": true + }, + { + "idx": 19, + "version": "7", + "when": 1786144800000, + "tag": "0020_ai_corrections_training_bank", + "breakpoints": true } ] } \ No newline at end of file diff --git a/packages/database/src/schema/scans.ts b/packages/database/src/schema/scans.ts index afb3e9a..98ea94a 100644 --- a/packages/database/src/schema/scans.ts +++ b/packages/database/src/schema/scans.ts @@ -6,6 +6,7 @@ import { pgTable, text, timestamp, + uniqueIndex, uuid, } from "drizzle-orm/pg-core"; import { createdAt, jobStatusEnum, jobTypeEnum, scanTypeEnum, updatedAt } from "./_shared.js"; @@ -62,6 +63,10 @@ export const aiCorrections = pgTable( taskType: text("task_type").notNull(), aiOutput: jsonb("ai_output").notNull(), userCorrection: jsonb("user_correction").notNull(), + /** The specific AI proposal item this correction refers to. */ + proposal: jsonb("proposal"), + /** First stored image key when image_training consent granted, otherwise null. */ + imageS3Key: text("image_s3_key"), modelVersion: text("model_version"), promptVersion: text("prompt_version"), /** Snapshot av samtyckesläget när korrigeringen skapades. */ @@ -72,3 +77,37 @@ export const aiCorrections = pgTable( }, (t) => [index("ai_corrections_task_idx").on(t.taskType, t.createdAt)], ); + +/** + * App-owned, versioned training dataset (spec §33 + Skiva 1 fixrunda). + * BUILD_TRAINING_SAMPLE banks eligible ai_corrections here instead of + * calling external AAMOS/Gemini runTask. The bank is the consented place + * for richer (image, proposal, correction) data. + */ +export const aiTrainingBank = pgTable( + "ai_training_bank", + { + id: uuid("id").primaryKey().defaultRandom(), + correctionId: uuid("correction_id") + .notNull() + .references(() => aiCorrections.id, { onDelete: "cascade" }), + scanJobId: uuid("scan_job_id").references(() => scanJobs.id, { onDelete: "set null" }), + version: text("version").notNull().default("v1"), + taskType: text("task_type").notNull(), + imageS3Key: text("image_s3_key"), + proposal: jsonb("proposal").notNull(), + action: text("action").notNull(), + corrected: jsonb("corrected"), + modelVersion: text("model_version"), + promptVersion: text("prompt_version"), + consentSnapshot: jsonb("consent_snapshot").notNull().default({}), + exportedAt: timestamp("exported_at", { withTimezone: true }).notNull().defaultNow(), + createdAt: createdAt(), + }, + (t) => [ + uniqueIndex("ai_training_bank_correction_unique").on(t.correctionId), + index("ai_training_bank_version_idx").on(t.version, t.taskType), + index("ai_training_bank_scan_job_idx").on(t.scanJobId), + index("ai_training_bank_created_at_idx").on(t.createdAt), + ], +); diff --git a/turbo.json b/turbo.json index e4f6c8f..bf06e91 100644 --- a/turbo.json +++ b/turbo.json @@ -31,6 +31,11 @@ "outputs": [], "env": ["TEST_DATABASE_URL"] }, + "@app/worker#test": { + "dependsOn": ["@app/database#db:test-setup"], + "outputs": [], + "env": ["TEST_DATABASE_URL"] + }, "dev": { "cache": false, "persistent": true