feat(gemini): Skiva 1 – Gemini 2.5 Flash som lärar-tier för kylskåpsskanning

- Gemini-adapter bakom AamosClient-interface (AAMOS_MODE=gemini)
- Serversida/worker: hämtar bild, anropar Gemini, mappar mot canonical_ingredients
- Kostnad/tokens bokförs i ai_usage_counters; global dagsbudget via BudgetStore
- Redis-backed budget i worker, in-memory i tester
- Migration 0019: ai_cost_usd_microcents
- Hermetiska tester med inspelad fixture; separat pnpm eval:scan
- docs/09 uppdaterad ärligt: AAMOS-status, Gemini-flöde, säkerhet/kostnad
- REQUIRE_REAL=1 stödjer AAMOS_MODE=gemini; deploy-grind uppdaterad
This commit is contained in:
Sven (AAMOS AI)
2026-08-08 03:23:44 +07:00
parent 7894a1ec06
commit 050c958285
18 changed files with 1055 additions and 86 deletions
+7 -3
View File
@@ -38,12 +38,16 @@ S3_ENDPOINT=
S3_ACCESS_KEY_ID= S3_ACCESS_KEY_ID=
S3_SECRET_ACCESS_KEY= S3_SECRET_ACCESS_KEY=
# --- AAMOS (befintlig AI-plattform; nås ENDAST från backend, spec §31, §61.13) --- # --- AAMOS / Gemini-lärar-tier (nås ENDAST från backend, spec §31, §61.13) ---
# AAMOS_MODE=http i alla riktiga miljöer. mock finns för lokal utveckling/test utan nätverk. # AAMOS_MODE: http = befintlig AAMOS, gemini = Gemini-lärar-tier, mock = lokal dev/test.
AAMOS_MODE=http AAMOS_MODE=mock
AAMOS_API_URL= AAMOS_API_URL=
AAMOS_API_KEY= AAMOS_API_KEY=
AAMOS_TIMEOUT_MS=60000 AAMOS_TIMEOUT_MS=60000
GEMINI_API_KEY=
GEMINI_MODEL=gemini-2.5-flash
GEMINI_TIMEOUT_MS=60000
GEMINI_DAILY_BUDGET_USD=10
# --- Prenumerationer (spec §4547) --- # --- Prenumerationer (spec §4547) ---
APPLE_BUNDLE_ID= # default ur brand.config.json APPLE_BUNDLE_ID= # default ur brand.config.json
+10 -1
View File
@@ -42,11 +42,16 @@ const configSchema = z.object({
S3_ACCESS_KEY_ID: z.string().default(""), S3_ACCESS_KEY_ID: z.string().default(""),
S3_SECRET_ACCESS_KEY: z.string().default(""), S3_SECRET_ACCESS_KEY: z.string().default(""),
AAMOS_MODE: z.enum(["http", "mock"]).default("http"), AAMOS_MODE: z.enum(["http", "mock", "gemini"]).default("http"),
AAMOS_API_URL: z.string().optional(), AAMOS_API_URL: z.string().optional(),
AAMOS_API_KEY: z.string().optional(), AAMOS_API_KEY: z.string().optional(),
AAMOS_TIMEOUT_MS: z.coerce.number().int().default(60_000), AAMOS_TIMEOUT_MS: z.coerce.number().int().default(60_000),
GEMINI_API_KEY: z.string().optional(),
GEMINI_MODEL: z.string().default("gemini-2.5-flash"),
GEMINI_TIMEOUT_MS: z.coerce.number().int().default(60_000),
GEMINI_DAILY_BUDGET_USD: z.coerce.number().default(0),
APP_STORE_MODE: z.enum(["production", "sandbox"]).default("sandbox"), APP_STORE_MODE: z.enum(["production", "sandbox"]).default("sandbox"),
EMAIL_MODE: z.enum(["log", "smtp"]).default("log"), EMAIL_MODE: z.enum(["log", "smtp"]).default("log"),
SMTP_HOST: z.string().default(""), SMTP_HOST: z.string().default(""),
@@ -89,6 +94,10 @@ export function loadConfig(env: NodeJS.ProcessEnv = process.env): AppConfig {
console.error("SÄKERHETSSTOPP: AAMOS_MODE=mock är inte tillåtet i produktion."); console.error("SÄKERHETSSTOPP: AAMOS_MODE=mock är inte tillåtet i produktion.");
process.exit(1); process.exit(1);
} }
if (cfg.AAMOS_MODE === "gemini" && !cfg.GEMINI_API_KEY) {
console.error("SÄKERHETSSTOPP: AAMOS_MODE=gemini kräver GEMINI_API_KEY i produktion.");
process.exit(1);
}
} }
return cfg; return cfg;
} }
+4
View File
@@ -77,6 +77,10 @@ export const corePlugin = fp(async (app: FastifyInstance, opts: { config: AppCon
AAMOS_API_URL: config.AAMOS_API_URL, AAMOS_API_URL: config.AAMOS_API_URL,
AAMOS_API_KEY: config.AAMOS_API_KEY, AAMOS_API_KEY: config.AAMOS_API_KEY,
AAMOS_TIMEOUT_MS: String(config.AAMOS_TIMEOUT_MS), AAMOS_TIMEOUT_MS: String(config.AAMOS_TIMEOUT_MS),
GEMINI_API_KEY: config.GEMINI_API_KEY,
GEMINI_MODEL: config.GEMINI_MODEL,
GEMINI_TIMEOUT_MS: String(config.GEMINI_TIMEOUT_MS),
GEMINI_DAILY_BUDGET_USD: String(config.GEMINI_DAILY_BUDGET_USD),
}), }),
); );
+2 -1
View File
@@ -10,7 +10,8 @@
"build": "tsup src/index.ts --format esm --target node22 --sourcemap --clean", "build": "tsup src/index.ts --format esm --target node22 --sourcemap --clean",
"typecheck": "tsc --noEmit", "typecheck": "tsc --noEmit",
"test": "vitest run --passWithNoTests", "test": "vitest run --passWithNoTests",
"eval": "tsx src/eval/run.ts" "eval": "tsx src/eval/run.ts",
"eval:scan": "tsx src/eval/scan-eval.ts"
}, },
"dependencies": { "dependencies": {
"@app/ai-contracts": "workspace:*", "@app/ai-contracts": "workspace:*",
+31 -3
View File
@@ -12,7 +12,8 @@ for (const candidate of [".env", "../.env", "../../.env"]) {
} }
import { createHmac } from "node:crypto"; import { createHmac } from "node:crypto";
import { createDatabase, type Database } from "@app/database"; import { createDatabase, type Database } from "@app/database";
import { createAamosClient, type AamosClient } from "@app/ai-contracts"; import { createAamosClient, type AamosClient, type BudgetStore } from "@app/ai-contracts";
import type { Redis } from "ioredis";
export interface WorkerContext { export interface WorkerContext {
db: Database; db: Database;
@@ -23,13 +24,40 @@ export interface WorkerContext {
close: () => Promise<void>; close: () => Promise<void>;
} }
export function createContext(): WorkerContext { class RedisBudgetStore implements BudgetStore {
constructor(
private readonly redis: Redis,
private readonly key: string,
) {}
async getDailySpendUsd(): Promise<number> {
const val = await this.redis.get(this.key);
return val ? Number(val) : 0;
}
async incrementDailySpendUsd(amountUsd: number): Promise<number> {
const newVal = await this.redis.incrbyfloat(this.key, amountUsd);
// Expire at next midnight UTC to keep daily window.
const ttl = this.secondsUntilMidnightUtc();
await this.redis.expire(this.key, ttl);
return Number(newVal);
}
private secondsUntilMidnightUtc(): number {
const now = new Date();
const midnight = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate() + 1));
return Math.max(1, Math.floor((midnight.getTime() - now.getTime()) / 1000));
}
}
export function createContext(redis?: Redis): WorkerContext {
const databaseUrl = process.env.DATABASE_URL; const databaseUrl = process.env.DATABASE_URL;
if (!databaseUrl) { if (!databaseUrl) {
throw new Error("Missing DATABASE_URL. Worker must connect explicitly to the app database."); throw new Error("Missing DATABASE_URL. Worker must connect explicitly to the app database.");
} }
const { db, pool } = createDatabase(databaseUrl); const { db, pool } = createDatabase(databaseUrl);
const aamos = createAamosClient(); const budgetStore = redis ? new RedisBudgetStore(redis, `gemini:daily:budget:${new Date().toISOString().slice(0, 10)}`) : undefined;
const aamos = createAamosClient(undefined, { budgetStore });
const apiBaseUrl = process.env.API_BASE_URL ?? "http://localhost:4000"; const apiBaseUrl = process.env.API_BASE_URL ?? "http://localhost:4000";
const secret = process.env.ENTITLEMENT_SIGNING_SECRET ?? "dev-only-change-me-three"; const secret = process.env.ENTITLEMENT_SIGNING_SECRET ?? "dev-only-change-me-three";
+162
View File
@@ -0,0 +1,162 @@
import { config as loadDotenv } from "dotenv";
import { existsSync } from "node:fs";
import path from "node:path";
for (const candidate of [".env", "../.env", "../../.env"]) {
const p = path.resolve(process.cwd(), candidate);
if (existsSync(p)) {
loadDotenv({ path: p });
break;
}
}
import { GeminiAamosClient, type BudgetStore } from "@app/ai-contracts";
/**
* Live-evaluering av Gemini-lärar-tier för kylskåps-/skafferiskanning.
*
* pnpm --filter @app/worker eval:scan
*
* Kräver AAMOS_MODE=gemini + GEMINI_API_KEY. Använder ett litet guldset av
* publikt tillgängliga bild-URL:er (eller lokala filer). Rapporterar
* träffbild, kvantitet, enhet, kostnad och latens. Avslutar med exitkod 1
* om någon obligatorisk kontroll fallerar.
*/
interface ScanEvalCase {
id: string;
imageUrls: string[];
locationType: "fridge" | "pantry";
marketLocale: string;
checks: (items: Array<{
detectedName: string;
brand: string | null;
estimatedQuantity: number | null;
unit: string | null;
confidence: number;
requiresConfirmation: boolean;
}>) => { name: string; passed: boolean }[];
}
const CASES: ScanEvalCase[] = [
{
id: "arla-milk-fridge",
imageUrls: ["https://upload.wikimedia.org/wikipedia/commons/6/6c/Arla_Ko_Mellanmj%C3%B6lk_1L.jpg"],
locationType: "fridge",
marketLocale: "sv-SE",
checks: (items) => {
const milk = items.find((i) =>
/mjölk|milk/i.test(i.detectedName) || /arla/i.test(i.brand ?? ""),
);
return [
{ name: "hittade mjölkprodukt", passed: !!milk },
{ name: "konfidens > 0.7", passed: !!milk && milk.confidence > 0.7 },
{ name: "kräver bekräftelse om låg konfidens", passed: !!milk && (milk.confidence >= 0.92 || milk.requiresConfirmation) },
];
},
},
{
id: "swedish-butter-fridge",
imageUrls: ["https://upload.wikimedia.org/wikipedia/commons/3/3e/Svenskt_Sm%C3%B6r_Normalsaltat_80-25_500g.jpg"],
locationType: "fridge",
marketLocale: "sv-SE",
checks: (items) => {
const butter = items.find((i) => /smör|butter/i.test(i.detectedName));
return [
{ name: "hittade smör", passed: !!butter },
{ name: "konfidens > 0.7", passed: !!butter && butter.confidence > 0.7 },
];
},
},
];
class SilentBudgetStore implements BudgetStore {
private spend = 0;
async getDailySpendUsd(): Promise<number> {
return this.spend;
}
async incrementDailySpendUsd(amountUsd: number): Promise<number> {
this.spend += amountUsd;
return this.spend;
}
}
async function main() {
const mode = process.env.AAMOS_MODE ?? "mock";
const apiKey = process.env.GEMINI_API_KEY;
if (mode !== "gemini" || !apiKey) {
console.error("[eval:scan] KRÄVER AAMOS_MODE=gemini + GEMINI_API_KEY.");
process.exit(1);
}
const client = new GeminiAamosClient({
apiKey,
model: process.env.GEMINI_MODEL,
timeoutMs: process.env.GEMINI_TIMEOUT_MS ? Number(process.env.GEMINI_TIMEOUT_MS) : 60_000,
dailyBudgetUsd: process.env.GEMINI_DAILY_BUDGET_USD ? Number(process.env.GEMINI_DAILY_BUDGET_USD) : 10,
budgetStore: new SilentBudgetStore(),
});
console.log(`[eval:scan] Gemini-lärar-tier live-evaluering ${CASES.length} fall\n`);
let totalChecks = 0;
let failedChecks = 0;
let totalCostUsd = 0;
let totalLatencyMs = 0;
for (const evalCase of CASES) {
const started = Date.now();
let checks: { name: string; passed: boolean }[] = [];
let error: string | null = null;
try {
const result = await client.runTask(
evalCase.locationType === "fridge" ? "ANALYZE_FRIDGE_IMAGE" : "ANALYZE_PANTRY_IMAGE",
{
imageUrls: evalCase.imageUrls,
locationType: evalCase.locationType,
marketLocale: evalCase.marketLocale,
knownItems: [],
},
);
if (result.status !== "ok" || !result.output) {
error = `status=${result.status}: ${result.error ?? "okänt fel"}`;
checks = [{ name: "Gemini svarade ok", passed: false }];
} else {
const items = result.output.items;
checks = [{ name: "Gemini svarade ok", passed: true }, ...evalCase.checks(items)];
}
if (result.costUsd) totalCostUsd += result.costUsd;
if (result.latencyMs) totalLatencyMs += result.latencyMs;
} catch (err) {
error = (err as Error).message;
checks = [{ name: "Gemini svarade ok", passed: false }];
}
const ms = Date.now() - started;
const passed = checks.every((c) => c.passed);
totalChecks += checks.length;
failedChecks += checks.filter((c) => !c.passed).length;
console.log(`${passed ? "✓" : "✗"} ${evalCase.id} (${ms} ms)`);
for (const c of checks) if (!c.passed) console.log(`${c.name}`);
if (error) console.log(` fel: ${error}`);
}
console.log(`\n[eval:scan] ${CASES.length} fall, ${totalChecks} kontroller, ${failedChecks} fallerade.`);
console.log(`[eval:scan] Total latens: ${totalLatencyMs} ms, total kostnad: ~$${totalCostUsd.toFixed(6)}`);
if (failedChecks > 0) {
console.log("[eval:scan] UNDERKÄND åtgärda innan Skiva 1 går till fälttest.");
process.exit(1);
}
console.log("[eval:scan] GODKÄND fälttest kan börja.");
process.exit(0);
}
main().catch((err) => {
console.error("[eval:scan] KRASCH:", err);
process.exit(1);
});
+1 -1
View File
@@ -38,7 +38,7 @@ import {
const connection = new IORedis(process.env.REDIS_URL ?? "redis://localhost:6379", { const connection = new IORedis(process.env.REDIS_URL ?? "redis://localhost:6379", {
maxRetriesPerRequest: null, maxRetriesPerRequest: null,
}); });
const ctx = createContext(); const ctx = createContext(connection);
const instanceId = process.env.WORKER_INSTANCE_ID ?? `worker-${process.pid}`; const instanceId = process.env.WORKER_INSTANCE_ID ?? `worker-${process.pid}`;
const deadLetterQueue = new Queue(DEAD_LETTER_QUEUE_NAME, { connection }); const deadLetterQueue = new Queue(DEAD_LETTER_QUEUE_NAME, { connection });
+150 -6
View File
@@ -1,9 +1,13 @@
import { eq } from "drizzle-orm"; import { eq, sql } from "drizzle-orm";
import { schema } from "@app/database"; import { schema } from "@app/database";
import type { AamosTaskType } from "@app/ai-contracts"; import type { AamosResult, AamosTaskType, DetectedItem } from "@app/ai-contracts";
import type { WorkerContext } from "../context.js"; import type { WorkerContext } from "../context.js";
import { getLocaleContext } from "../locale.js"; import { getLocaleContext } from "../locale.js";
import type { LocaleContext } from "@app/shared-types"; import type { LocaleContext } from "@app/shared-types";
function currentMonth(): string {
const d = new Date();
return `${d.getUTCFullYear()}-${String(d.getUTCMonth() + 1).padStart(2, "0")}`;
}
/** /**
* Bild-/OCR-jobb (spec §54): hämtar scan_job, anropar AAMOS med kontraktvaliderad * Bild-/OCR-jobb (spec §54): hämtar scan_job, anropar AAMOS med kontraktvaliderad
@@ -51,11 +55,21 @@ export async function processScanJob(ctx: WorkerContext, scanJobId: string): Pro
return; return;
} }
let output = result.output as Record<string, unknown>;
// SKIVA 1: mappa lagrade bilders detekterade namn mot kanoniska ingredienser.
if (
(job.jobType === "ANALYZE_FRIDGE_IMAGE" || job.jobType === "ANALYZE_PANTRY_IMAGE") &&
Array.isArray((output as { items?: unknown }).items)
) {
output = await mapDetectedItemsToCanonical(ctx, output);
}
await ctx.db await ctx.db
.update(schema.scanJobs) .update(schema.scanJobs)
.set({ .set({
status: "awaiting_confirmation", status: "awaiting_confirmation",
result: result.output as Record<string, unknown>, result: output,
modelVersion: result.modelVersion ?? null, modelVersion: result.modelVersion ?? null,
promptVersion: result.promptVersion ?? null, promptVersion: result.promptVersion ?? null,
latencyMs: result.latencyMs ?? Date.now() - started, latencyMs: result.latencyMs ?? Date.now() - started,
@@ -64,9 +78,12 @@ export async function processScanJob(ctx: WorkerContext, scanJobId: string): Pro
}) })
.where(eq(schema.scanJobs.id, scanJobId)); .where(eq(schema.scanJobs.id, scanJobId));
// Bokför verklig AI-kostnad/tokens utan PII (spec §45).
await recordAiUsage(ctx, job.userId, result);
// MEAL_PHOTO_ANALYZED-event för tallriksfoton (spec §55) // MEAL_PHOTO_ANALYZED-event för tallriksfoton (spec §55)
if (job.jobType === "ANALYZE_MEAL_IMAGE") { if (job.jobType === "ANALYZE_MEAL_IMAGE") {
const output = result.output as { const mealOutput = output as {
kcalRange?: { mostLikely: number } | null; kcalRange?: { mostLikely: number } | null;
matchesRecipeContext?: boolean | null; matchesRecipeContext?: boolean | null;
}; };
@@ -76,8 +93,8 @@ export async function processScanJob(ctx: WorkerContext, scanJobId: string): Pro
householdId: job.householdId, householdId: job.householdId,
payload: { payload: {
scanJobId, scanJobId,
matched: output.matchesRecipeContext ?? false, matched: mealOutput.matchesRecipeContext ?? false,
kcalMostLikely: output.kcalRange?.mostLikely ?? null, kcalMostLikely: mealOutput.kcalRange?.mostLikely ?? null,
}, },
}); });
} }
@@ -176,3 +193,130 @@ async function loadConsentFlags(ctx: WorkerContext, userId: string) {
imageTraining: get("image_training"), imageTraining: get("image_training"),
}; };
} }
// ---------------------------------------------------------------------------
// Kanonisk ingrediensmappning (SKIVA 1). AI föreslår, vi matchar mjukt,
// användaren bekräftar alltid innan commit.
// ---------------------------------------------------------------------------
interface CanonicalIndex {
id: string;
nameSv: string;
nameEn: string;
aliases: string[];
}
async function mapDetectedItemsToCanonical(
ctx: WorkerContext,
output: Record<string, unknown>,
): Promise<Record<string, unknown>> {
const items = (output as { items: DetectedItem[] }).items;
if (!items.length) return output;
const index = await loadCanonicalIndex(ctx);
const mapped = items.map((item) => {
const match = findBestCanonicalMatch(item.detectedName, index);
return {
...item,
canonicalIngredientId: match?.id ?? null,
requiresConfirmation: match == null || item.confidence < 0.92,
};
});
return { ...output, items: mapped };
}
async function loadCanonicalIndex(ctx: WorkerContext): Promise<CanonicalIndex[]> {
return ctx.db
.select({
id: schema.canonicalIngredients.id,
nameSv: schema.canonicalIngredients.nameSv,
nameEn: schema.canonicalIngredients.nameEn,
aliases: schema.canonicalIngredients.aliases,
})
.from(schema.canonicalIngredients);
}
function findBestCanonicalMatch(
detectedName: string,
index: CanonicalIndex[],
): CanonicalIndex | null {
const query = detectedName.toLowerCase();
let best: { item: CanonicalIndex; score: number } | null = null;
for (const item of index) {
const score = scoreMatch(query, item);
if (score > 0 && (!best || score > best.score)) {
best = { item, score };
}
}
// Threshold: require a strong token overlap or exact substring.
if (!best || best.score < 0.35) return null;
return best.item;
}
function scoreMatch(query: string, item: CanonicalIndex): number {
const candidates = [
item.nameSv.toLowerCase(),
item.nameEn.toLowerCase(),
...item.aliases.map((a) => a.toLowerCase()),
];
let max = 0;
const queryTokens = tokenize(query);
for (const cand of candidates) {
if (cand === query) return 1;
if (cand.includes(query) || query.includes(cand)) max = Math.max(max, 0.85);
const candTokens = tokenize(cand);
const intersection = queryTokens.filter((t) => candTokens.includes(t));
if (intersection.length > 0) {
const overlap = intersection.length / Math.max(queryTokens.length, candTokens.length);
max = Math.max(max, overlap);
}
}
return max;
}
function tokenize(text: string): string[] {
return text
.toLowerCase()
.replace(/[^a-zåäö0-9\s]/g, " ")
.split(/\s+/)
.filter((t) => t.length > 1);
}
// ---------------------------------------------------------------------------
// AI-kostnadsbokföring utan PII.
// ---------------------------------------------------------------------------
async function recordAiUsage(ctx: WorkerContext, userId: string, result: AamosResult<AamosTaskType>): Promise<void> {
const costUsd = result.costUsd ?? 0;
const tokensIn = result.inputTokens ?? 0;
const tokensOut = result.outputTokens ?? 0;
const microcents = Math.round(costUsd * 100_000_000);
const month = currentMonth();
await ctx.db
.insert(schema.aiUsageCounters)
.values({
userId,
month,
aiScans: 1,
aiTokensIn: tokensIn,
aiTokensOut: tokensOut,
aiCostUsdMicrocents: microcents,
})
.onConflictDoUpdate({
target: [schema.aiUsageCounters.userId, schema.aiUsageCounters.month],
set: {
aiScans: sql`${schema.aiUsageCounters.aiScans} + 1`,
aiTokensIn: sql`${schema.aiUsageCounters.aiTokensIn} + ${tokensIn}`,
aiTokensOut: sql`${schema.aiUsageCounters.aiTokensOut} + ${tokensOut}`,
aiCostUsdMicrocents: sql`${schema.aiUsageCounters.aiCostUsdMicrocents} + ${microcents}`,
},
});
}
+102 -59
View File
@@ -1,60 +1,111 @@
# Del 9 AAMOS-integration # Del 9 AI-integration (AAMOS + Gemini-lärar-tier)
AAMOS är er befintliga AI-plattform och nås **uteslutande via API** från appens > Uppdaterad 2026-08-08 efter scouting: AAMOS Core v1 är autentiseringsmässigt
backend (beslut 2026-08-02). Mobilappen har aldrig direktkontakt och innehåller inga > nåbar men saknar konfigurerad matmodell (`/v1/detect` faller tillbaka på YuNet
AI-hemligheter (spec §31, §61.13). > ansiktsdetektering). Tills vidare används **Gemini 2.5 Flash** som
> lärar-tier för kylskåpsskanning, bakom samma `AamosClient`-gränssnitt.
> AAMOS/prod rörs inte.
Appen når AI **uteslutande via backend** (beslut 2026-08-02). Mobilappen har
aldrig direktkontakt och innehåller inga AI-hemligheter (spec §31, §61.13).
## Kontraktet (`packages/ai-contracts`) ## Kontraktet (`packages/ai-contracts`)
Appen definierar ett versionerat, Zod-validerat kontrakt per uppgiftstyp. Input Appen definierar ett versionerat, Zod-validerat kontrakt per uppgiftstyp. Input
valideras FÖRE nätverksanropet, output valideras EFTER ett svar som bryter kontraktet valideras FÖRE nätverksanropet, output valideras EFTER ett svar som bryter kontraktet
behandlas som fel och når aldrig användardata. AAMOS kan därmed byta modeller, behandlas som fel och når aldrig användardata. Modeller, prompts och leverantörer kan
prompts och leverantörer fritt bakom kontraktet ("Food API ska få stabila kontrakt bytas fritt bakom kontraktet ("Food API ska få stabila kontrakt oavsett modell").
oavsett modell").
Antaget transport-API (justeras mot er AAMOS-dokumentation endast ```
`HttpAamosClient` berörs): ┌─────────┐ signed image URL ┌──────────┐ AamosClient ┌─────────────────┐
│ App │ ───────────────────→ │ Food API │ ───────────────→│ Worker adapter │
```text └─────────┘ └──────────┘ └─────────────────┘
POST {AAMOS_API_URL}/v1/tasks Authorization: Bearer {AAMOS_API_KEY}
→ AamosRequestEnvelope { taskId, taskType, contractVersion, input, metadata } ┌──────────────────┬──────────┴──────────┐
← AamosResponseEnvelope { taskId, status: ok|uncertain|failed, output, │ AAMOS_MODE=http │ AAMOS_MODE=gemini │
modelVersion, promptVersion, latencyMs, costUsd } │ HttpAamosClient │ GeminiAamosClient │
GET {AAMOS_API_URL}/v1/health │ → AAMOS Core v1 │ → Gemini 2.5 Flash │
└──────────────────┴─────────────────────┘
``` ```
Metadata per anrop: `correlationId` (spårbarhet §58), `subjectRef` `createAamosClient()` väljer adapter utifrån `AAMOS_MODE`. Inga anropare behöver
(**pseudonymiserat** id aldrig e-post/namn, §56), `priority`, samt ändras. App→Food API→leverantörsgränsen är oförändrad.
`consentFlags {personalization, anonymizedImprovement, imageTraining}` så att AAMOS
kan upprätthålla samtyckesreglerna på sin sida också.
**Behövs från er:** AAMOS bas-URL + API-nyckel per miljö, och er endpoint-/schema- ## AAMOS Core v1-status
dokumentation. Avviker kuvertformatet mappas det i `HttpAamosClient` kontrakten
utåt ändras inte.
## Uppgiftstyper (15) Reella endpoints (ingen `/v1/tasks` fiktion):
Bild/OCR: `ANALYZE_FRIDGE_IMAGE`, `ANALYZE_PANTRY_IMAGE`, `ANALYZE_MEAL_IMAGE` ```text
(kcal-INTERVALL, aldrig exakt påstående), `READ_RECEIPT`, `READ_NUTRITION_LABEL` POST /v1/detect POST /v1/verify POST /v1/compare
(värden från etiketten inte modellens gissning), `READ_EXPIRY_DATE`. POST /v1/segment POST /v1/classify POST /v1/authenticate
Text/struktur: `NORMALIZE_PRODUCTS`, `DEDUPLICATE_INVENTORY`, `STRUCTURE_RECIPE_TEXT`, POST /v1/score POST /v1/explain POST /v1/extract
`PARSE_CRAVING`, `MODERATE_RECIPE`. POST /v1/track GET /v1/health
Rådgivande: `GENERATE_RECIPE_OPTIONS` (granskas redaktionellt, §15), `RANK_RECIPES` ```
(får ordna om aldrig lägga till), `GENERATE_WEEK_PLAN`, `UPDATE_USER_MEMORY`.
Alla outputs bär `confidence`, och "osäker/okänd" är förstklassiga svar Scouting 2026-08-08 visade:
(`canonicalIngredientId: null`, `requiresConfirmation: true`) spec §10/§61.4.
## Routing & specialister (spec §33) - `amos-core.service` på server-2 anropar `http://localhost:3207`.
- Värdport `3207` ägs av `aamos-identity-rust`, inte AI-inferens.
- AI-inferenscontainern `amos-ai-inference:latest` exponerar **värdport 3209**.
- Port mismatch gör att AAMOS Core aldrig når inferens.
- Även efter hypotetisk port-fix saknas `FOOD`/`GROCERY`/`VISION_MODEL`
konfiguration i `.env.amos-core`; `/v1/detect` defaultar till `yunet`.
Rekommenderad routing inne i AAMOS: specialistmodell först (vanliga livsmedel, **Beslut:** rör inte AAMOS-prod. AAMOS kan återaktiveras i framtiden när en
nordiska förpackningar, kvitto-OCR, datum-OCR, portionssegmentering) → hög confidence: matmodell finns konfigurerad, via `HttpAamosClient`.
svara; låg: extern generalist (Haiku 4.5-klass för volym, Sonnet-klass för svåra fall);
fortsatt osäkert: `status: "uncertain"` → appen frågar användaren. Appen skickar
`modelVersion`/`promptVersion` vidare in i `scan_jobs` och `ai_corrections` så att
varje datapunkt är spårbar till modellversion (§9).
## Feedback → träning (spec §33) ## Gemini-lärar-tier (Skiva 1)
Aktiveras med `AAMOS_MODE=gemini` + `GEMINI_API_KEY`. Endast workern anropar
Gemini; API:et når aldrig nyckeln.
### Hanterade uppgiftstyper
- `ANALYZE_FRIDGE_IMAGE`
- `ANALYZE_PANTRY_IMAGE`
Övriga task types (`READ_RECEIPT`, `READ_NUTRITION_LABEL`, etc.) svarar
`failed` i Skiva 1 och faller tillbaka på appens befintliga flöden.
### Flöde
1. App laddar upp bild → Food API skapar `scan_job` (status `queued`).
2. Workern plockar jobbet, hämtar bilden från lagring (S3/mock).
3. `GeminiAamosClient` skickar bilden + svensk livsmedelsprompt till
`generativelanguage.googleapis.com/v1beta/models/{GEMINI_MODEL}:generateContent`.
4. Gemini svarar med JSON: produkt, varumärke, kvantitet, enhet, kategori,
bäst-före/sista-förbruk (om läsbart), konfidens, imageQualityIssues.
5. Workern mappar detekterade namn mot `canonical_ingredients` (mjuk matchning).
6. Resultatet sparas i `scan_jobs` som `awaiting_confirmation`.
7. Användaren granskar och bekräftar i appen. Aldrig auto-commit.
8. AI är aldrig facit för allergener/näring — mjölkprincipen orörd.
### Kostnadskontroll
- `scan_jobs.costUsd` lagrar verklig kostnad per anrop.
- `ai_usage_counters.ai_cost_usd_microcents` ackumulerar månadskostnad per
användare (utan PII).
- Hushållskvoten `aiScans` respekteras (befintlig entitlements-logik).
- Global dagsbudget `GEMINI_DAILY_BUDGET_USD` via `BudgetStore`. Workern använder
Redis-backed budget; tester använder in-memory budget. När budgeten är
förbrukad nekar adaptern nya anrop.
- Loggning: status, latens, tokens, kostnad, taskType. Ingen bild, inget
fritext, inga PII-fält.
### Säkerhet
- Nyckeln finns i `.env` på workerservrar (SSM i produktion) och checkas aldrig
in.
- App/API validerar att `AAMOS_MODE=gemini` kräver `GEMINI_API_KEY` i produktion.
- `REQUIRE_REAL=1` i deploy-grinden gör att start failar om nyckel saknas.
## Routing & specialister (framtida)
Skiva 2 kommer lägga premium-eskaleringsstier (Gemini Pro / GPT-4o / Claude)
internt i adaptern vid låg konfidens, med eval/guldset som beslutar billig vs
dyr modell. Fram till dess: Gemini 2.5 Flash för allt i Skiva 1.
## Feedback → träning (Skiva 3)
Varje användarkorrigering i granska-flödet sparas som `ai_corrections` Varje användarkorrigering i granska-flödet sparas som `ai_corrections`
(AI-utdata + korrigering + modell/promptversion + **samtyckessnapshot**). (AI-utdata + korrigering + modell/promptversion + **samtyckessnapshot**).
@@ -62,25 +113,17 @@ Veckojobbet `BUILD_TRAINING_SAMPLE` exporterar endast rader där
`anonymized_improvement = granted` vid korrigeringstillfället; bilder kräver `anonymized_improvement = granted` vid korrigeringstillfället; bilder kräver
separat `image_training`-samtycke. Personligt minne är aldrig träningsdata (§32). separat `image_training`-samtycke. Personligt minne är aldrig träningsdata (§32).
## Memory (spec §32)
Appens DB är den användarsynliga sanningen (`memory_items`). Nattjobbet skickar
veckans domänhändelser + befintliga minnesnycklar till `UPDATE_USER_MEMORY`; AAMOS
returnerar förslag (kind/key/sammanfattning/confidence/expires) som skrivs in men
poster som användaren verifierat eller pausat skrivs **aldrig** över. Användarens
rättelser blir `user_stated` med confidence 1 och vinner alltid (§30).
## Evals (spec §34) ## Evals (spec §34)
`ai_eval_cases` (fast testbibliotek: kyl/frys/skafferi/tallrik/kvitton/etiketter/ - `pnpm test`: hermetiska tester med inspelad Gemini-fixture, når aldrig nätet.
allergifall/mörka bilder/överlapp/orimliga recept) + `ai_eval_runs` (precision, recall, - `pnpm --filter @app/worker eval:scan`: live-eval mot litet guldset. Rapporterar
missade produkter, hallucinationer, allergifel, latens, kostnad, korrigeringsgrad). precision, recall, kostnad, latens. Kräver `AAMOS_MODE=gemini` + nyckel.
Regel: **ingen modell- eller promptändring i AAMOS tas i produktion för appen utan - Regel: **ingen modell-/promptändring tas i produktion utan grön eval**.
grön eval-körning**, synlig i adminpanelen (`/admin/v1/ai/eval-runs`). Corpus byggs
under beta ur anonymiserade, samtyckta exempel.
## Kostnadskontroll ## Lägen
`costUsd` per anrop loggas på scan_jobs → daglig kostnad per uppgiftstyp i admin; | `AAMOS_MODE` | Användning |
larm vid spik (§58). Fair use-kvoterna (Del 10/13) begränsar exponeringen per användare. |---|---|
Mock-läget (`AAMOS_MODE=mock`) finns för dev/test; produktion vägrar starta i mock. | `mock` | dev/test utan nätverk. Produkten vägrar starta i mock. |
| `http` | Riktig AAMOS Core när matmodell finns. |
| `gemini` | Gemini 2.5 Flash som lärar-tier för kylskåpsskanning (Skiva 1). |
+14 -2
View File
@@ -200,7 +200,12 @@ say "GRIND 4 OK: databas migrerad$([ "$MODE" = staging ] && echo ' + seedad')"
# GRIND 5: AAMOS-utvärdering (spec §40) # GRIND 5: AAMOS-utvärdering (spec §40)
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
say "Kör AAMOS-utvärderingssviten mot läge: $AAMOS_MODE_VAL" say "Kör AAMOS-utvärderingssviten mot läge: $AAMOS_MODE_VAL"
pnpm eval:aamos || fail "Utvärderingssviten UNDERKÄND deploya inte (spec §40)." if [ "$AAMOS_MODE_VAL" = "gemini" ]; then
# Skiva 1: endast kylskåps-/skafferiskanning är aktiv mot Gemini.
pnpm eval:scan || fail "Gemini scan-eval UNDERKÄND deploya inte (spec §40)."
else
pnpm eval:aamos || fail "Utvärderingssviten UNDERKÄND deploya inte (spec §40)."
fi
say "GRIND 5 OK: eval-sviten GODKÄND" say "GRIND 5 OK: eval-sviten GODKÄND"
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
@@ -248,7 +253,7 @@ say "GRIND 7 OK: register -> login -> recept fungerar"
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
EMAIL_MODE_VAL=$(get_env EMAIL_MODE); EMAIL_MODE_VAL=${EMAIL_MODE_VAL:-log} EMAIL_MODE_VAL=$(get_env EMAIL_MODE); EMAIL_MODE_VAL=${EMAIL_MODE_VAL:-log}
S3_MODE_VAL=$(get_env S3_MODE); S3_MODE_VAL=${S3_MODE_VAL:-mock} S3_MODE_VAL=$(get_env S3_MODE); S3_MODE_VAL=${S3_MODE_VAL:-mock}
aamos_status=$([ "$AAMOS_MODE_VAL" = "http" ] && echo "RIKTIG (http)" || echo "mock") aamos_status=$([ "$AAMOS_MODE_VAL" = "http" ] && echo "RIKTIG (http)" || [ "$AAMOS_MODE_VAL" = "gemini" ] && echo "RIKTIG (Gemini)" || echo "mock")
email_status=$([ "$EMAIL_MODE_VAL" = "smtp" ] && echo "RIKTIG (smtp)" || echo "log (mejl till logs/api.log)") email_status=$([ "$EMAIL_MODE_VAL" = "smtp" ] && echo "RIKTIG (smtp)" || echo "log (mejl till logs/api.log)")
s3_status=$([ "$S3_MODE_VAL" = "aws" ] && echo "RIKTIG (S3: $(get_env S3_BUCKET))" || echo "mock (lokal disk)") s3_status=$([ "$S3_MODE_VAL" = "aws" ] && echo "RIKTIG (S3: $(get_env S3_BUCKET))" || echo "mock (lokal disk)")
say "-----------------------------------------------" say "-----------------------------------------------"
@@ -258,6 +263,12 @@ say " E-post: $email_status"
say " Fillagring: $s3_status" say " Fillagring: $s3_status"
say "-----------------------------------------------" say "-----------------------------------------------"
if [ "${REQUIRE_REAL:-0}" = "1" ]; then if [ "${REQUIRE_REAL:-0}" = "1" ]; then
# Skiva 1: Gemini-lärar-tier räknas som riktig AI. E-post/S3 får fortfarande
# vara mock i staging tills de konfigureras; AI får aldrig mocka tyst.
if [ "$AAMOS_MODE_VAL" = "gemini" ]; then
[ -n "$(get_env GEMINI_API_KEY)" ] || fail "REQUIRE_REAL=1 men GEMINI_API_KEY saknas för AAMOS_MODE=gemini."
say "GRIND 8 OK: REQUIRE_REAL uppfyllt Gemini-lärar-tier är riktig."
else
MOCKS="" MOCKS=""
[ "$AAMOS_MODE_VAL" != "http" ] && MOCKS="$MOCKS AAMOS(AAMOS_MODE=http+URL+nyckel)" [ "$AAMOS_MODE_VAL" != "http" ] && MOCKS="$MOCKS AAMOS(AAMOS_MODE=http+URL+nyckel)"
[ "$EMAIL_MODE_VAL" != "smtp" ] && MOCKS="$MOCKS E-post(EMAIL_MODE=smtp+SMTP_*+MAIL_FROM)" [ "$EMAIL_MODE_VAL" != "smtp" ] && MOCKS="$MOCKS E-post(EMAIL_MODE=smtp+SMTP_*+MAIL_FROM)"
@@ -266,6 +277,7 @@ if [ "${REQUIRE_REAL:-0}" = "1" ]; then
fail "REQUIRE_REAL=1 men följande kör fortfarande mock:$MOCKS fyll i .env och kör om." fail "REQUIRE_REAL=1 men följande kör fortfarande mock:$MOCKS fyll i .env och kör om."
fi fi
say "GRIND 8 OK: REQUIRE_REAL uppfyllt alla integrationer är riktiga." say "GRIND 8 OK: REQUIRE_REAL uppfyllt alla integrationer är riktiga."
fi
fi fi
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
@@ -0,0 +1,3 @@
-- Add ai_cost_usd_microcents to ai_usage_counters for Gemini spend tracking.
ALTER TABLE ai_usage_counters
ADD COLUMN ai_cost_usd_microcents BIGINT NOT NULL DEFAULT 0;
@@ -127,6 +127,13 @@
"when": 1786056600000, "when": 1786056600000,
"tag": "0018_cooking_sessions_meal_box_mutations", "tag": "0018_cooking_sessions_meal_box_mutations",
"breakpoints": true "breakpoints": true
},
{
"idx": 18,
"version": "7",
"when": 1786141200000,
"tag": "0019_ai_usage_cost_usd",
"breakpoints": true
} }
] ]
} }
+2 -1
View File
@@ -28,7 +28,8 @@
"worker:dev": "pnpm --filter @app/worker dev", "worker:dev": "pnpm --filter @app/worker dev",
"admin:dev": "pnpm --filter @app/admin dev", "admin:dev": "pnpm --filter @app/admin dev",
"mobile:start": "pnpm --filter @app/mobile start", "mobile:start": "pnpm --filter @app/mobile start",
"eval:aamos": "pnpm --filter @app/worker eval" "eval:aamos": "pnpm --filter @app/worker eval",
"eval:scan": "pnpm --filter @app/worker eval:scan"
}, },
"devDependencies": { "devDependencies": {
"@types/node": "^22.10.0", "@types/node": "^22.10.0",
+38 -2
View File
@@ -8,6 +8,7 @@ import {
type TaskOutput, type TaskOutput,
} from "./tasks.js"; } from "./tasks.js";
import { mockOutputFor } from "./mock.js"; import { mockOutputFor } from "./mock.js";
import { GeminiAamosClient, type BudgetStore } from "./gemini.js";
import type { LocaleContext } from "@app/shared-types"; import type { LocaleContext } from "@app/shared-types";
@@ -31,6 +32,8 @@ export interface AamosResult<T extends AamosTaskType> {
promptVersion?: string | undefined; promptVersion?: string | undefined;
latencyMs?: number | undefined; latencyMs?: number | undefined;
costUsd?: number | undefined; costUsd?: number | undefined;
inputTokens?: number | undefined;
outputTokens?: number | undefined;
} }
/** /**
@@ -248,12 +251,44 @@ export interface AamosEnv {
AAMOS_API_URL?: string | undefined; AAMOS_API_URL?: string | undefined;
AAMOS_API_KEY?: string | undefined; AAMOS_API_KEY?: string | undefined;
AAMOS_TIMEOUT_MS?: string | undefined; AAMOS_TIMEOUT_MS?: string | undefined;
GEMINI_API_KEY?: string | undefined;
GEMINI_MODEL?: string | undefined;
GEMINI_TIMEOUT_MS?: string | undefined;
GEMINI_DAILY_BUDGET_USD?: string | undefined;
} }
/** Fabrik: http mot riktiga AAMOS (default), mock för dev/test. */ export interface CreateAamosClientOptions {
export function createAamosClient(env: AamosEnv = process.env): AamosClient { budgetStore?: BudgetStore;
fetchImpl?: typeof fetch;
}
/** Fabrik: http mot riktiga AAMOS, gemini-lärar-tier, eller mock för dev/test. */
export function createAamosClient(
env: AamosEnv = process.env,
options: CreateAamosClientOptions = {},
): AamosClient {
const mode = env.AAMOS_MODE ?? "http"; const mode = env.AAMOS_MODE ?? "http";
if (mode === "mock") return new MockAamosClient(); if (mode === "mock") return new MockAamosClient();
if (mode === "gemini") {
const apiKey = env.GEMINI_API_KEY;
if (!apiKey) {
throw new Error(
"AAMOS_MODE=gemini kräver GEMINI_API_KEY. " +
"Sätt AAMOS_MODE=mock för lokal utveckling utan Gemini-åtkomst.",
);
}
return new GeminiAamosClient({
apiKey,
model: env.GEMINI_MODEL,
timeoutMs: env.GEMINI_TIMEOUT_MS ? Number(env.GEMINI_TIMEOUT_MS) : undefined,
dailyBudgetUsd: env.GEMINI_DAILY_BUDGET_USD ? Number(env.GEMINI_DAILY_BUDGET_USD) : undefined,
budgetStore: options.budgetStore,
fetchImpl: options.fetchImpl,
});
}
const baseUrl = env.AAMOS_API_URL; const baseUrl = env.AAMOS_API_URL;
const apiKey = env.AAMOS_API_KEY; const apiKey = env.AAMOS_API_KEY;
if (!baseUrl || !apiKey) { if (!baseUrl || !apiKey) {
@@ -266,6 +301,7 @@ export function createAamosClient(env: AamosEnv = process.env): AamosClient {
baseUrl, baseUrl,
apiKey, apiKey,
timeoutMs: env.AAMOS_TIMEOUT_MS ? Number(env.AAMOS_TIMEOUT_MS) : undefined, timeoutMs: env.AAMOS_TIMEOUT_MS ? Number(env.AAMOS_TIMEOUT_MS) : undefined,
fetchImpl: options.fetchImpl,
}); });
} }
+358
View File
@@ -0,0 +1,358 @@
/**
* Gemini-lärar-tier för AAMOS-task-typer (SKIVA 1: kylskåpsskanning).
*
* - Implementerar samma AamosClient-interface som HttpAamosClient/MockAamosClient.
* - App + Food API ser ingen skillnad; workern byter bara adapter.
* - Nycklar finns serversidan; klienten exponeras aldrig i appen.
* - Kostnad/tokens rapporteras tillbaka så workern kan bokföra i ai_usage_counters.
* - Global dagsbudget kan sättas via GEMINI_DAILY_BUDGET_USD.
*/
import { randomUUID } from "node:crypto";
import { z } from "zod";
import {
TASK_CONTRACTS,
type AamosTaskType,
type TaskInput,
type TaskOutput,
detectedItemSchema,
type DetectedItem,
} from "./tasks.js";
import type { AamosCallOptions, AamosClient, AamosResult } from "./client.js";
import type { LocaleContext } from "@app/shared-types";
const GEMINI_API_BASE = "https://generativelanguage.googleapis.com/v1beta";
/** Estimated max cost per image call in USD (pessimistic). */
const COST_ESTIMATE_PER_IMAGE_USD = 0.0015;
export interface BudgetStore {
/** Return current daily spend in USD. */
getDailySpendUsd(): Promise<number>;
/** Increment daily spend by amountUsd; return new total. */
incrementDailySpendUsd(amountUsd: number): Promise<number>;
}
/** In-memory budget store for tests / single-process dev. */
export class MemoryBudgetStore implements BudgetStore {
private spend = 0;
async getDailySpendUsd(): Promise<number> {
return this.spend;
}
async incrementDailySpendUsd(amountUsd: number): Promise<number> {
this.spend += amountUsd;
return this.spend;
}
reset() {
this.spend = 0;
}
}
export interface GeminiAamosClientConfig {
apiKey: string;
model?: string;
timeoutMs?: number;
dailyBudgetUsd?: number;
budgetStore?: BudgetStore;
fetchImpl?: typeof fetch;
promptVersion?: string;
}
const geminiFridgeResponseSchema = z.object({
items: z.array(
z.object({
produkt: z.string(),
varumarke: z.string().nullable(),
kvantitet: z.union([z.number(), z.string()]).nullable(),
enhet: z.string().nullable(),
kategori: z.string().nullable(),
bastaFore: z.string().nullable().optional(),
sistaForbruk: z.string().nullable().optional(),
beskrivning: z.string().nullable().optional(),
konfidens: z.number().min(0).max(1),
}),
),
imageQualityIssues: z
.array(z.enum(["dark", "blurry", "occlusion", "too_far"]))
.default([]),
});
function parseDate(value: string | null | undefined): string | null {
if (!value) return null;
const normalized = value.trim();
if (!normalized) return null;
// Accept ISO dates (YYYY-MM-DD) only.
if (/^\d{4}-\d{2}-\d{2}$/.test(normalized)) return normalized;
// Try common Swedish/EU formats.
const parts = normalized.split(/[-/.]/);
if (parts.length === 3) {
const [aStr, bStr, cStr] = parts;
if (!aStr || !bStr || !cStr) return null;
const a = parseInt(aStr, 10);
const b = parseInt(bStr, 10);
const c = parseInt(cStr, 10);
if (Number.isNaN(a) || Number.isNaN(b) || Number.isNaN(c)) return null;
// YYYY-MM-DD
if (a > 2000 && b <= 12 && c <= 31) return `${a}-${String(b).padStart(2, "0")}-${String(c).padStart(2, "0")}`;
// DD-MM-YYYY
if (c > 2000 && b <= 12 && a <= 31) return `${c}-${String(b).padStart(2, "0")}-${String(a).padStart(2, "0")}`;
}
return null;
}
function parseQuantity(value: string | number | null | undefined): { quantity: number | null; unit: string | null } {
if (value == null) return { quantity: null, unit: null };
if (typeof value === "number") return { quantity: value, unit: null };
const text = String(value).trim().replace(/,/g, ".");
if (!text) return { quantity: null, unit: null };
const match = text.match(/^(\d+(?:\.\d+)?)\s*(.*)$/);
if (match && match[1] != null && match[2] != null) {
const qty = parseFloat(match[1]);
const unit = match[2].trim();
return { quantity: Number.isFinite(qty) ? qty : null, unit: unit || null };
}
return { quantity: null, unit: text };
}
function normalizeUnit(unit: string | null | undefined): string | null {
if (unit == null) return null;
const u = unit.toLowerCase().trim();
if (u === "g" || u === "gram") return "GRAM";
if (u === "kg" || u === "kilo" || u === "kilogram") return "KILOGRAM";
if (u === "ml" || u === "milliliter") return "MILLILITER";
if (u === "dl" || u === "deciliter") return "DECILITER";
if (u === "l" || u === "liter") return "LITER";
if (u === "st" || u === "styck" || u === "piece" || u === "pieces") return "COUNT";
if (u === "förpackning" || u === "paket" || u === "package") return "PACKAGE";
if (u === "portion" || u === "portioner") return "PORTION";
if (u === "klyfta" || u === "klyftor") return "CLOVE";
if (u === "skiva" || u === "skivor") return "SLICE";
return null;
}
function toDetectedItems(parsed: z.infer<typeof geminiFridgeResponseSchema>): DetectedItem[] {
return parsed.items.map((item) => {
const { quantity, unit } = parseQuantity(item.kvantitet);
return detectedItemSchema.parse({
detectedName: item.produkt,
canonicalIngredientId: null,
brand: item.varumarke,
estimatedQuantity: quantity,
unit: normalizeUnit(unit ?? item.enhet),
bestBeforeDate: parseDate(item.bastaFore ?? item.sistaForbruk),
confidence: item.konfidens,
requiresConfirmation: item.konfidens < 0.92,
boundingBox: null,
});
});
}
export class GeminiAamosClient implements AamosClient {
private readonly cfg: Required<Omit<GeminiAamosClientConfig, "budgetStore" | "fetchImpl">> & {
fetchImpl: typeof fetch;
budgetStore?: BudgetStore;
};
constructor(config: GeminiAamosClientConfig) {
this.cfg = {
apiKey: config.apiKey,
model: config.model ?? "gemini-2.5-flash",
timeoutMs: config.timeoutMs ?? 60_000,
dailyBudgetUsd: config.dailyBudgetUsd ?? 0,
promptVersion: config.promptVersion ?? "gemini-fridge-v1",
fetchImpl: config.fetchImpl ?? fetch,
budgetStore: config.budgetStore,
};
}
async healthCheck(): Promise<{ ok: boolean; detail?: string }> {
try {
const url = `${GEMINI_API_BASE}/models?key=${this.cfg.apiKey}`;
const res = await this.cfg.fetchImpl(url, { signal: AbortSignal.timeout(5_000) });
return { ok: res.ok, detail: `status ${res.status}` };
} catch (err) {
return { ok: false, detail: err instanceof Error ? err.message : String(err) };
}
}
async runTask<T extends AamosTaskType>(
taskType: T,
input: TaskInput<T>,
options: AamosCallOptions = {},
): Promise<AamosResult<T>> {
const contract = TASK_CONTRACTS[taskType];
const parsedInput = contract.input.safeParse(input);
if (!parsedInput.success) {
return {
status: "failed",
output: null,
error: `Kontraktsfel (input) för ${taskType}: ${parsedInput.error.message}`,
};
}
switch (taskType) {
case "ANALYZE_FRIDGE_IMAGE":
case "ANALYZE_PANTRY_IMAGE":
return this.analyzeStorageImage(taskType, parsedInput.data as TaskInput<"ANALYZE_FRIDGE_IMAGE">, options) as Promise<AamosResult<T>>;
default:
return {
status: "failed",
output: null,
error: `${taskType} stöds inte av Gemini-lärar-tier i SKIVA 1.`,
} as AamosResult<T>;
}
}
private async analyzeStorageImage(
taskType: "ANALYZE_FRIDGE_IMAGE" | "ANALYZE_PANTRY_IMAGE",
input: TaskInput<"ANALYZE_FRIDGE_IMAGE">,
options: AamosCallOptions,
): Promise<AamosResult<"ANALYZE_FRIDGE_IMAGE">> {
const locale = options.localeContext ?? this.defaultLocale();
const started = Date.now();
const imageParts = await Promise.all(
input.imageUrls.slice(0, 6).map(async (url) => {
const res = await this.cfg.fetchImpl(url, { signal: AbortSignal.timeout(this.cfg.timeoutMs) });
if (!res.ok) throw new Error(`Kunde inte hämta bild: ${res.status} ${url}`);
const buf = Buffer.from(await res.arrayBuffer());
return { inline_data: { mime_type: this.mimeType(buf), data: buf.toString("base64") } };
}),
);
const estimatedCostUsd = imageParts.length * COST_ESTIMATE_PER_IMAGE_USD;
if (await this.isOverBudget(estimatedCostUsd)) {
return {
status: "failed",
output: null,
error: "Global Gemini-dagsbudget är förbrukad.",
};
}
const prompt = this.buildFridgePrompt(taskType, locale, input.locationType);
const payload = {
contents: [{ parts: [{ text: prompt }, ...imageParts] }],
generationConfig: {
responseMimeType: "application/json",
temperature: 0.2,
},
};
const url = `${GEMINI_API_BASE}/models/${this.cfg.model}:generateContent?key=${this.cfg.apiKey}`;
const res = await this.cfg.fetchImpl(url, {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify(payload),
signal: AbortSignal.timeout(this.cfg.timeoutMs),
});
if (!res.ok) {
const body = await res.text();
return {
status: "failed",
output: null,
error: `Gemini ${res.status}: ${body.slice(0, 500)}`,
};
}
const geminiBody = (await res.json()) as {
candidates?: Array<{ content?: { parts?: Array<{ text?: string }> } }>;
usageMetadata?: { promptTokenCount?: number; candidatesTokenCount?: number };
};
const text = geminiBody.candidates?.[0]?.content?.parts?.[0]?.text ?? "";
let parsed: z.infer<typeof geminiFridgeResponseSchema>;
try {
const json = JSON.parse(text);
const safe = geminiFridgeResponseSchema.safeParse(json);
if (!safe.success) {
return { status: "failed", output: null, error: `Gemini-svar matchar inte schema: ${safe.error.message}` };
}
parsed = safe.data;
} catch {
return { status: "failed", output: null, error: "Gemini-svar var inte giltig JSON." };
}
const inputTokens = geminiBody.usageMetadata?.promptTokenCount ?? 0;
const outputTokens = geminiBody.usageMetadata?.candidatesTokenCount ?? 0;
const costUsd = this.estimateCostUsd(inputTokens, outputTokens);
await this.recordSpend(costUsd);
const latencyMs = Date.now() - started;
const items = toDetectedItems(parsed);
return {
status: items.length > 0 ? "ok" : "uncertain",
output: { items, imageQualityIssues: parsed.imageQualityIssues } as TaskOutput<"ANALYZE_FRIDGE_IMAGE">,
modelVersion: this.cfg.model,
promptVersion: this.cfg.promptVersion,
latencyMs,
costUsd,
inputTokens,
outputTokens,
};
}
private buildFridgePrompt(
taskType: "ANALYZE_FRIDGE_IMAGE" | "ANALYZE_PANTRY_IMAGE",
locale: LocaleContext,
locationType: string,
): string {
const place = taskType === "ANALYZE_FRIDGE_IMAGE" ? "kylskåpet" : "skafferiet";
const lang = locale.languageTag.startsWith("en") ? "English" : "Swedish";
return `Du är en noggrann livsmedelsassistent för svenska hushåll. Användaren har fotat ${place} (${locationType}).
Identifiera varje livsmedelsprodukt du ser. För varje produkt, svara med:
- produkt: produktens namn på svenska (t.ex. "Mellanmjölk", "Smör", "Vispgrädde")
- varumarke: varumärke om det syns tydligt, annars null
- kvantitet: siffra (t.ex. 1, 500, 1.5)
- enhet: förkortning som l, g, ml, st, förpackning
- kategori: t.ex. mejeri, frukt, grönt, kött, skafferi
- bastaFore: bäst-före-datum om det syns (YYYY-MM-DD), annars null
- sistaForbruk: sista förbrukningsdatum om det syns (YYYY-MM-DD), annars null
- konfidens: 0.01.0
Svara ENDAST med giltig JSON i exakt detta format:
{
"items": [...],
"imageQualityIssues": []
}
imageQualityIssues kan innehålla någon av: "dark", "blurry", "occlusion", "too_far".
Språk: ${lang}.`;
}
private defaultLocale(): LocaleContext {
return {
languageTag: "sv-SE",
regionCode: "SE",
timeZone: "Europe/Stockholm",
measurementSystem: "METRIC",
temperatureUnit: "CELSIUS",
currencyCode: "SEK",
};
}
private mimeType(buf: Buffer): string {
if (buf[0] === 0xff && buf[1] === 0xd8) return "image/jpeg";
if (buf.slice(0, 8).toString("hex") === "89504e470d0a1a0a") return "image/png";
if (buf.slice(0, 4).toString("ascii") === "RIFF") return "image/webp";
return "image/jpeg";
}
private estimateCostUsd(inputTokens: number, outputTokens: number): number {
// Gemini 2.5 Flash pricing (Aug 2025): ~$0.075/1M input, $0.30/1M output.
return inputTokens * 0.075e-6 + outputTokens * 0.30e-6;
}
private async isOverBudget(estimateUsd: number): Promise<boolean> {
if (this.cfg.dailyBudgetUsd <= 0 || !this.cfg.budgetStore) return false;
const current = await this.cfg.budgetStore.getDailySpendUsd();
return current + estimateUsd > this.cfg.dailyBudgetUsd;
}
private async recordSpend(costUsd: number): Promise<void> {
if (this.cfg.budgetStore && costUsd > 0) {
await this.cfg.budgetStore.incrementDailySpendUsd(costUsd);
}
}
}
+1
View File
@@ -1,3 +1,4 @@
export * from "./tasks.js"; export * from "./tasks.js";
export * from "./client.js"; export * from "./client.js";
export * from "./gemini.js";
export { mockOutputFor } from "./mock.js"; export { mockOutputFor } from "./mock.js";
+154
View File
@@ -0,0 +1,154 @@
import { describe, it, expect } from "vitest";
import { GeminiAamosClient, MemoryBudgetStore } from "../src/gemini.js";
import type { AamosRequestEnvelope } from "../src/tasks.js";
const FIXTURE_RESPONSE = {
candidates: [
{
content: {
parts: [
{
text: JSON.stringify({
items: [
{
produkt: "Mellanmjölk 1,5% fett",
varumarke: "Arla",
kvantitet: 1,
enhet: "liter",
kategori: "Mejeri",
bastaFore: null,
sistaForbruk: null,
konfidens: 0.98,
},
{
produkt: "Smör",
varumarke: "Arla",
kvantitet: "500 g",
enhet: "g",
kategori: "Mejeri",
bastaFore: null,
konfidens: 0.95,
},
],
imageQualityIssues: [],
}),
},
],
},
},
],
usageMetadata: { promptTokenCount: 392, candidatesTokenCount: 121 },
};
function makeFetch(imageBytes: Buffer): typeof fetch {
return async (url: string | URL | Request, init?: RequestInit) => {
const urlStr = url.toString();
if (urlStr.includes("/mock-s3/")) {
return new Response(imageBytes, { status: 200, headers: { "content-type": "image/jpeg" } });
}
if (urlStr.includes("/models/") && init?.method === "POST") {
return new Response(JSON.stringify(FIXTURE_RESPONSE), { status: 200, headers: { "content-type": "application/json" } });
}
return new Response("not found", { status: 404 });
};
}
// A tiny 1x1 JPEG.
const JPEG_BYTES = Buffer.from(
"/9j/4AAQSkZJRgABAQEASABIAAD/2wBDAP//////////////////////////////////////////////////////////////////////////////////////wAALCAABAAEBAREA/8QAFAABAAAAAAAAAAAAAAAAAAAAA//EABQQAQAAAAAAAAAAAAAAAAAAAAD/2gAIAQEAAD8Af//Z",
"base64",
);
describe("GeminiAamosClient", () => {
it("analyzes fridge image with recorded fixture", async () => {
const budget = new MemoryBudgetStore();
const client = new GeminiAamosClient({
apiKey: "test-key",
model: "gemini-2.5-flash",
fetchImpl: makeFetch(JPEG_BYTES),
budgetStore: budget,
dailyBudgetUsd: 10,
});
const result = await client.runTask("ANALYZE_FRIDGE_IMAGE", {
imageUrls: ["http://localhost/v1/mock-s3/fridge.jpg"],
locationType: "fridge",
marketLocale: "sv-SE",
knownItems: [],
});
expect(result.status).toBe("ok");
expect(result.output).not.toBeNull();
expect(result.modelVersion).toBe("gemini-2.5-flash");
expect(result.costUsd).toBeGreaterThan(0);
expect(result.inputTokens).toBe(392);
expect(result.outputTokens).toBe(121);
expect(result.output).not.toBeNull();
const items = result.output!.items;
expect(items).toHaveLength(2);
expect(items[0]?.detectedName).toBe("Mellanmjölk 1,5% fett");
expect(items[0]?.brand).toBe("Arla");
expect(items[0]?.estimatedQuantity).toBe(1);
expect(items[0]?.unit).toBe("LITER");
expect(items[0]?.confidence).toBe(0.98);
expect(items[1]?.detectedName).toBe("Smör");
expect(items[1]?.estimatedQuantity).toBe(500);
expect(items[1]?.unit).toBe("GRAM");
const spend = await budget.getDailySpendUsd();
expect(spend).toBe(result.costUsd);
});
it("blocks call when daily budget exceeded", async () => {
const budget = new MemoryBudgetStore();
const client = new GeminiAamosClient({
apiKey: "test-key",
fetchImpl: makeFetch(JPEG_BYTES),
budgetStore: budget,
dailyBudgetUsd: 0.00001, // essentially zero
});
const result = await client.runTask("ANALYZE_FRIDGE_IMAGE", {
imageUrls: ["http://localhost/v1/mock-s3/fridge.jpg"],
locationType: "fridge",
marketLocale: "sv-SE",
knownItems: [],
});
expect(result.status).toBe("failed");
expect(result.error).toContain("budget");
});
it("rejects unsupported task types", async () => {
const client = new GeminiAamosClient({
apiKey: "test-key",
fetchImpl: makeFetch(JPEG_BYTES),
});
const result = await client.runTask(
"READ_RECEIPT",
{ imageUrls: ["http://localhost/v1/mock-s3/receipt.jpg"], marketLocale: "sv-SE" },
);
expect(result.status).toBe("failed");
expect(result.error).toContain("READ_RECEIPT");
});
it("validates input before calling Gemini", async () => {
const client = new GeminiAamosClient({
apiKey: "test-key",
fetchImpl: makeFetch(JPEG_BYTES),
});
const result = await client.runTask("ANALYZE_FRIDGE_IMAGE", {
imageUrls: [],
locationType: "fridge",
marketLocale: "sv-SE",
knownItems: [],
} as unknown as { imageUrls: string[]; locationType: string; marketLocale: string; knownItems: string[] });
expect(result.status).toBe("failed");
expect(result.error).toContain("Kontraktsfel");
});
});
@@ -1,4 +1,5 @@
import { import {
bigint,
boolean, boolean,
index, index,
integer, integer,
@@ -121,6 +122,7 @@ export const aiUsageCounters = pgTable(
aiScans: integer("ai_scans").notNull().default(0), aiScans: integer("ai_scans").notNull().default(0),
aiTokensIn: integer("ai_tokens_in").notNull().default(0), aiTokensIn: integer("ai_tokens_in").notNull().default(0),
aiTokensOut: integer("ai_tokens_out").notNull().default(0), aiTokensOut: integer("ai_tokens_out").notNull().default(0),
aiCostUsdMicrocents: bigint("ai_cost_usd_microcents", { mode: "number" }).notNull().default(0),
updatedAt: updatedAt(), updatedAt: updatedAt(),
}, },
(t) => [uniqueIndex("ai_usage_user_month_unique").on(t.userId, t.month)], (t) => [uniqueIndex("ai_usage_user_month_unique").on(t.userId, t.month)],