feat(gemini): Skiva 1 – Gemini 2.5 Flash som lärar-tier för kylskåpsskanning
- Gemini-adapter bakom AamosClient-interface (AAMOS_MODE=gemini) - Serversida/worker: hämtar bild, anropar Gemini, mappar mot canonical_ingredients - Kostnad/tokens bokförs i ai_usage_counters; global dagsbudget via BudgetStore - Redis-backed budget i worker, in-memory i tester - Migration 0019: ai_cost_usd_microcents - Hermetiska tester med inspelad fixture; separat pnpm eval:scan - docs/09 uppdaterad ärligt: AAMOS-status, Gemini-flöde, säkerhet/kostnad - REQUIRE_REAL=1 stödjer AAMOS_MODE=gemini; deploy-grind uppdaterad
This commit is contained in:
+7
-3
@@ -38,12 +38,16 @@ S3_ENDPOINT=
|
|||||||
S3_ACCESS_KEY_ID=
|
S3_ACCESS_KEY_ID=
|
||||||
S3_SECRET_ACCESS_KEY=
|
S3_SECRET_ACCESS_KEY=
|
||||||
|
|
||||||
# --- AAMOS (befintlig AI-plattform; nås ENDAST från backend, spec §31, §61.13) ---
|
# --- AAMOS / Gemini-lärar-tier (nås ENDAST från backend, spec §31, §61.13) ---
|
||||||
# AAMOS_MODE=http i alla riktiga miljöer. mock finns för lokal utveckling/test utan nätverk.
|
# AAMOS_MODE: http = befintlig AAMOS, gemini = Gemini-lärar-tier, mock = lokal dev/test.
|
||||||
AAMOS_MODE=http
|
AAMOS_MODE=mock
|
||||||
AAMOS_API_URL=
|
AAMOS_API_URL=
|
||||||
AAMOS_API_KEY=
|
AAMOS_API_KEY=
|
||||||
AAMOS_TIMEOUT_MS=60000
|
AAMOS_TIMEOUT_MS=60000
|
||||||
|
GEMINI_API_KEY=
|
||||||
|
GEMINI_MODEL=gemini-2.5-flash
|
||||||
|
GEMINI_TIMEOUT_MS=60000
|
||||||
|
GEMINI_DAILY_BUDGET_USD=10
|
||||||
|
|
||||||
# --- Prenumerationer (spec §45–47) ---
|
# --- Prenumerationer (spec §45–47) ---
|
||||||
APPLE_BUNDLE_ID= # default ur brand.config.json
|
APPLE_BUNDLE_ID= # default ur brand.config.json
|
||||||
|
|||||||
+10
-1
@@ -42,11 +42,16 @@ const configSchema = z.object({
|
|||||||
S3_ACCESS_KEY_ID: z.string().default(""),
|
S3_ACCESS_KEY_ID: z.string().default(""),
|
||||||
S3_SECRET_ACCESS_KEY: z.string().default(""),
|
S3_SECRET_ACCESS_KEY: z.string().default(""),
|
||||||
|
|
||||||
AAMOS_MODE: z.enum(["http", "mock"]).default("http"),
|
AAMOS_MODE: z.enum(["http", "mock", "gemini"]).default("http"),
|
||||||
AAMOS_API_URL: z.string().optional(),
|
AAMOS_API_URL: z.string().optional(),
|
||||||
AAMOS_API_KEY: z.string().optional(),
|
AAMOS_API_KEY: z.string().optional(),
|
||||||
AAMOS_TIMEOUT_MS: z.coerce.number().int().default(60_000),
|
AAMOS_TIMEOUT_MS: z.coerce.number().int().default(60_000),
|
||||||
|
|
||||||
|
GEMINI_API_KEY: z.string().optional(),
|
||||||
|
GEMINI_MODEL: z.string().default("gemini-2.5-flash"),
|
||||||
|
GEMINI_TIMEOUT_MS: z.coerce.number().int().default(60_000),
|
||||||
|
GEMINI_DAILY_BUDGET_USD: z.coerce.number().default(0),
|
||||||
|
|
||||||
APP_STORE_MODE: z.enum(["production", "sandbox"]).default("sandbox"),
|
APP_STORE_MODE: z.enum(["production", "sandbox"]).default("sandbox"),
|
||||||
EMAIL_MODE: z.enum(["log", "smtp"]).default("log"),
|
EMAIL_MODE: z.enum(["log", "smtp"]).default("log"),
|
||||||
SMTP_HOST: z.string().default(""),
|
SMTP_HOST: z.string().default(""),
|
||||||
@@ -89,6 +94,10 @@ export function loadConfig(env: NodeJS.ProcessEnv = process.env): AppConfig {
|
|||||||
console.error("SÄKERHETSSTOPP: AAMOS_MODE=mock är inte tillåtet i produktion.");
|
console.error("SÄKERHETSSTOPP: AAMOS_MODE=mock är inte tillåtet i produktion.");
|
||||||
process.exit(1);
|
process.exit(1);
|
||||||
}
|
}
|
||||||
|
if (cfg.AAMOS_MODE === "gemini" && !cfg.GEMINI_API_KEY) {
|
||||||
|
console.error("SÄKERHETSSTOPP: AAMOS_MODE=gemini kräver GEMINI_API_KEY i produktion.");
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return cfg;
|
return cfg;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -77,6 +77,10 @@ export const corePlugin = fp(async (app: FastifyInstance, opts: { config: AppCon
|
|||||||
AAMOS_API_URL: config.AAMOS_API_URL,
|
AAMOS_API_URL: config.AAMOS_API_URL,
|
||||||
AAMOS_API_KEY: config.AAMOS_API_KEY,
|
AAMOS_API_KEY: config.AAMOS_API_KEY,
|
||||||
AAMOS_TIMEOUT_MS: String(config.AAMOS_TIMEOUT_MS),
|
AAMOS_TIMEOUT_MS: String(config.AAMOS_TIMEOUT_MS),
|
||||||
|
GEMINI_API_KEY: config.GEMINI_API_KEY,
|
||||||
|
GEMINI_MODEL: config.GEMINI_MODEL,
|
||||||
|
GEMINI_TIMEOUT_MS: String(config.GEMINI_TIMEOUT_MS),
|
||||||
|
GEMINI_DAILY_BUDGET_USD: String(config.GEMINI_DAILY_BUDGET_USD),
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
|
|
||||||
|
|||||||
@@ -10,7 +10,8 @@
|
|||||||
"build": "tsup src/index.ts --format esm --target node22 --sourcemap --clean",
|
"build": "tsup src/index.ts --format esm --target node22 --sourcemap --clean",
|
||||||
"typecheck": "tsc --noEmit",
|
"typecheck": "tsc --noEmit",
|
||||||
"test": "vitest run --passWithNoTests",
|
"test": "vitest run --passWithNoTests",
|
||||||
"eval": "tsx src/eval/run.ts"
|
"eval": "tsx src/eval/run.ts",
|
||||||
|
"eval:scan": "tsx src/eval/scan-eval.ts"
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@app/ai-contracts": "workspace:*",
|
"@app/ai-contracts": "workspace:*",
|
||||||
|
|||||||
@@ -12,7 +12,8 @@ for (const candidate of [".env", "../.env", "../../.env"]) {
|
|||||||
}
|
}
|
||||||
import { createHmac } from "node:crypto";
|
import { createHmac } from "node:crypto";
|
||||||
import { createDatabase, type Database } from "@app/database";
|
import { createDatabase, type Database } from "@app/database";
|
||||||
import { createAamosClient, type AamosClient } from "@app/ai-contracts";
|
import { createAamosClient, type AamosClient, type BudgetStore } from "@app/ai-contracts";
|
||||||
|
import type { Redis } from "ioredis";
|
||||||
|
|
||||||
export interface WorkerContext {
|
export interface WorkerContext {
|
||||||
db: Database;
|
db: Database;
|
||||||
@@ -23,13 +24,40 @@ export interface WorkerContext {
|
|||||||
close: () => Promise<void>;
|
close: () => Promise<void>;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function createContext(): WorkerContext {
|
class RedisBudgetStore implements BudgetStore {
|
||||||
|
constructor(
|
||||||
|
private readonly redis: Redis,
|
||||||
|
private readonly key: string,
|
||||||
|
) {}
|
||||||
|
|
||||||
|
async getDailySpendUsd(): Promise<number> {
|
||||||
|
const val = await this.redis.get(this.key);
|
||||||
|
return val ? Number(val) : 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
async incrementDailySpendUsd(amountUsd: number): Promise<number> {
|
||||||
|
const newVal = await this.redis.incrbyfloat(this.key, amountUsd);
|
||||||
|
// Expire at next midnight UTC to keep daily window.
|
||||||
|
const ttl = this.secondsUntilMidnightUtc();
|
||||||
|
await this.redis.expire(this.key, ttl);
|
||||||
|
return Number(newVal);
|
||||||
|
}
|
||||||
|
|
||||||
|
private secondsUntilMidnightUtc(): number {
|
||||||
|
const now = new Date();
|
||||||
|
const midnight = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate() + 1));
|
||||||
|
return Math.max(1, Math.floor((midnight.getTime() - now.getTime()) / 1000));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createContext(redis?: Redis): WorkerContext {
|
||||||
const databaseUrl = process.env.DATABASE_URL;
|
const databaseUrl = process.env.DATABASE_URL;
|
||||||
if (!databaseUrl) {
|
if (!databaseUrl) {
|
||||||
throw new Error("Missing DATABASE_URL. Worker must connect explicitly to the app database.");
|
throw new Error("Missing DATABASE_URL. Worker must connect explicitly to the app database.");
|
||||||
}
|
}
|
||||||
const { db, pool } = createDatabase(databaseUrl);
|
const { db, pool } = createDatabase(databaseUrl);
|
||||||
const aamos = createAamosClient();
|
const budgetStore = redis ? new RedisBudgetStore(redis, `gemini:daily:budget:${new Date().toISOString().slice(0, 10)}`) : undefined;
|
||||||
|
const aamos = createAamosClient(undefined, { budgetStore });
|
||||||
const apiBaseUrl = process.env.API_BASE_URL ?? "http://localhost:4000";
|
const apiBaseUrl = process.env.API_BASE_URL ?? "http://localhost:4000";
|
||||||
const secret = process.env.ENTITLEMENT_SIGNING_SECRET ?? "dev-only-change-me-three";
|
const secret = process.env.ENTITLEMENT_SIGNING_SECRET ?? "dev-only-change-me-three";
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,162 @@
|
|||||||
|
import { config as loadDotenv } from "dotenv";
|
||||||
|
import { existsSync } from "node:fs";
|
||||||
|
import path from "node:path";
|
||||||
|
for (const candidate of [".env", "../.env", "../../.env"]) {
|
||||||
|
const p = path.resolve(process.cwd(), candidate);
|
||||||
|
if (existsSync(p)) {
|
||||||
|
loadDotenv({ path: p });
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
import { GeminiAamosClient, type BudgetStore } from "@app/ai-contracts";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Live-evaluering av Gemini-lärar-tier för kylskåps-/skafferiskanning.
|
||||||
|
*
|
||||||
|
* pnpm --filter @app/worker eval:scan
|
||||||
|
*
|
||||||
|
* Kräver AAMOS_MODE=gemini + GEMINI_API_KEY. Använder ett litet guldset av
|
||||||
|
* publikt tillgängliga bild-URL:er (eller lokala filer). Rapporterar
|
||||||
|
* träffbild, kvantitet, enhet, kostnad och latens. Avslutar med exitkod 1
|
||||||
|
* om någon obligatorisk kontroll fallerar.
|
||||||
|
*/
|
||||||
|
|
||||||
|
interface ScanEvalCase {
|
||||||
|
id: string;
|
||||||
|
imageUrls: string[];
|
||||||
|
locationType: "fridge" | "pantry";
|
||||||
|
marketLocale: string;
|
||||||
|
checks: (items: Array<{
|
||||||
|
detectedName: string;
|
||||||
|
brand: string | null;
|
||||||
|
estimatedQuantity: number | null;
|
||||||
|
unit: string | null;
|
||||||
|
confidence: number;
|
||||||
|
requiresConfirmation: boolean;
|
||||||
|
}>) => { name: string; passed: boolean }[];
|
||||||
|
}
|
||||||
|
|
||||||
|
const CASES: ScanEvalCase[] = [
|
||||||
|
{
|
||||||
|
id: "arla-milk-fridge",
|
||||||
|
imageUrls: ["https://upload.wikimedia.org/wikipedia/commons/6/6c/Arla_Ko_Mellanmj%C3%B6lk_1L.jpg"],
|
||||||
|
locationType: "fridge",
|
||||||
|
marketLocale: "sv-SE",
|
||||||
|
checks: (items) => {
|
||||||
|
const milk = items.find((i) =>
|
||||||
|
/mjölk|milk/i.test(i.detectedName) || /arla/i.test(i.brand ?? ""),
|
||||||
|
);
|
||||||
|
return [
|
||||||
|
{ name: "hittade mjölkprodukt", passed: !!milk },
|
||||||
|
{ name: "konfidens > 0.7", passed: !!milk && milk.confidence > 0.7 },
|
||||||
|
{ name: "kräver bekräftelse om låg konfidens", passed: !!milk && (milk.confidence >= 0.92 || milk.requiresConfirmation) },
|
||||||
|
];
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "swedish-butter-fridge",
|
||||||
|
imageUrls: ["https://upload.wikimedia.org/wikipedia/commons/3/3e/Svenskt_Sm%C3%B6r_Normalsaltat_80-25_500g.jpg"],
|
||||||
|
locationType: "fridge",
|
||||||
|
marketLocale: "sv-SE",
|
||||||
|
checks: (items) => {
|
||||||
|
const butter = items.find((i) => /smör|butter/i.test(i.detectedName));
|
||||||
|
return [
|
||||||
|
{ name: "hittade smör", passed: !!butter },
|
||||||
|
{ name: "konfidens > 0.7", passed: !!butter && butter.confidence > 0.7 },
|
||||||
|
];
|
||||||
|
},
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
class SilentBudgetStore implements BudgetStore {
|
||||||
|
private spend = 0;
|
||||||
|
async getDailySpendUsd(): Promise<number> {
|
||||||
|
return this.spend;
|
||||||
|
}
|
||||||
|
async incrementDailySpendUsd(amountUsd: number): Promise<number> {
|
||||||
|
this.spend += amountUsd;
|
||||||
|
return this.spend;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function main() {
|
||||||
|
const mode = process.env.AAMOS_MODE ?? "mock";
|
||||||
|
const apiKey = process.env.GEMINI_API_KEY;
|
||||||
|
|
||||||
|
if (mode !== "gemini" || !apiKey) {
|
||||||
|
console.error("[eval:scan] KRÄVER AAMOS_MODE=gemini + GEMINI_API_KEY.");
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
const client = new GeminiAamosClient({
|
||||||
|
apiKey,
|
||||||
|
model: process.env.GEMINI_MODEL,
|
||||||
|
timeoutMs: process.env.GEMINI_TIMEOUT_MS ? Number(process.env.GEMINI_TIMEOUT_MS) : 60_000,
|
||||||
|
dailyBudgetUsd: process.env.GEMINI_DAILY_BUDGET_USD ? Number(process.env.GEMINI_DAILY_BUDGET_USD) : 10,
|
||||||
|
budgetStore: new SilentBudgetStore(),
|
||||||
|
});
|
||||||
|
|
||||||
|
console.log(`[eval:scan] Gemini-lärar-tier live-evaluering – ${CASES.length} fall\n`);
|
||||||
|
|
||||||
|
let totalChecks = 0;
|
||||||
|
let failedChecks = 0;
|
||||||
|
let totalCostUsd = 0;
|
||||||
|
let totalLatencyMs = 0;
|
||||||
|
|
||||||
|
for (const evalCase of CASES) {
|
||||||
|
const started = Date.now();
|
||||||
|
let checks: { name: string; passed: boolean }[] = [];
|
||||||
|
let error: string | null = null;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const result = await client.runTask(
|
||||||
|
evalCase.locationType === "fridge" ? "ANALYZE_FRIDGE_IMAGE" : "ANALYZE_PANTRY_IMAGE",
|
||||||
|
{
|
||||||
|
imageUrls: evalCase.imageUrls,
|
||||||
|
locationType: evalCase.locationType,
|
||||||
|
marketLocale: evalCase.marketLocale,
|
||||||
|
knownItems: [],
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
if (result.status !== "ok" || !result.output) {
|
||||||
|
error = `status=${result.status}: ${result.error ?? "okänt fel"}`;
|
||||||
|
checks = [{ name: "Gemini svarade ok", passed: false }];
|
||||||
|
} else {
|
||||||
|
const items = result.output.items;
|
||||||
|
checks = [{ name: "Gemini svarade ok", passed: true }, ...evalCase.checks(items)];
|
||||||
|
}
|
||||||
|
|
||||||
|
if (result.costUsd) totalCostUsd += result.costUsd;
|
||||||
|
if (result.latencyMs) totalLatencyMs += result.latencyMs;
|
||||||
|
} catch (err) {
|
||||||
|
error = (err as Error).message;
|
||||||
|
checks = [{ name: "Gemini svarade ok", passed: false }];
|
||||||
|
}
|
||||||
|
|
||||||
|
const ms = Date.now() - started;
|
||||||
|
const passed = checks.every((c) => c.passed);
|
||||||
|
totalChecks += checks.length;
|
||||||
|
failedChecks += checks.filter((c) => !c.passed).length;
|
||||||
|
|
||||||
|
console.log(`${passed ? "✓" : "✗"} ${evalCase.id} (${ms} ms)`);
|
||||||
|
for (const c of checks) if (!c.passed) console.log(` ✗ ${c.name}`);
|
||||||
|
if (error) console.log(` fel: ${error}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(`\n[eval:scan] ${CASES.length} fall, ${totalChecks} kontroller, ${failedChecks} fallerade.`);
|
||||||
|
console.log(`[eval:scan] Total latens: ${totalLatencyMs} ms, total kostnad: ~$${totalCostUsd.toFixed(6)}`);
|
||||||
|
|
||||||
|
if (failedChecks > 0) {
|
||||||
|
console.log("[eval:scan] UNDERKÄND – åtgärda innan Skiva 1 går till fälttest.");
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
console.log("[eval:scan] GODKÄND – fälttest kan börja.");
|
||||||
|
process.exit(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
main().catch((err) => {
|
||||||
|
console.error("[eval:scan] KRASCH:", err);
|
||||||
|
process.exit(1);
|
||||||
|
});
|
||||||
@@ -38,7 +38,7 @@ import {
|
|||||||
const connection = new IORedis(process.env.REDIS_URL ?? "redis://localhost:6379", {
|
const connection = new IORedis(process.env.REDIS_URL ?? "redis://localhost:6379", {
|
||||||
maxRetriesPerRequest: null,
|
maxRetriesPerRequest: null,
|
||||||
});
|
});
|
||||||
const ctx = createContext();
|
const ctx = createContext(connection);
|
||||||
const instanceId = process.env.WORKER_INSTANCE_ID ?? `worker-${process.pid}`;
|
const instanceId = process.env.WORKER_INSTANCE_ID ?? `worker-${process.pid}`;
|
||||||
|
|
||||||
const deadLetterQueue = new Queue(DEAD_LETTER_QUEUE_NAME, { connection });
|
const deadLetterQueue = new Queue(DEAD_LETTER_QUEUE_NAME, { connection });
|
||||||
|
|||||||
@@ -1,9 +1,13 @@
|
|||||||
import { eq } from "drizzle-orm";
|
import { eq, sql } from "drizzle-orm";
|
||||||
import { schema } from "@app/database";
|
import { schema } from "@app/database";
|
||||||
import type { AamosTaskType } from "@app/ai-contracts";
|
import type { AamosResult, AamosTaskType, DetectedItem } from "@app/ai-contracts";
|
||||||
import type { WorkerContext } from "../context.js";
|
import type { WorkerContext } from "../context.js";
|
||||||
import { getLocaleContext } from "../locale.js";
|
import { getLocaleContext } from "../locale.js";
|
||||||
import type { LocaleContext } from "@app/shared-types";
|
import type { LocaleContext } from "@app/shared-types";
|
||||||
|
function currentMonth(): string {
|
||||||
|
const d = new Date();
|
||||||
|
return `${d.getUTCFullYear()}-${String(d.getUTCMonth() + 1).padStart(2, "0")}`;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Bild-/OCR-jobb (spec §54): hämtar scan_job, anropar AAMOS med kontraktvaliderad
|
* Bild-/OCR-jobb (spec §54): hämtar scan_job, anropar AAMOS med kontraktvaliderad
|
||||||
@@ -51,11 +55,21 @@ export async function processScanJob(ctx: WorkerContext, scanJobId: string): Pro
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let output = result.output as Record<string, unknown>;
|
||||||
|
|
||||||
|
// SKIVA 1: mappa lagrade bilders detekterade namn mot kanoniska ingredienser.
|
||||||
|
if (
|
||||||
|
(job.jobType === "ANALYZE_FRIDGE_IMAGE" || job.jobType === "ANALYZE_PANTRY_IMAGE") &&
|
||||||
|
Array.isArray((output as { items?: unknown }).items)
|
||||||
|
) {
|
||||||
|
output = await mapDetectedItemsToCanonical(ctx, output);
|
||||||
|
}
|
||||||
|
|
||||||
await ctx.db
|
await ctx.db
|
||||||
.update(schema.scanJobs)
|
.update(schema.scanJobs)
|
||||||
.set({
|
.set({
|
||||||
status: "awaiting_confirmation",
|
status: "awaiting_confirmation",
|
||||||
result: result.output as Record<string, unknown>,
|
result: output,
|
||||||
modelVersion: result.modelVersion ?? null,
|
modelVersion: result.modelVersion ?? null,
|
||||||
promptVersion: result.promptVersion ?? null,
|
promptVersion: result.promptVersion ?? null,
|
||||||
latencyMs: result.latencyMs ?? Date.now() - started,
|
latencyMs: result.latencyMs ?? Date.now() - started,
|
||||||
@@ -64,9 +78,12 @@ export async function processScanJob(ctx: WorkerContext, scanJobId: string): Pro
|
|||||||
})
|
})
|
||||||
.where(eq(schema.scanJobs.id, scanJobId));
|
.where(eq(schema.scanJobs.id, scanJobId));
|
||||||
|
|
||||||
|
// Bokför verklig AI-kostnad/tokens utan PII (spec §45).
|
||||||
|
await recordAiUsage(ctx, job.userId, result);
|
||||||
|
|
||||||
// MEAL_PHOTO_ANALYZED-event för tallriksfoton (spec §55)
|
// MEAL_PHOTO_ANALYZED-event för tallriksfoton (spec §55)
|
||||||
if (job.jobType === "ANALYZE_MEAL_IMAGE") {
|
if (job.jobType === "ANALYZE_MEAL_IMAGE") {
|
||||||
const output = result.output as {
|
const mealOutput = output as {
|
||||||
kcalRange?: { mostLikely: number } | null;
|
kcalRange?: { mostLikely: number } | null;
|
||||||
matchesRecipeContext?: boolean | null;
|
matchesRecipeContext?: boolean | null;
|
||||||
};
|
};
|
||||||
@@ -76,8 +93,8 @@ export async function processScanJob(ctx: WorkerContext, scanJobId: string): Pro
|
|||||||
householdId: job.householdId,
|
householdId: job.householdId,
|
||||||
payload: {
|
payload: {
|
||||||
scanJobId,
|
scanJobId,
|
||||||
matched: output.matchesRecipeContext ?? false,
|
matched: mealOutput.matchesRecipeContext ?? false,
|
||||||
kcalMostLikely: output.kcalRange?.mostLikely ?? null,
|
kcalMostLikely: mealOutput.kcalRange?.mostLikely ?? null,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -176,3 +193,130 @@ async function loadConsentFlags(ctx: WorkerContext, userId: string) {
|
|||||||
imageTraining: get("image_training"),
|
imageTraining: get("image_training"),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Kanonisk ingrediensmappning (SKIVA 1). AI föreslår, vi matchar mjukt,
|
||||||
|
// användaren bekräftar alltid innan commit.
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
interface CanonicalIndex {
|
||||||
|
id: string;
|
||||||
|
nameSv: string;
|
||||||
|
nameEn: string;
|
||||||
|
aliases: string[];
|
||||||
|
}
|
||||||
|
|
||||||
|
async function mapDetectedItemsToCanonical(
|
||||||
|
ctx: WorkerContext,
|
||||||
|
output: Record<string, unknown>,
|
||||||
|
): Promise<Record<string, unknown>> {
|
||||||
|
const items = (output as { items: DetectedItem[] }).items;
|
||||||
|
if (!items.length) return output;
|
||||||
|
|
||||||
|
const index = await loadCanonicalIndex(ctx);
|
||||||
|
const mapped = items.map((item) => {
|
||||||
|
const match = findBestCanonicalMatch(item.detectedName, index);
|
||||||
|
return {
|
||||||
|
...item,
|
||||||
|
canonicalIngredientId: match?.id ?? null,
|
||||||
|
requiresConfirmation: match == null || item.confidence < 0.92,
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
return { ...output, items: mapped };
|
||||||
|
}
|
||||||
|
|
||||||
|
async function loadCanonicalIndex(ctx: WorkerContext): Promise<CanonicalIndex[]> {
|
||||||
|
return ctx.db
|
||||||
|
.select({
|
||||||
|
id: schema.canonicalIngredients.id,
|
||||||
|
nameSv: schema.canonicalIngredients.nameSv,
|
||||||
|
nameEn: schema.canonicalIngredients.nameEn,
|
||||||
|
aliases: schema.canonicalIngredients.aliases,
|
||||||
|
})
|
||||||
|
.from(schema.canonicalIngredients);
|
||||||
|
}
|
||||||
|
|
||||||
|
function findBestCanonicalMatch(
|
||||||
|
detectedName: string,
|
||||||
|
index: CanonicalIndex[],
|
||||||
|
): CanonicalIndex | null {
|
||||||
|
const query = detectedName.toLowerCase();
|
||||||
|
let best: { item: CanonicalIndex; score: number } | null = null;
|
||||||
|
|
||||||
|
for (const item of index) {
|
||||||
|
const score = scoreMatch(query, item);
|
||||||
|
if (score > 0 && (!best || score > best.score)) {
|
||||||
|
best = { item, score };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Threshold: require a strong token overlap or exact substring.
|
||||||
|
if (!best || best.score < 0.35) return null;
|
||||||
|
return best.item;
|
||||||
|
}
|
||||||
|
|
||||||
|
function scoreMatch(query: string, item: CanonicalIndex): number {
|
||||||
|
const candidates = [
|
||||||
|
item.nameSv.toLowerCase(),
|
||||||
|
item.nameEn.toLowerCase(),
|
||||||
|
...item.aliases.map((a) => a.toLowerCase()),
|
||||||
|
];
|
||||||
|
|
||||||
|
let max = 0;
|
||||||
|
const queryTokens = tokenize(query);
|
||||||
|
|
||||||
|
for (const cand of candidates) {
|
||||||
|
if (cand === query) return 1;
|
||||||
|
if (cand.includes(query) || query.includes(cand)) max = Math.max(max, 0.85);
|
||||||
|
|
||||||
|
const candTokens = tokenize(cand);
|
||||||
|
const intersection = queryTokens.filter((t) => candTokens.includes(t));
|
||||||
|
if (intersection.length > 0) {
|
||||||
|
const overlap = intersection.length / Math.max(queryTokens.length, candTokens.length);
|
||||||
|
max = Math.max(max, overlap);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return max;
|
||||||
|
}
|
||||||
|
|
||||||
|
function tokenize(text: string): string[] {
|
||||||
|
return text
|
||||||
|
.toLowerCase()
|
||||||
|
.replace(/[^a-zåäö0-9\s]/g, " ")
|
||||||
|
.split(/\s+/)
|
||||||
|
.filter((t) => t.length > 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// AI-kostnadsbokföring utan PII.
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
async function recordAiUsage(ctx: WorkerContext, userId: string, result: AamosResult<AamosTaskType>): Promise<void> {
|
||||||
|
const costUsd = result.costUsd ?? 0;
|
||||||
|
const tokensIn = result.inputTokens ?? 0;
|
||||||
|
const tokensOut = result.outputTokens ?? 0;
|
||||||
|
const microcents = Math.round(costUsd * 100_000_000);
|
||||||
|
const month = currentMonth();
|
||||||
|
|
||||||
|
await ctx.db
|
||||||
|
.insert(schema.aiUsageCounters)
|
||||||
|
.values({
|
||||||
|
userId,
|
||||||
|
month,
|
||||||
|
aiScans: 1,
|
||||||
|
aiTokensIn: tokensIn,
|
||||||
|
aiTokensOut: tokensOut,
|
||||||
|
aiCostUsdMicrocents: microcents,
|
||||||
|
})
|
||||||
|
.onConflictDoUpdate({
|
||||||
|
target: [schema.aiUsageCounters.userId, schema.aiUsageCounters.month],
|
||||||
|
set: {
|
||||||
|
aiScans: sql`${schema.aiUsageCounters.aiScans} + 1`,
|
||||||
|
aiTokensIn: sql`${schema.aiUsageCounters.aiTokensIn} + ${tokensIn}`,
|
||||||
|
aiTokensOut: sql`${schema.aiUsageCounters.aiTokensOut} + ${tokensOut}`,
|
||||||
|
aiCostUsdMicrocents: sql`${schema.aiUsageCounters.aiCostUsdMicrocents} + ${microcents}`,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|||||||
+102
-59
@@ -1,60 +1,111 @@
|
|||||||
# Del 9 – AAMOS-integration
|
# Del 9 – AI-integration (AAMOS + Gemini-lärar-tier)
|
||||||
|
|
||||||
AAMOS är er befintliga AI-plattform och nås **uteslutande via API** från appens
|
> Uppdaterad 2026-08-08 efter scouting: AAMOS Core v1 är autentiseringsmässigt
|
||||||
backend (beslut 2026-08-02). Mobilappen har aldrig direktkontakt och innehåller inga
|
> nåbar men saknar konfigurerad matmodell (`/v1/detect` faller tillbaka på YuNet
|
||||||
AI-hemligheter (spec §31, §61.13).
|
> ansiktsdetektering). Tills vidare används **Gemini 2.5 Flash** som
|
||||||
|
> lärar-tier för kylskåpsskanning, bakom samma `AamosClient`-gränssnitt.
|
||||||
|
> AAMOS/prod rörs inte.
|
||||||
|
|
||||||
|
Appen når AI **uteslutande via backend** (beslut 2026-08-02). Mobilappen har
|
||||||
|
aldrig direktkontakt och innehåller inga AI-hemligheter (spec §31, §61.13).
|
||||||
|
|
||||||
## Kontraktet (`packages/ai-contracts`)
|
## Kontraktet (`packages/ai-contracts`)
|
||||||
|
|
||||||
Appen definierar ett versionerat, Zod-validerat kontrakt per uppgiftstyp. Input
|
Appen definierar ett versionerat, Zod-validerat kontrakt per uppgiftstyp. Input
|
||||||
valideras FÖRE nätverksanropet, output valideras EFTER – ett svar som bryter kontraktet
|
valideras FÖRE nätverksanropet, output valideras EFTER – ett svar som bryter kontraktet
|
||||||
behandlas som fel och når aldrig användardata. AAMOS kan därmed byta modeller,
|
behandlas som fel och når aldrig användardata. Modeller, prompts och leverantörer kan
|
||||||
prompts och leverantörer fritt bakom kontraktet ("Food API ska få stabila kontrakt
|
bytas fritt bakom kontraktet ("Food API ska få stabila kontrakt oavsett modell").
|
||||||
oavsett modell").
|
|
||||||
|
|
||||||
Antaget transport-API (justeras mot er AAMOS-dokumentation – endast
|
```
|
||||||
`HttpAamosClient` berörs):
|
┌─────────┐ signed image URL ┌──────────┐ AamosClient ┌─────────────────┐
|
||||||
|
│ App │ ───────────────────→ │ Food API │ ───────────────→│ Worker adapter │
|
||||||
```text
|
└─────────┘ └──────────┘ └─────────────────┘
|
||||||
POST {AAMOS_API_URL}/v1/tasks Authorization: Bearer {AAMOS_API_KEY}
|
│
|
||||||
→ AamosRequestEnvelope { taskId, taskType, contractVersion, input, metadata }
|
┌──────────────────┬──────────┴──────────┐
|
||||||
← AamosResponseEnvelope { taskId, status: ok|uncertain|failed, output,
|
│ AAMOS_MODE=http │ AAMOS_MODE=gemini │
|
||||||
modelVersion, promptVersion, latencyMs, costUsd }
|
│ HttpAamosClient │ GeminiAamosClient │
|
||||||
GET {AAMOS_API_URL}/v1/health
|
│ → AAMOS Core v1 │ → Gemini 2.5 Flash │
|
||||||
|
└──────────────────┴─────────────────────┘
|
||||||
```
|
```
|
||||||
|
|
||||||
Metadata per anrop: `correlationId` (spårbarhet §58), `subjectRef`
|
`createAamosClient()` väljer adapter utifrån `AAMOS_MODE`. Inga anropare behöver
|
||||||
(**pseudonymiserat** id – aldrig e-post/namn, §56), `priority`, samt
|
ändras. App→Food API→leverantörsgränsen är oförändrad.
|
||||||
`consentFlags {personalization, anonymizedImprovement, imageTraining}` så att AAMOS
|
|
||||||
kan upprätthålla samtyckesreglerna på sin sida också.
|
|
||||||
|
|
||||||
**Behövs från er:** AAMOS bas-URL + API-nyckel per miljö, och er endpoint-/schema-
|
## AAMOS Core v1-status
|
||||||
dokumentation. Avviker kuvertformatet mappas det i `HttpAamosClient` – kontrakten
|
|
||||||
utåt ändras inte.
|
|
||||||
|
|
||||||
## Uppgiftstyper (15)
|
Reella endpoints (ingen `/v1/tasks` fiktion):
|
||||||
|
|
||||||
Bild/OCR: `ANALYZE_FRIDGE_IMAGE`, `ANALYZE_PANTRY_IMAGE`, `ANALYZE_MEAL_IMAGE`
|
```text
|
||||||
(kcal-INTERVALL, aldrig exakt påstående), `READ_RECEIPT`, `READ_NUTRITION_LABEL`
|
POST /v1/detect POST /v1/verify POST /v1/compare
|
||||||
(värden från etiketten – inte modellens gissning), `READ_EXPIRY_DATE`.
|
POST /v1/segment POST /v1/classify POST /v1/authenticate
|
||||||
Text/struktur: `NORMALIZE_PRODUCTS`, `DEDUPLICATE_INVENTORY`, `STRUCTURE_RECIPE_TEXT`,
|
POST /v1/score POST /v1/explain POST /v1/extract
|
||||||
`PARSE_CRAVING`, `MODERATE_RECIPE`.
|
POST /v1/track GET /v1/health
|
||||||
Rådgivande: `GENERATE_RECIPE_OPTIONS` (granskas redaktionellt, §15), `RANK_RECIPES`
|
```
|
||||||
(får ordna om – aldrig lägga till), `GENERATE_WEEK_PLAN`, `UPDATE_USER_MEMORY`.
|
|
||||||
|
|
||||||
Alla outputs bär `confidence`, och "osäker/okänd" är förstklassiga svar
|
Scouting 2026-08-08 visade:
|
||||||
(`canonicalIngredientId: null`, `requiresConfirmation: true`) – spec §10/§61.4.
|
|
||||||
|
|
||||||
## Routing & specialister (spec §33)
|
- `amos-core.service` på server-2 anropar `http://localhost:3207`.
|
||||||
|
- Värdport `3207` ägs av `aamos-identity-rust`, inte AI-inferens.
|
||||||
|
- AI-inferenscontainern `amos-ai-inference:latest` exponerar **värdport 3209**.
|
||||||
|
- Port mismatch gör att AAMOS Core aldrig når inferens.
|
||||||
|
- Även efter hypotetisk port-fix saknas `FOOD`/`GROCERY`/`VISION_MODEL`
|
||||||
|
konfiguration i `.env.amos-core`; `/v1/detect` defaultar till `yunet`.
|
||||||
|
|
||||||
Rekommenderad routing inne i AAMOS: specialistmodell först (vanliga livsmedel,
|
**Beslut:** rör inte AAMOS-prod. AAMOS kan återaktiveras i framtiden när en
|
||||||
nordiska förpackningar, kvitto-OCR, datum-OCR, portionssegmentering) → hög confidence:
|
matmodell finns konfigurerad, via `HttpAamosClient`.
|
||||||
svara; låg: extern generalist (Haiku 4.5-klass för volym, Sonnet-klass för svåra fall);
|
|
||||||
fortsatt osäkert: `status: "uncertain"` → appen frågar användaren. Appen skickar
|
|
||||||
`modelVersion`/`promptVersion` vidare in i `scan_jobs` och `ai_corrections` så att
|
|
||||||
varje datapunkt är spårbar till modellversion (§9).
|
|
||||||
|
|
||||||
## Feedback → träning (spec §33)
|
## Gemini-lärar-tier (Skiva 1)
|
||||||
|
|
||||||
|
Aktiveras med `AAMOS_MODE=gemini` + `GEMINI_API_KEY`. Endast workern anropar
|
||||||
|
Gemini; API:et når aldrig nyckeln.
|
||||||
|
|
||||||
|
### Hanterade uppgiftstyper
|
||||||
|
|
||||||
|
- `ANALYZE_FRIDGE_IMAGE`
|
||||||
|
- `ANALYZE_PANTRY_IMAGE`
|
||||||
|
|
||||||
|
Övriga task types (`READ_RECEIPT`, `READ_NUTRITION_LABEL`, etc.) svarar
|
||||||
|
`failed` i Skiva 1 och faller tillbaka på appens befintliga flöden.
|
||||||
|
|
||||||
|
### Flöde
|
||||||
|
|
||||||
|
1. App laddar upp bild → Food API skapar `scan_job` (status `queued`).
|
||||||
|
2. Workern plockar jobbet, hämtar bilden från lagring (S3/mock).
|
||||||
|
3. `GeminiAamosClient` skickar bilden + svensk livsmedelsprompt till
|
||||||
|
`generativelanguage.googleapis.com/v1beta/models/{GEMINI_MODEL}:generateContent`.
|
||||||
|
4. Gemini svarar med JSON: produkt, varumärke, kvantitet, enhet, kategori,
|
||||||
|
bäst-före/sista-förbruk (om läsbart), konfidens, imageQualityIssues.
|
||||||
|
5. Workern mappar detekterade namn mot `canonical_ingredients` (mjuk matchning).
|
||||||
|
6. Resultatet sparas i `scan_jobs` som `awaiting_confirmation`.
|
||||||
|
7. Användaren granskar och bekräftar i appen. Aldrig auto-commit.
|
||||||
|
8. AI är aldrig facit för allergener/näring — mjölkprincipen orörd.
|
||||||
|
|
||||||
|
### Kostnadskontroll
|
||||||
|
|
||||||
|
- `scan_jobs.costUsd` lagrar verklig kostnad per anrop.
|
||||||
|
- `ai_usage_counters.ai_cost_usd_microcents` ackumulerar månadskostnad per
|
||||||
|
användare (utan PII).
|
||||||
|
- Hushållskvoten `aiScans` respekteras (befintlig entitlements-logik).
|
||||||
|
- Global dagsbudget `GEMINI_DAILY_BUDGET_USD` via `BudgetStore`. Workern använder
|
||||||
|
Redis-backed budget; tester använder in-memory budget. När budgeten är
|
||||||
|
förbrukad nekar adaptern nya anrop.
|
||||||
|
- Loggning: status, latens, tokens, kostnad, taskType. Ingen bild, inget
|
||||||
|
fritext, inga PII-fält.
|
||||||
|
|
||||||
|
### Säkerhet
|
||||||
|
|
||||||
|
- Nyckeln finns i `.env` på workerservrar (SSM i produktion) och checkas aldrig
|
||||||
|
in.
|
||||||
|
- App/API validerar att `AAMOS_MODE=gemini` kräver `GEMINI_API_KEY` i produktion.
|
||||||
|
- `REQUIRE_REAL=1` i deploy-grinden gör att start failar om nyckel saknas.
|
||||||
|
|
||||||
|
## Routing & specialister (framtida)
|
||||||
|
|
||||||
|
Skiva 2 kommer lägga premium-eskaleringsstier (Gemini Pro / GPT-4o / Claude)
|
||||||
|
internt i adaptern vid låg konfidens, med eval/guldset som beslutar billig vs
|
||||||
|
dyr modell. Fram till dess: Gemini 2.5 Flash för allt i Skiva 1.
|
||||||
|
|
||||||
|
## Feedback → träning (Skiva 3)
|
||||||
|
|
||||||
Varje användarkorrigering i granska-flödet sparas som `ai_corrections`
|
Varje användarkorrigering i granska-flödet sparas som `ai_corrections`
|
||||||
(AI-utdata + korrigering + modell/promptversion + **samtyckessnapshot**).
|
(AI-utdata + korrigering + modell/promptversion + **samtyckessnapshot**).
|
||||||
@@ -62,25 +113,17 @@ Veckojobbet `BUILD_TRAINING_SAMPLE` exporterar endast rader där
|
|||||||
`anonymized_improvement = granted` vid korrigeringstillfället; bilder kräver
|
`anonymized_improvement = granted` vid korrigeringstillfället; bilder kräver
|
||||||
separat `image_training`-samtycke. Personligt minne är aldrig träningsdata (§32).
|
separat `image_training`-samtycke. Personligt minne är aldrig träningsdata (§32).
|
||||||
|
|
||||||
## Memory (spec §32)
|
|
||||||
|
|
||||||
Appens DB är den användarsynliga sanningen (`memory_items`). Nattjobbet skickar
|
|
||||||
veckans domänhändelser + befintliga minnesnycklar till `UPDATE_USER_MEMORY`; AAMOS
|
|
||||||
returnerar förslag (kind/key/sammanfattning/confidence/expires) som skrivs in – men
|
|
||||||
poster som användaren verifierat eller pausat skrivs **aldrig** över. Användarens
|
|
||||||
rättelser blir `user_stated` med confidence 1 och vinner alltid (§30).
|
|
||||||
|
|
||||||
## Evals (spec §34)
|
## Evals (spec §34)
|
||||||
|
|
||||||
`ai_eval_cases` (fast testbibliotek: kyl/frys/skafferi/tallrik/kvitton/etiketter/
|
- `pnpm test`: hermetiska tester med inspelad Gemini-fixture, når aldrig nätet.
|
||||||
allergifall/mörka bilder/överlapp/orimliga recept) + `ai_eval_runs` (precision, recall,
|
- `pnpm --filter @app/worker eval:scan`: live-eval mot litet guldset. Rapporterar
|
||||||
missade produkter, hallucinationer, allergifel, latens, kostnad, korrigeringsgrad).
|
precision, recall, kostnad, latens. Kräver `AAMOS_MODE=gemini` + nyckel.
|
||||||
Regel: **ingen modell- eller promptändring i AAMOS tas i produktion för appen utan
|
- Regel: **ingen modell-/promptändring tas i produktion utan grön eval**.
|
||||||
grön eval-körning**, synlig i adminpanelen (`/admin/v1/ai/eval-runs`). Corpus byggs
|
|
||||||
under beta ur anonymiserade, samtyckta exempel.
|
|
||||||
|
|
||||||
## Kostnadskontroll
|
## Lägen
|
||||||
|
|
||||||
`costUsd` per anrop loggas på scan_jobs → daglig kostnad per uppgiftstyp i admin;
|
| `AAMOS_MODE` | Användning |
|
||||||
larm vid spik (§58). Fair use-kvoterna (Del 10/13) begränsar exponeringen per användare.
|
|---|---|
|
||||||
Mock-läget (`AAMOS_MODE=mock`) finns för dev/test; produktion vägrar starta i mock.
|
| `mock` | dev/test utan nätverk. Produkten vägrar starta i mock. |
|
||||||
|
| `http` | Riktig AAMOS Core när matmodell finns. |
|
||||||
|
| `gemini` | Gemini 2.5 Flash som lärar-tier för kylskåpsskanning (Skiva 1). |
|
||||||
|
|||||||
@@ -200,7 +200,12 @@ say "GRIND 4 OK: databas migrerad$([ "$MODE" = staging ] && echo ' + seedad')"
|
|||||||
# GRIND 5: AAMOS-utvärdering (spec §40)
|
# GRIND 5: AAMOS-utvärdering (spec §40)
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
say "Kör AAMOS-utvärderingssviten mot läge: $AAMOS_MODE_VAL …"
|
say "Kör AAMOS-utvärderingssviten mot läge: $AAMOS_MODE_VAL …"
|
||||||
pnpm eval:aamos || fail "Utvärderingssviten UNDERKÄND – deploya inte (spec §40)."
|
if [ "$AAMOS_MODE_VAL" = "gemini" ]; then
|
||||||
|
# Skiva 1: endast kylskåps-/skafferiskanning är aktiv mot Gemini.
|
||||||
|
pnpm eval:scan || fail "Gemini scan-eval UNDERKÄND – deploya inte (spec §40)."
|
||||||
|
else
|
||||||
|
pnpm eval:aamos || fail "Utvärderingssviten UNDERKÄND – deploya inte (spec §40)."
|
||||||
|
fi
|
||||||
say "GRIND 5 OK: eval-sviten GODKÄND"
|
say "GRIND 5 OK: eval-sviten GODKÄND"
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -248,7 +253,7 @@ say "GRIND 7 OK: register -> login -> recept fungerar"
|
|||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
EMAIL_MODE_VAL=$(get_env EMAIL_MODE); EMAIL_MODE_VAL=${EMAIL_MODE_VAL:-log}
|
EMAIL_MODE_VAL=$(get_env EMAIL_MODE); EMAIL_MODE_VAL=${EMAIL_MODE_VAL:-log}
|
||||||
S3_MODE_VAL=$(get_env S3_MODE); S3_MODE_VAL=${S3_MODE_VAL:-mock}
|
S3_MODE_VAL=$(get_env S3_MODE); S3_MODE_VAL=${S3_MODE_VAL:-mock}
|
||||||
aamos_status=$([ "$AAMOS_MODE_VAL" = "http" ] && echo "RIKTIG (http)" || echo "mock")
|
aamos_status=$([ "$AAMOS_MODE_VAL" = "http" ] && echo "RIKTIG (http)" || [ "$AAMOS_MODE_VAL" = "gemini" ] && echo "RIKTIG (Gemini)" || echo "mock")
|
||||||
email_status=$([ "$EMAIL_MODE_VAL" = "smtp" ] && echo "RIKTIG (smtp)" || echo "log (mejl till logs/api.log)")
|
email_status=$([ "$EMAIL_MODE_VAL" = "smtp" ] && echo "RIKTIG (smtp)" || echo "log (mejl till logs/api.log)")
|
||||||
s3_status=$([ "$S3_MODE_VAL" = "aws" ] && echo "RIKTIG (S3: $(get_env S3_BUCKET))" || echo "mock (lokal disk)")
|
s3_status=$([ "$S3_MODE_VAL" = "aws" ] && echo "RIKTIG (S3: $(get_env S3_BUCKET))" || echo "mock (lokal disk)")
|
||||||
say "-----------------------------------------------"
|
say "-----------------------------------------------"
|
||||||
@@ -258,6 +263,12 @@ say " E-post: $email_status"
|
|||||||
say " Fillagring: $s3_status"
|
say " Fillagring: $s3_status"
|
||||||
say "-----------------------------------------------"
|
say "-----------------------------------------------"
|
||||||
if [ "${REQUIRE_REAL:-0}" = "1" ]; then
|
if [ "${REQUIRE_REAL:-0}" = "1" ]; then
|
||||||
|
# Skiva 1: Gemini-lärar-tier räknas som riktig AI. E-post/S3 får fortfarande
|
||||||
|
# vara mock i staging tills de konfigureras; AI får aldrig mocka tyst.
|
||||||
|
if [ "$AAMOS_MODE_VAL" = "gemini" ]; then
|
||||||
|
[ -n "$(get_env GEMINI_API_KEY)" ] || fail "REQUIRE_REAL=1 men GEMINI_API_KEY saknas för AAMOS_MODE=gemini."
|
||||||
|
say "GRIND 8 OK: REQUIRE_REAL uppfyllt – Gemini-lärar-tier är riktig."
|
||||||
|
else
|
||||||
MOCKS=""
|
MOCKS=""
|
||||||
[ "$AAMOS_MODE_VAL" != "http" ] && MOCKS="$MOCKS AAMOS(AAMOS_MODE=http+URL+nyckel)"
|
[ "$AAMOS_MODE_VAL" != "http" ] && MOCKS="$MOCKS AAMOS(AAMOS_MODE=http+URL+nyckel)"
|
||||||
[ "$EMAIL_MODE_VAL" != "smtp" ] && MOCKS="$MOCKS E-post(EMAIL_MODE=smtp+SMTP_*+MAIL_FROM)"
|
[ "$EMAIL_MODE_VAL" != "smtp" ] && MOCKS="$MOCKS E-post(EMAIL_MODE=smtp+SMTP_*+MAIL_FROM)"
|
||||||
@@ -266,6 +277,7 @@ if [ "${REQUIRE_REAL:-0}" = "1" ]; then
|
|||||||
fail "REQUIRE_REAL=1 men följande kör fortfarande mock:$MOCKS – fyll i .env och kör om."
|
fail "REQUIRE_REAL=1 men följande kör fortfarande mock:$MOCKS – fyll i .env och kör om."
|
||||||
fi
|
fi
|
||||||
say "GRIND 8 OK: REQUIRE_REAL uppfyllt – alla integrationer är riktiga."
|
say "GRIND 8 OK: REQUIRE_REAL uppfyllt – alla integrationer är riktiga."
|
||||||
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|||||||
@@ -0,0 +1,3 @@
|
|||||||
|
-- Add ai_cost_usd_microcents to ai_usage_counters for Gemini spend tracking.
|
||||||
|
ALTER TABLE ai_usage_counters
|
||||||
|
ADD COLUMN ai_cost_usd_microcents BIGINT NOT NULL DEFAULT 0;
|
||||||
@@ -127,6 +127,13 @@
|
|||||||
"when": 1786056600000,
|
"when": 1786056600000,
|
||||||
"tag": "0018_cooking_sessions_meal_box_mutations",
|
"tag": "0018_cooking_sessions_meal_box_mutations",
|
||||||
"breakpoints": true
|
"breakpoints": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"idx": 18,
|
||||||
|
"version": "7",
|
||||||
|
"when": 1786141200000,
|
||||||
|
"tag": "0019_ai_usage_cost_usd",
|
||||||
|
"breakpoints": true
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
+2
-1
@@ -28,7 +28,8 @@
|
|||||||
"worker:dev": "pnpm --filter @app/worker dev",
|
"worker:dev": "pnpm --filter @app/worker dev",
|
||||||
"admin:dev": "pnpm --filter @app/admin dev",
|
"admin:dev": "pnpm --filter @app/admin dev",
|
||||||
"mobile:start": "pnpm --filter @app/mobile start",
|
"mobile:start": "pnpm --filter @app/mobile start",
|
||||||
"eval:aamos": "pnpm --filter @app/worker eval"
|
"eval:aamos": "pnpm --filter @app/worker eval",
|
||||||
|
"eval:scan": "pnpm --filter @app/worker eval:scan"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@types/node": "^22.10.0",
|
"@types/node": "^22.10.0",
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import {
|
|||||||
type TaskOutput,
|
type TaskOutput,
|
||||||
} from "./tasks.js";
|
} from "./tasks.js";
|
||||||
import { mockOutputFor } from "./mock.js";
|
import { mockOutputFor } from "./mock.js";
|
||||||
|
import { GeminiAamosClient, type BudgetStore } from "./gemini.js";
|
||||||
|
|
||||||
import type { LocaleContext } from "@app/shared-types";
|
import type { LocaleContext } from "@app/shared-types";
|
||||||
|
|
||||||
@@ -31,6 +32,8 @@ export interface AamosResult<T extends AamosTaskType> {
|
|||||||
promptVersion?: string | undefined;
|
promptVersion?: string | undefined;
|
||||||
latencyMs?: number | undefined;
|
latencyMs?: number | undefined;
|
||||||
costUsd?: number | undefined;
|
costUsd?: number | undefined;
|
||||||
|
inputTokens?: number | undefined;
|
||||||
|
outputTokens?: number | undefined;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -248,12 +251,44 @@ export interface AamosEnv {
|
|||||||
AAMOS_API_URL?: string | undefined;
|
AAMOS_API_URL?: string | undefined;
|
||||||
AAMOS_API_KEY?: string | undefined;
|
AAMOS_API_KEY?: string | undefined;
|
||||||
AAMOS_TIMEOUT_MS?: string | undefined;
|
AAMOS_TIMEOUT_MS?: string | undefined;
|
||||||
|
|
||||||
|
GEMINI_API_KEY?: string | undefined;
|
||||||
|
GEMINI_MODEL?: string | undefined;
|
||||||
|
GEMINI_TIMEOUT_MS?: string | undefined;
|
||||||
|
GEMINI_DAILY_BUDGET_USD?: string | undefined;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Fabrik: http mot riktiga AAMOS (default), mock för dev/test. */
|
export interface CreateAamosClientOptions {
|
||||||
export function createAamosClient(env: AamosEnv = process.env): AamosClient {
|
budgetStore?: BudgetStore;
|
||||||
|
fetchImpl?: typeof fetch;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Fabrik: http mot riktiga AAMOS, gemini-lärar-tier, eller mock för dev/test. */
|
||||||
|
export function createAamosClient(
|
||||||
|
env: AamosEnv = process.env,
|
||||||
|
options: CreateAamosClientOptions = {},
|
||||||
|
): AamosClient {
|
||||||
const mode = env.AAMOS_MODE ?? "http";
|
const mode = env.AAMOS_MODE ?? "http";
|
||||||
if (mode === "mock") return new MockAamosClient();
|
if (mode === "mock") return new MockAamosClient();
|
||||||
|
|
||||||
|
if (mode === "gemini") {
|
||||||
|
const apiKey = env.GEMINI_API_KEY;
|
||||||
|
if (!apiKey) {
|
||||||
|
throw new Error(
|
||||||
|
"AAMOS_MODE=gemini kräver GEMINI_API_KEY. " +
|
||||||
|
"Sätt AAMOS_MODE=mock för lokal utveckling utan Gemini-åtkomst.",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return new GeminiAamosClient({
|
||||||
|
apiKey,
|
||||||
|
model: env.GEMINI_MODEL,
|
||||||
|
timeoutMs: env.GEMINI_TIMEOUT_MS ? Number(env.GEMINI_TIMEOUT_MS) : undefined,
|
||||||
|
dailyBudgetUsd: env.GEMINI_DAILY_BUDGET_USD ? Number(env.GEMINI_DAILY_BUDGET_USD) : undefined,
|
||||||
|
budgetStore: options.budgetStore,
|
||||||
|
fetchImpl: options.fetchImpl,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
const baseUrl = env.AAMOS_API_URL;
|
const baseUrl = env.AAMOS_API_URL;
|
||||||
const apiKey = env.AAMOS_API_KEY;
|
const apiKey = env.AAMOS_API_KEY;
|
||||||
if (!baseUrl || !apiKey) {
|
if (!baseUrl || !apiKey) {
|
||||||
@@ -266,6 +301,7 @@ export function createAamosClient(env: AamosEnv = process.env): AamosClient {
|
|||||||
baseUrl,
|
baseUrl,
|
||||||
apiKey,
|
apiKey,
|
||||||
timeoutMs: env.AAMOS_TIMEOUT_MS ? Number(env.AAMOS_TIMEOUT_MS) : undefined,
|
timeoutMs: env.AAMOS_TIMEOUT_MS ? Number(env.AAMOS_TIMEOUT_MS) : undefined,
|
||||||
|
fetchImpl: options.fetchImpl,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,358 @@
|
|||||||
|
/**
|
||||||
|
* Gemini-lärar-tier för AAMOS-task-typer (SKIVA 1: kylskåpsskanning).
|
||||||
|
*
|
||||||
|
* - Implementerar samma AamosClient-interface som HttpAamosClient/MockAamosClient.
|
||||||
|
* - App + Food API ser ingen skillnad; workern byter bara adapter.
|
||||||
|
* - Nycklar finns serversidan; klienten exponeras aldrig i appen.
|
||||||
|
* - Kostnad/tokens rapporteras tillbaka så workern kan bokföra i ai_usage_counters.
|
||||||
|
* - Global dagsbudget kan sättas via GEMINI_DAILY_BUDGET_USD.
|
||||||
|
*/
|
||||||
|
import { randomUUID } from "node:crypto";
|
||||||
|
import { z } from "zod";
|
||||||
|
import {
|
||||||
|
TASK_CONTRACTS,
|
||||||
|
type AamosTaskType,
|
||||||
|
type TaskInput,
|
||||||
|
type TaskOutput,
|
||||||
|
detectedItemSchema,
|
||||||
|
type DetectedItem,
|
||||||
|
} from "./tasks.js";
|
||||||
|
import type { AamosCallOptions, AamosClient, AamosResult } from "./client.js";
|
||||||
|
import type { LocaleContext } from "@app/shared-types";
|
||||||
|
|
||||||
|
const GEMINI_API_BASE = "https://generativelanguage.googleapis.com/v1beta";
|
||||||
|
|
||||||
|
/** Estimated max cost per image call in USD (pessimistic). */
|
||||||
|
const COST_ESTIMATE_PER_IMAGE_USD = 0.0015;
|
||||||
|
|
||||||
|
export interface BudgetStore {
|
||||||
|
/** Return current daily spend in USD. */
|
||||||
|
getDailySpendUsd(): Promise<number>;
|
||||||
|
/** Increment daily spend by amountUsd; return new total. */
|
||||||
|
incrementDailySpendUsd(amountUsd: number): Promise<number>;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** In-memory budget store for tests / single-process dev. */
|
||||||
|
export class MemoryBudgetStore implements BudgetStore {
|
||||||
|
private spend = 0;
|
||||||
|
async getDailySpendUsd(): Promise<number> {
|
||||||
|
return this.spend;
|
||||||
|
}
|
||||||
|
async incrementDailySpendUsd(amountUsd: number): Promise<number> {
|
||||||
|
this.spend += amountUsd;
|
||||||
|
return this.spend;
|
||||||
|
}
|
||||||
|
reset() {
|
||||||
|
this.spend = 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface GeminiAamosClientConfig {
|
||||||
|
apiKey: string;
|
||||||
|
model?: string;
|
||||||
|
timeoutMs?: number;
|
||||||
|
dailyBudgetUsd?: number;
|
||||||
|
budgetStore?: BudgetStore;
|
||||||
|
fetchImpl?: typeof fetch;
|
||||||
|
promptVersion?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
const geminiFridgeResponseSchema = z.object({
|
||||||
|
items: z.array(
|
||||||
|
z.object({
|
||||||
|
produkt: z.string(),
|
||||||
|
varumarke: z.string().nullable(),
|
||||||
|
kvantitet: z.union([z.number(), z.string()]).nullable(),
|
||||||
|
enhet: z.string().nullable(),
|
||||||
|
kategori: z.string().nullable(),
|
||||||
|
bastaFore: z.string().nullable().optional(),
|
||||||
|
sistaForbruk: z.string().nullable().optional(),
|
||||||
|
beskrivning: z.string().nullable().optional(),
|
||||||
|
konfidens: z.number().min(0).max(1),
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
imageQualityIssues: z
|
||||||
|
.array(z.enum(["dark", "blurry", "occlusion", "too_far"]))
|
||||||
|
.default([]),
|
||||||
|
});
|
||||||
|
|
||||||
|
function parseDate(value: string | null | undefined): string | null {
|
||||||
|
if (!value) return null;
|
||||||
|
const normalized = value.trim();
|
||||||
|
if (!normalized) return null;
|
||||||
|
// Accept ISO dates (YYYY-MM-DD) only.
|
||||||
|
if (/^\d{4}-\d{2}-\d{2}$/.test(normalized)) return normalized;
|
||||||
|
// Try common Swedish/EU formats.
|
||||||
|
const parts = normalized.split(/[-/.]/);
|
||||||
|
if (parts.length === 3) {
|
||||||
|
const [aStr, bStr, cStr] = parts;
|
||||||
|
if (!aStr || !bStr || !cStr) return null;
|
||||||
|
const a = parseInt(aStr, 10);
|
||||||
|
const b = parseInt(bStr, 10);
|
||||||
|
const c = parseInt(cStr, 10);
|
||||||
|
if (Number.isNaN(a) || Number.isNaN(b) || Number.isNaN(c)) return null;
|
||||||
|
// YYYY-MM-DD
|
||||||
|
if (a > 2000 && b <= 12 && c <= 31) return `${a}-${String(b).padStart(2, "0")}-${String(c).padStart(2, "0")}`;
|
||||||
|
// DD-MM-YYYY
|
||||||
|
if (c > 2000 && b <= 12 && a <= 31) return `${c}-${String(b).padStart(2, "0")}-${String(a).padStart(2, "0")}`;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseQuantity(value: string | number | null | undefined): { quantity: number | null; unit: string | null } {
|
||||||
|
if (value == null) return { quantity: null, unit: null };
|
||||||
|
if (typeof value === "number") return { quantity: value, unit: null };
|
||||||
|
const text = String(value).trim().replace(/,/g, ".");
|
||||||
|
if (!text) return { quantity: null, unit: null };
|
||||||
|
const match = text.match(/^(\d+(?:\.\d+)?)\s*(.*)$/);
|
||||||
|
if (match && match[1] != null && match[2] != null) {
|
||||||
|
const qty = parseFloat(match[1]);
|
||||||
|
const unit = match[2].trim();
|
||||||
|
return { quantity: Number.isFinite(qty) ? qty : null, unit: unit || null };
|
||||||
|
}
|
||||||
|
return { quantity: null, unit: text };
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeUnit(unit: string | null | undefined): string | null {
|
||||||
|
if (unit == null) return null;
|
||||||
|
const u = unit.toLowerCase().trim();
|
||||||
|
if (u === "g" || u === "gram") return "GRAM";
|
||||||
|
if (u === "kg" || u === "kilo" || u === "kilogram") return "KILOGRAM";
|
||||||
|
if (u === "ml" || u === "milliliter") return "MILLILITER";
|
||||||
|
if (u === "dl" || u === "deciliter") return "DECILITER";
|
||||||
|
if (u === "l" || u === "liter") return "LITER";
|
||||||
|
if (u === "st" || u === "styck" || u === "piece" || u === "pieces") return "COUNT";
|
||||||
|
if (u === "förpackning" || u === "paket" || u === "package") return "PACKAGE";
|
||||||
|
if (u === "portion" || u === "portioner") return "PORTION";
|
||||||
|
if (u === "klyfta" || u === "klyftor") return "CLOVE";
|
||||||
|
if (u === "skiva" || u === "skivor") return "SLICE";
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function toDetectedItems(parsed: z.infer<typeof geminiFridgeResponseSchema>): DetectedItem[] {
|
||||||
|
return parsed.items.map((item) => {
|
||||||
|
const { quantity, unit } = parseQuantity(item.kvantitet);
|
||||||
|
return detectedItemSchema.parse({
|
||||||
|
detectedName: item.produkt,
|
||||||
|
canonicalIngredientId: null,
|
||||||
|
brand: item.varumarke,
|
||||||
|
estimatedQuantity: quantity,
|
||||||
|
unit: normalizeUnit(unit ?? item.enhet),
|
||||||
|
bestBeforeDate: parseDate(item.bastaFore ?? item.sistaForbruk),
|
||||||
|
confidence: item.konfidens,
|
||||||
|
requiresConfirmation: item.konfidens < 0.92,
|
||||||
|
boundingBox: null,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export class GeminiAamosClient implements AamosClient {
|
||||||
|
private readonly cfg: Required<Omit<GeminiAamosClientConfig, "budgetStore" | "fetchImpl">> & {
|
||||||
|
fetchImpl: typeof fetch;
|
||||||
|
budgetStore?: BudgetStore;
|
||||||
|
};
|
||||||
|
|
||||||
|
constructor(config: GeminiAamosClientConfig) {
|
||||||
|
this.cfg = {
|
||||||
|
apiKey: config.apiKey,
|
||||||
|
model: config.model ?? "gemini-2.5-flash",
|
||||||
|
timeoutMs: config.timeoutMs ?? 60_000,
|
||||||
|
dailyBudgetUsd: config.dailyBudgetUsd ?? 0,
|
||||||
|
promptVersion: config.promptVersion ?? "gemini-fridge-v1",
|
||||||
|
fetchImpl: config.fetchImpl ?? fetch,
|
||||||
|
budgetStore: config.budgetStore,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async healthCheck(): Promise<{ ok: boolean; detail?: string }> {
|
||||||
|
try {
|
||||||
|
const url = `${GEMINI_API_BASE}/models?key=${this.cfg.apiKey}`;
|
||||||
|
const res = await this.cfg.fetchImpl(url, { signal: AbortSignal.timeout(5_000) });
|
||||||
|
return { ok: res.ok, detail: `status ${res.status}` };
|
||||||
|
} catch (err) {
|
||||||
|
return { ok: false, detail: err instanceof Error ? err.message : String(err) };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async runTask<T extends AamosTaskType>(
|
||||||
|
taskType: T,
|
||||||
|
input: TaskInput<T>,
|
||||||
|
options: AamosCallOptions = {},
|
||||||
|
): Promise<AamosResult<T>> {
|
||||||
|
const contract = TASK_CONTRACTS[taskType];
|
||||||
|
const parsedInput = contract.input.safeParse(input);
|
||||||
|
if (!parsedInput.success) {
|
||||||
|
return {
|
||||||
|
status: "failed",
|
||||||
|
output: null,
|
||||||
|
error: `Kontraktsfel (input) för ${taskType}: ${parsedInput.error.message}`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
switch (taskType) {
|
||||||
|
case "ANALYZE_FRIDGE_IMAGE":
|
||||||
|
case "ANALYZE_PANTRY_IMAGE":
|
||||||
|
return this.analyzeStorageImage(taskType, parsedInput.data as TaskInput<"ANALYZE_FRIDGE_IMAGE">, options) as Promise<AamosResult<T>>;
|
||||||
|
default:
|
||||||
|
return {
|
||||||
|
status: "failed",
|
||||||
|
output: null,
|
||||||
|
error: `${taskType} stöds inte av Gemini-lärar-tier i SKIVA 1.`,
|
||||||
|
} as AamosResult<T>;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async analyzeStorageImage(
|
||||||
|
taskType: "ANALYZE_FRIDGE_IMAGE" | "ANALYZE_PANTRY_IMAGE",
|
||||||
|
input: TaskInput<"ANALYZE_FRIDGE_IMAGE">,
|
||||||
|
options: AamosCallOptions,
|
||||||
|
): Promise<AamosResult<"ANALYZE_FRIDGE_IMAGE">> {
|
||||||
|
const locale = options.localeContext ?? this.defaultLocale();
|
||||||
|
const started = Date.now();
|
||||||
|
|
||||||
|
const imageParts = await Promise.all(
|
||||||
|
input.imageUrls.slice(0, 6).map(async (url) => {
|
||||||
|
const res = await this.cfg.fetchImpl(url, { signal: AbortSignal.timeout(this.cfg.timeoutMs) });
|
||||||
|
if (!res.ok) throw new Error(`Kunde inte hämta bild: ${res.status} ${url}`);
|
||||||
|
const buf = Buffer.from(await res.arrayBuffer());
|
||||||
|
return { inline_data: { mime_type: this.mimeType(buf), data: buf.toString("base64") } };
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
const estimatedCostUsd = imageParts.length * COST_ESTIMATE_PER_IMAGE_USD;
|
||||||
|
if (await this.isOverBudget(estimatedCostUsd)) {
|
||||||
|
return {
|
||||||
|
status: "failed",
|
||||||
|
output: null,
|
||||||
|
error: "Global Gemini-dagsbudget är förbrukad.",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const prompt = this.buildFridgePrompt(taskType, locale, input.locationType);
|
||||||
|
const payload = {
|
||||||
|
contents: [{ parts: [{ text: prompt }, ...imageParts] }],
|
||||||
|
generationConfig: {
|
||||||
|
responseMimeType: "application/json",
|
||||||
|
temperature: 0.2,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
const url = `${GEMINI_API_BASE}/models/${this.cfg.model}:generateContent?key=${this.cfg.apiKey}`;
|
||||||
|
const res = await this.cfg.fetchImpl(url, {
|
||||||
|
method: "POST",
|
||||||
|
headers: { "content-type": "application/json" },
|
||||||
|
body: JSON.stringify(payload),
|
||||||
|
signal: AbortSignal.timeout(this.cfg.timeoutMs),
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!res.ok) {
|
||||||
|
const body = await res.text();
|
||||||
|
return {
|
||||||
|
status: "failed",
|
||||||
|
output: null,
|
||||||
|
error: `Gemini ${res.status}: ${body.slice(0, 500)}`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const geminiBody = (await res.json()) as {
|
||||||
|
candidates?: Array<{ content?: { parts?: Array<{ text?: string }> } }>;
|
||||||
|
usageMetadata?: { promptTokenCount?: number; candidatesTokenCount?: number };
|
||||||
|
};
|
||||||
|
const text = geminiBody.candidates?.[0]?.content?.parts?.[0]?.text ?? "";
|
||||||
|
|
||||||
|
let parsed: z.infer<typeof geminiFridgeResponseSchema>;
|
||||||
|
try {
|
||||||
|
const json = JSON.parse(text);
|
||||||
|
const safe = geminiFridgeResponseSchema.safeParse(json);
|
||||||
|
if (!safe.success) {
|
||||||
|
return { status: "failed", output: null, error: `Gemini-svar matchar inte schema: ${safe.error.message}` };
|
||||||
|
}
|
||||||
|
parsed = safe.data;
|
||||||
|
} catch {
|
||||||
|
return { status: "failed", output: null, error: "Gemini-svar var inte giltig JSON." };
|
||||||
|
}
|
||||||
|
|
||||||
|
const inputTokens = geminiBody.usageMetadata?.promptTokenCount ?? 0;
|
||||||
|
const outputTokens = geminiBody.usageMetadata?.candidatesTokenCount ?? 0;
|
||||||
|
const costUsd = this.estimateCostUsd(inputTokens, outputTokens);
|
||||||
|
await this.recordSpend(costUsd);
|
||||||
|
|
||||||
|
const latencyMs = Date.now() - started;
|
||||||
|
const items = toDetectedItems(parsed);
|
||||||
|
|
||||||
|
return {
|
||||||
|
status: items.length > 0 ? "ok" : "uncertain",
|
||||||
|
output: { items, imageQualityIssues: parsed.imageQualityIssues } as TaskOutput<"ANALYZE_FRIDGE_IMAGE">,
|
||||||
|
modelVersion: this.cfg.model,
|
||||||
|
promptVersion: this.cfg.promptVersion,
|
||||||
|
latencyMs,
|
||||||
|
costUsd,
|
||||||
|
inputTokens,
|
||||||
|
outputTokens,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private buildFridgePrompt(
|
||||||
|
taskType: "ANALYZE_FRIDGE_IMAGE" | "ANALYZE_PANTRY_IMAGE",
|
||||||
|
locale: LocaleContext,
|
||||||
|
locationType: string,
|
||||||
|
): string {
|
||||||
|
const place = taskType === "ANALYZE_FRIDGE_IMAGE" ? "kylskåpet" : "skafferiet";
|
||||||
|
const lang = locale.languageTag.startsWith("en") ? "English" : "Swedish";
|
||||||
|
return `Du är en noggrann livsmedelsassistent för svenska hushåll. Användaren har fotat ${place} (${locationType}).
|
||||||
|
|
||||||
|
Identifiera varje livsmedelsprodukt du ser. För varje produkt, svara med:
|
||||||
|
- produkt: produktens namn på svenska (t.ex. "Mellanmjölk", "Smör", "Vispgrädde")
|
||||||
|
- varumarke: varumärke om det syns tydligt, annars null
|
||||||
|
- kvantitet: siffra (t.ex. 1, 500, 1.5)
|
||||||
|
- enhet: förkortning som l, g, ml, st, förpackning
|
||||||
|
- kategori: t.ex. mejeri, frukt, grönt, kött, skafferi
|
||||||
|
- bastaFore: bäst-före-datum om det syns (YYYY-MM-DD), annars null
|
||||||
|
- sistaForbruk: sista förbrukningsdatum om det syns (YYYY-MM-DD), annars null
|
||||||
|
- konfidens: 0.0–1.0
|
||||||
|
|
||||||
|
Svara ENDAST med giltig JSON i exakt detta format:
|
||||||
|
{
|
||||||
|
"items": [...],
|
||||||
|
"imageQualityIssues": []
|
||||||
|
}
|
||||||
|
imageQualityIssues kan innehålla någon av: "dark", "blurry", "occlusion", "too_far".
|
||||||
|
|
||||||
|
Språk: ${lang}.`;
|
||||||
|
}
|
||||||
|
|
||||||
|
private defaultLocale(): LocaleContext {
|
||||||
|
return {
|
||||||
|
languageTag: "sv-SE",
|
||||||
|
regionCode: "SE",
|
||||||
|
timeZone: "Europe/Stockholm",
|
||||||
|
measurementSystem: "METRIC",
|
||||||
|
temperatureUnit: "CELSIUS",
|
||||||
|
currencyCode: "SEK",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private mimeType(buf: Buffer): string {
|
||||||
|
if (buf[0] === 0xff && buf[1] === 0xd8) return "image/jpeg";
|
||||||
|
if (buf.slice(0, 8).toString("hex") === "89504e470d0a1a0a") return "image/png";
|
||||||
|
if (buf.slice(0, 4).toString("ascii") === "RIFF") return "image/webp";
|
||||||
|
return "image/jpeg";
|
||||||
|
}
|
||||||
|
|
||||||
|
private estimateCostUsd(inputTokens: number, outputTokens: number): number {
|
||||||
|
// Gemini 2.5 Flash pricing (Aug 2025): ~$0.075/1M input, $0.30/1M output.
|
||||||
|
return inputTokens * 0.075e-6 + outputTokens * 0.30e-6;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async isOverBudget(estimateUsd: number): Promise<boolean> {
|
||||||
|
if (this.cfg.dailyBudgetUsd <= 0 || !this.cfg.budgetStore) return false;
|
||||||
|
const current = await this.cfg.budgetStore.getDailySpendUsd();
|
||||||
|
return current + estimateUsd > this.cfg.dailyBudgetUsd;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async recordSpend(costUsd: number): Promise<void> {
|
||||||
|
if (this.cfg.budgetStore && costUsd > 0) {
|
||||||
|
await this.cfg.budgetStore.incrementDailySpendUsd(costUsd);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,3 +1,4 @@
|
|||||||
export * from "./tasks.js";
|
export * from "./tasks.js";
|
||||||
export * from "./client.js";
|
export * from "./client.js";
|
||||||
|
export * from "./gemini.js";
|
||||||
export { mockOutputFor } from "./mock.js";
|
export { mockOutputFor } from "./mock.js";
|
||||||
|
|||||||
@@ -0,0 +1,154 @@
|
|||||||
|
import { describe, it, expect } from "vitest";
|
||||||
|
import { GeminiAamosClient, MemoryBudgetStore } from "../src/gemini.js";
|
||||||
|
import type { AamosRequestEnvelope } from "../src/tasks.js";
|
||||||
|
|
||||||
|
const FIXTURE_RESPONSE = {
|
||||||
|
candidates: [
|
||||||
|
{
|
||||||
|
content: {
|
||||||
|
parts: [
|
||||||
|
{
|
||||||
|
text: JSON.stringify({
|
||||||
|
items: [
|
||||||
|
{
|
||||||
|
produkt: "Mellanmjölk 1,5% fett",
|
||||||
|
varumarke: "Arla",
|
||||||
|
kvantitet: 1,
|
||||||
|
enhet: "liter",
|
||||||
|
kategori: "Mejeri",
|
||||||
|
bastaFore: null,
|
||||||
|
sistaForbruk: null,
|
||||||
|
konfidens: 0.98,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
produkt: "Smör",
|
||||||
|
varumarke: "Arla",
|
||||||
|
kvantitet: "500 g",
|
||||||
|
enhet: "g",
|
||||||
|
kategori: "Mejeri",
|
||||||
|
bastaFore: null,
|
||||||
|
konfidens: 0.95,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
imageQualityIssues: [],
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
usageMetadata: { promptTokenCount: 392, candidatesTokenCount: 121 },
|
||||||
|
};
|
||||||
|
|
||||||
|
function makeFetch(imageBytes: Buffer): typeof fetch {
|
||||||
|
return async (url: string | URL | Request, init?: RequestInit) => {
|
||||||
|
const urlStr = url.toString();
|
||||||
|
if (urlStr.includes("/mock-s3/")) {
|
||||||
|
return new Response(imageBytes, { status: 200, headers: { "content-type": "image/jpeg" } });
|
||||||
|
}
|
||||||
|
if (urlStr.includes("/models/") && init?.method === "POST") {
|
||||||
|
return new Response(JSON.stringify(FIXTURE_RESPONSE), { status: 200, headers: { "content-type": "application/json" } });
|
||||||
|
}
|
||||||
|
return new Response("not found", { status: 404 });
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// A tiny 1x1 JPEG.
|
||||||
|
const JPEG_BYTES = Buffer.from(
|
||||||
|
"/9j/4AAQSkZJRgABAQEASABIAAD/2wBDAP//////////////////////////////////////////////////////////////////////////////////////wAALCAABAAEBAREA/8QAFAABAAAAAAAAAAAAAAAAAAAAA//EABQQAQAAAAAAAAAAAAAAAAAAAAD/2gAIAQEAAD8Af//Z",
|
||||||
|
"base64",
|
||||||
|
);
|
||||||
|
|
||||||
|
describe("GeminiAamosClient", () => {
|
||||||
|
it("analyzes fridge image with recorded fixture", async () => {
|
||||||
|
const budget = new MemoryBudgetStore();
|
||||||
|
const client = new GeminiAamosClient({
|
||||||
|
apiKey: "test-key",
|
||||||
|
model: "gemini-2.5-flash",
|
||||||
|
fetchImpl: makeFetch(JPEG_BYTES),
|
||||||
|
budgetStore: budget,
|
||||||
|
dailyBudgetUsd: 10,
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await client.runTask("ANALYZE_FRIDGE_IMAGE", {
|
||||||
|
imageUrls: ["http://localhost/v1/mock-s3/fridge.jpg"],
|
||||||
|
locationType: "fridge",
|
||||||
|
marketLocale: "sv-SE",
|
||||||
|
knownItems: [],
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result.status).toBe("ok");
|
||||||
|
expect(result.output).not.toBeNull();
|
||||||
|
expect(result.modelVersion).toBe("gemini-2.5-flash");
|
||||||
|
expect(result.costUsd).toBeGreaterThan(0);
|
||||||
|
expect(result.inputTokens).toBe(392);
|
||||||
|
expect(result.outputTokens).toBe(121);
|
||||||
|
|
||||||
|
expect(result.output).not.toBeNull();
|
||||||
|
const items = result.output!.items;
|
||||||
|
expect(items).toHaveLength(2);
|
||||||
|
expect(items[0]?.detectedName).toBe("Mellanmjölk 1,5% fett");
|
||||||
|
expect(items[0]?.brand).toBe("Arla");
|
||||||
|
expect(items[0]?.estimatedQuantity).toBe(1);
|
||||||
|
expect(items[0]?.unit).toBe("LITER");
|
||||||
|
expect(items[0]?.confidence).toBe(0.98);
|
||||||
|
expect(items[1]?.detectedName).toBe("Smör");
|
||||||
|
expect(items[1]?.estimatedQuantity).toBe(500);
|
||||||
|
expect(items[1]?.unit).toBe("GRAM");
|
||||||
|
|
||||||
|
const spend = await budget.getDailySpendUsd();
|
||||||
|
expect(spend).toBe(result.costUsd);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("blocks call when daily budget exceeded", async () => {
|
||||||
|
const budget = new MemoryBudgetStore();
|
||||||
|
const client = new GeminiAamosClient({
|
||||||
|
apiKey: "test-key",
|
||||||
|
fetchImpl: makeFetch(JPEG_BYTES),
|
||||||
|
budgetStore: budget,
|
||||||
|
dailyBudgetUsd: 0.00001, // essentially zero
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await client.runTask("ANALYZE_FRIDGE_IMAGE", {
|
||||||
|
imageUrls: ["http://localhost/v1/mock-s3/fridge.jpg"],
|
||||||
|
locationType: "fridge",
|
||||||
|
marketLocale: "sv-SE",
|
||||||
|
knownItems: [],
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result.status).toBe("failed");
|
||||||
|
expect(result.error).toContain("budget");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects unsupported task types", async () => {
|
||||||
|
const client = new GeminiAamosClient({
|
||||||
|
apiKey: "test-key",
|
||||||
|
fetchImpl: makeFetch(JPEG_BYTES),
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await client.runTask(
|
||||||
|
"READ_RECEIPT",
|
||||||
|
{ imageUrls: ["http://localhost/v1/mock-s3/receipt.jpg"], marketLocale: "sv-SE" },
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(result.status).toBe("failed");
|
||||||
|
expect(result.error).toContain("READ_RECEIPT");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("validates input before calling Gemini", async () => {
|
||||||
|
const client = new GeminiAamosClient({
|
||||||
|
apiKey: "test-key",
|
||||||
|
fetchImpl: makeFetch(JPEG_BYTES),
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await client.runTask("ANALYZE_FRIDGE_IMAGE", {
|
||||||
|
imageUrls: [],
|
||||||
|
locationType: "fridge",
|
||||||
|
marketLocale: "sv-SE",
|
||||||
|
knownItems: [],
|
||||||
|
} as unknown as { imageUrls: string[]; locationType: string; marketLocale: string; knownItems: string[] });
|
||||||
|
|
||||||
|
expect(result.status).toBe("failed");
|
||||||
|
expect(result.error).toContain("Kontraktsfel");
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -1,4 +1,5 @@
|
|||||||
import {
|
import {
|
||||||
|
bigint,
|
||||||
boolean,
|
boolean,
|
||||||
index,
|
index,
|
||||||
integer,
|
integer,
|
||||||
@@ -121,6 +122,7 @@ export const aiUsageCounters = pgTable(
|
|||||||
aiScans: integer("ai_scans").notNull().default(0),
|
aiScans: integer("ai_scans").notNull().default(0),
|
||||||
aiTokensIn: integer("ai_tokens_in").notNull().default(0),
|
aiTokensIn: integer("ai_tokens_in").notNull().default(0),
|
||||||
aiTokensOut: integer("ai_tokens_out").notNull().default(0),
|
aiTokensOut: integer("ai_tokens_out").notNull().default(0),
|
||||||
|
aiCostUsdMicrocents: bigint("ai_cost_usd_microcents", { mode: "number" }).notNull().default(0),
|
||||||
updatedAt: updatedAt(),
|
updatedAt: updatedAt(),
|
||||||
},
|
},
|
||||||
(t) => [uniqueIndex("ai_usage_user_month_unique").on(t.userId, t.month)],
|
(t) => [uniqueIndex("ai_usage_user_month_unique").on(t.userId, t.month)],
|
||||||
|
|||||||
Reference in New Issue
Block a user