feat(gemini): Skiva 1 – Gemini 2.5 Flash som lärar-tier för kylskåpsskanning

- Gemini-adapter bakom AamosClient-interface (AAMOS_MODE=gemini)
- Serversida/worker: hämtar bild, anropar Gemini, mappar mot canonical_ingredients
- Kostnad/tokens bokförs i ai_usage_counters; global dagsbudget via BudgetStore
- Redis-backed budget i worker, in-memory i tester
- Migration 0019: ai_cost_usd_microcents
- Hermetiska tester med inspelad fixture; separat pnpm eval:scan
- docs/09 uppdaterad ärligt: AAMOS-status, Gemini-flöde, säkerhet/kostnad
- REQUIRE_REAL=1 stödjer AAMOS_MODE=gemini; deploy-grind uppdaterad
This commit is contained in:
Sven (AAMOS AI)
2026-08-08 03:23:44 +07:00
parent 7894a1ec06
commit 050c958285
18 changed files with 1055 additions and 86 deletions
+38 -2
View File
@@ -8,6 +8,7 @@ import {
type TaskOutput,
} from "./tasks.js";
import { mockOutputFor } from "./mock.js";
import { GeminiAamosClient, type BudgetStore } from "./gemini.js";
import type { LocaleContext } from "@app/shared-types";
@@ -31,6 +32,8 @@ export interface AamosResult<T extends AamosTaskType> {
promptVersion?: string | undefined;
latencyMs?: number | undefined;
costUsd?: number | undefined;
inputTokens?: number | undefined;
outputTokens?: number | undefined;
}
/**
@@ -248,12 +251,44 @@ export interface AamosEnv {
AAMOS_API_URL?: string | undefined;
AAMOS_API_KEY?: string | undefined;
AAMOS_TIMEOUT_MS?: string | undefined;
GEMINI_API_KEY?: string | undefined;
GEMINI_MODEL?: string | undefined;
GEMINI_TIMEOUT_MS?: string | undefined;
GEMINI_DAILY_BUDGET_USD?: string | undefined;
}
/** Fabrik: http mot riktiga AAMOS (default), mock för dev/test. */
export function createAamosClient(env: AamosEnv = process.env): AamosClient {
export interface CreateAamosClientOptions {
budgetStore?: BudgetStore;
fetchImpl?: typeof fetch;
}
/** Fabrik: http mot riktiga AAMOS, gemini-lärar-tier, eller mock för dev/test. */
export function createAamosClient(
env: AamosEnv = process.env,
options: CreateAamosClientOptions = {},
): AamosClient {
const mode = env.AAMOS_MODE ?? "http";
if (mode === "mock") return new MockAamosClient();
if (mode === "gemini") {
const apiKey = env.GEMINI_API_KEY;
if (!apiKey) {
throw new Error(
"AAMOS_MODE=gemini kräver GEMINI_API_KEY. " +
"Sätt AAMOS_MODE=mock för lokal utveckling utan Gemini-åtkomst.",
);
}
return new GeminiAamosClient({
apiKey,
model: env.GEMINI_MODEL,
timeoutMs: env.GEMINI_TIMEOUT_MS ? Number(env.GEMINI_TIMEOUT_MS) : undefined,
dailyBudgetUsd: env.GEMINI_DAILY_BUDGET_USD ? Number(env.GEMINI_DAILY_BUDGET_USD) : undefined,
budgetStore: options.budgetStore,
fetchImpl: options.fetchImpl,
});
}
const baseUrl = env.AAMOS_API_URL;
const apiKey = env.AAMOS_API_KEY;
if (!baseUrl || !apiKey) {
@@ -266,6 +301,7 @@ export function createAamosClient(env: AamosEnv = process.env): AamosClient {
baseUrl,
apiKey,
timeoutMs: env.AAMOS_TIMEOUT_MS ? Number(env.AAMOS_TIMEOUT_MS) : undefined,
fetchImpl: options.fetchImpl,
});
}
+358
View File
@@ -0,0 +1,358 @@
/**
* Gemini-lärar-tier för AAMOS-task-typer (SKIVA 1: kylskåpsskanning).
*
* - Implementerar samma AamosClient-interface som HttpAamosClient/MockAamosClient.
* - App + Food API ser ingen skillnad; workern byter bara adapter.
* - Nycklar finns serversidan; klienten exponeras aldrig i appen.
* - Kostnad/tokens rapporteras tillbaka så workern kan bokföra i ai_usage_counters.
* - Global dagsbudget kan sättas via GEMINI_DAILY_BUDGET_USD.
*/
import { randomUUID } from "node:crypto";
import { z } from "zod";
import {
TASK_CONTRACTS,
type AamosTaskType,
type TaskInput,
type TaskOutput,
detectedItemSchema,
type DetectedItem,
} from "./tasks.js";
import type { AamosCallOptions, AamosClient, AamosResult } from "./client.js";
import type { LocaleContext } from "@app/shared-types";
const GEMINI_API_BASE = "https://generativelanguage.googleapis.com/v1beta";
/** Estimated max cost per image call in USD (pessimistic). */
const COST_ESTIMATE_PER_IMAGE_USD = 0.0015;
export interface BudgetStore {
/** Return current daily spend in USD. */
getDailySpendUsd(): Promise<number>;
/** Increment daily spend by amountUsd; return new total. */
incrementDailySpendUsd(amountUsd: number): Promise<number>;
}
/** In-memory budget store for tests / single-process dev. */
export class MemoryBudgetStore implements BudgetStore {
private spend = 0;
async getDailySpendUsd(): Promise<number> {
return this.spend;
}
async incrementDailySpendUsd(amountUsd: number): Promise<number> {
this.spend += amountUsd;
return this.spend;
}
reset() {
this.spend = 0;
}
}
export interface GeminiAamosClientConfig {
apiKey: string;
model?: string;
timeoutMs?: number;
dailyBudgetUsd?: number;
budgetStore?: BudgetStore;
fetchImpl?: typeof fetch;
promptVersion?: string;
}
const geminiFridgeResponseSchema = z.object({
items: z.array(
z.object({
produkt: z.string(),
varumarke: z.string().nullable(),
kvantitet: z.union([z.number(), z.string()]).nullable(),
enhet: z.string().nullable(),
kategori: z.string().nullable(),
bastaFore: z.string().nullable().optional(),
sistaForbruk: z.string().nullable().optional(),
beskrivning: z.string().nullable().optional(),
konfidens: z.number().min(0).max(1),
}),
),
imageQualityIssues: z
.array(z.enum(["dark", "blurry", "occlusion", "too_far"]))
.default([]),
});
function parseDate(value: string | null | undefined): string | null {
if (!value) return null;
const normalized = value.trim();
if (!normalized) return null;
// Accept ISO dates (YYYY-MM-DD) only.
if (/^\d{4}-\d{2}-\d{2}$/.test(normalized)) return normalized;
// Try common Swedish/EU formats.
const parts = normalized.split(/[-/.]/);
if (parts.length === 3) {
const [aStr, bStr, cStr] = parts;
if (!aStr || !bStr || !cStr) return null;
const a = parseInt(aStr, 10);
const b = parseInt(bStr, 10);
const c = parseInt(cStr, 10);
if (Number.isNaN(a) || Number.isNaN(b) || Number.isNaN(c)) return null;
// YYYY-MM-DD
if (a > 2000 && b <= 12 && c <= 31) return `${a}-${String(b).padStart(2, "0")}-${String(c).padStart(2, "0")}`;
// DD-MM-YYYY
if (c > 2000 && b <= 12 && a <= 31) return `${c}-${String(b).padStart(2, "0")}-${String(a).padStart(2, "0")}`;
}
return null;
}
function parseQuantity(value: string | number | null | undefined): { quantity: number | null; unit: string | null } {
if (value == null) return { quantity: null, unit: null };
if (typeof value === "number") return { quantity: value, unit: null };
const text = String(value).trim().replace(/,/g, ".");
if (!text) return { quantity: null, unit: null };
const match = text.match(/^(\d+(?:\.\d+)?)\s*(.*)$/);
if (match && match[1] != null && match[2] != null) {
const qty = parseFloat(match[1]);
const unit = match[2].trim();
return { quantity: Number.isFinite(qty) ? qty : null, unit: unit || null };
}
return { quantity: null, unit: text };
}
function normalizeUnit(unit: string | null | undefined): string | null {
if (unit == null) return null;
const u = unit.toLowerCase().trim();
if (u === "g" || u === "gram") return "GRAM";
if (u === "kg" || u === "kilo" || u === "kilogram") return "KILOGRAM";
if (u === "ml" || u === "milliliter") return "MILLILITER";
if (u === "dl" || u === "deciliter") return "DECILITER";
if (u === "l" || u === "liter") return "LITER";
if (u === "st" || u === "styck" || u === "piece" || u === "pieces") return "COUNT";
if (u === "förpackning" || u === "paket" || u === "package") return "PACKAGE";
if (u === "portion" || u === "portioner") return "PORTION";
if (u === "klyfta" || u === "klyftor") return "CLOVE";
if (u === "skiva" || u === "skivor") return "SLICE";
return null;
}
function toDetectedItems(parsed: z.infer<typeof geminiFridgeResponseSchema>): DetectedItem[] {
return parsed.items.map((item) => {
const { quantity, unit } = parseQuantity(item.kvantitet);
return detectedItemSchema.parse({
detectedName: item.produkt,
canonicalIngredientId: null,
brand: item.varumarke,
estimatedQuantity: quantity,
unit: normalizeUnit(unit ?? item.enhet),
bestBeforeDate: parseDate(item.bastaFore ?? item.sistaForbruk),
confidence: item.konfidens,
requiresConfirmation: item.konfidens < 0.92,
boundingBox: null,
});
});
}
export class GeminiAamosClient implements AamosClient {
private readonly cfg: Required<Omit<GeminiAamosClientConfig, "budgetStore" | "fetchImpl">> & {
fetchImpl: typeof fetch;
budgetStore?: BudgetStore;
};
constructor(config: GeminiAamosClientConfig) {
this.cfg = {
apiKey: config.apiKey,
model: config.model ?? "gemini-2.5-flash",
timeoutMs: config.timeoutMs ?? 60_000,
dailyBudgetUsd: config.dailyBudgetUsd ?? 0,
promptVersion: config.promptVersion ?? "gemini-fridge-v1",
fetchImpl: config.fetchImpl ?? fetch,
budgetStore: config.budgetStore,
};
}
async healthCheck(): Promise<{ ok: boolean; detail?: string }> {
try {
const url = `${GEMINI_API_BASE}/models?key=${this.cfg.apiKey}`;
const res = await this.cfg.fetchImpl(url, { signal: AbortSignal.timeout(5_000) });
return { ok: res.ok, detail: `status ${res.status}` };
} catch (err) {
return { ok: false, detail: err instanceof Error ? err.message : String(err) };
}
}
async runTask<T extends AamosTaskType>(
taskType: T,
input: TaskInput<T>,
options: AamosCallOptions = {},
): Promise<AamosResult<T>> {
const contract = TASK_CONTRACTS[taskType];
const parsedInput = contract.input.safeParse(input);
if (!parsedInput.success) {
return {
status: "failed",
output: null,
error: `Kontraktsfel (input) för ${taskType}: ${parsedInput.error.message}`,
};
}
switch (taskType) {
case "ANALYZE_FRIDGE_IMAGE":
case "ANALYZE_PANTRY_IMAGE":
return this.analyzeStorageImage(taskType, parsedInput.data as TaskInput<"ANALYZE_FRIDGE_IMAGE">, options) as Promise<AamosResult<T>>;
default:
return {
status: "failed",
output: null,
error: `${taskType} stöds inte av Gemini-lärar-tier i SKIVA 1.`,
} as AamosResult<T>;
}
}
private async analyzeStorageImage(
taskType: "ANALYZE_FRIDGE_IMAGE" | "ANALYZE_PANTRY_IMAGE",
input: TaskInput<"ANALYZE_FRIDGE_IMAGE">,
options: AamosCallOptions,
): Promise<AamosResult<"ANALYZE_FRIDGE_IMAGE">> {
const locale = options.localeContext ?? this.defaultLocale();
const started = Date.now();
const imageParts = await Promise.all(
input.imageUrls.slice(0, 6).map(async (url) => {
const res = await this.cfg.fetchImpl(url, { signal: AbortSignal.timeout(this.cfg.timeoutMs) });
if (!res.ok) throw new Error(`Kunde inte hämta bild: ${res.status} ${url}`);
const buf = Buffer.from(await res.arrayBuffer());
return { inline_data: { mime_type: this.mimeType(buf), data: buf.toString("base64") } };
}),
);
const estimatedCostUsd = imageParts.length * COST_ESTIMATE_PER_IMAGE_USD;
if (await this.isOverBudget(estimatedCostUsd)) {
return {
status: "failed",
output: null,
error: "Global Gemini-dagsbudget är förbrukad.",
};
}
const prompt = this.buildFridgePrompt(taskType, locale, input.locationType);
const payload = {
contents: [{ parts: [{ text: prompt }, ...imageParts] }],
generationConfig: {
responseMimeType: "application/json",
temperature: 0.2,
},
};
const url = `${GEMINI_API_BASE}/models/${this.cfg.model}:generateContent?key=${this.cfg.apiKey}`;
const res = await this.cfg.fetchImpl(url, {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify(payload),
signal: AbortSignal.timeout(this.cfg.timeoutMs),
});
if (!res.ok) {
const body = await res.text();
return {
status: "failed",
output: null,
error: `Gemini ${res.status}: ${body.slice(0, 500)}`,
};
}
const geminiBody = (await res.json()) as {
candidates?: Array<{ content?: { parts?: Array<{ text?: string }> } }>;
usageMetadata?: { promptTokenCount?: number; candidatesTokenCount?: number };
};
const text = geminiBody.candidates?.[0]?.content?.parts?.[0]?.text ?? "";
let parsed: z.infer<typeof geminiFridgeResponseSchema>;
try {
const json = JSON.parse(text);
const safe = geminiFridgeResponseSchema.safeParse(json);
if (!safe.success) {
return { status: "failed", output: null, error: `Gemini-svar matchar inte schema: ${safe.error.message}` };
}
parsed = safe.data;
} catch {
return { status: "failed", output: null, error: "Gemini-svar var inte giltig JSON." };
}
const inputTokens = geminiBody.usageMetadata?.promptTokenCount ?? 0;
const outputTokens = geminiBody.usageMetadata?.candidatesTokenCount ?? 0;
const costUsd = this.estimateCostUsd(inputTokens, outputTokens);
await this.recordSpend(costUsd);
const latencyMs = Date.now() - started;
const items = toDetectedItems(parsed);
return {
status: items.length > 0 ? "ok" : "uncertain",
output: { items, imageQualityIssues: parsed.imageQualityIssues } as TaskOutput<"ANALYZE_FRIDGE_IMAGE">,
modelVersion: this.cfg.model,
promptVersion: this.cfg.promptVersion,
latencyMs,
costUsd,
inputTokens,
outputTokens,
};
}
private buildFridgePrompt(
taskType: "ANALYZE_FRIDGE_IMAGE" | "ANALYZE_PANTRY_IMAGE",
locale: LocaleContext,
locationType: string,
): string {
const place = taskType === "ANALYZE_FRIDGE_IMAGE" ? "kylskåpet" : "skafferiet";
const lang = locale.languageTag.startsWith("en") ? "English" : "Swedish";
return `Du är en noggrann livsmedelsassistent för svenska hushåll. Användaren har fotat ${place} (${locationType}).
Identifiera varje livsmedelsprodukt du ser. För varje produkt, svara med:
- produkt: produktens namn på svenska (t.ex. "Mellanmjölk", "Smör", "Vispgrädde")
- varumarke: varumärke om det syns tydligt, annars null
- kvantitet: siffra (t.ex. 1, 500, 1.5)
- enhet: förkortning som l, g, ml, st, förpackning
- kategori: t.ex. mejeri, frukt, grönt, kött, skafferi
- bastaFore: bäst-före-datum om det syns (YYYY-MM-DD), annars null
- sistaForbruk: sista förbrukningsdatum om det syns (YYYY-MM-DD), annars null
- konfidens: 0.01.0
Svara ENDAST med giltig JSON i exakt detta format:
{
"items": [...],
"imageQualityIssues": []
}
imageQualityIssues kan innehålla någon av: "dark", "blurry", "occlusion", "too_far".
Språk: ${lang}.`;
}
private defaultLocale(): LocaleContext {
return {
languageTag: "sv-SE",
regionCode: "SE",
timeZone: "Europe/Stockholm",
measurementSystem: "METRIC",
temperatureUnit: "CELSIUS",
currencyCode: "SEK",
};
}
private mimeType(buf: Buffer): string {
if (buf[0] === 0xff && buf[1] === 0xd8) return "image/jpeg";
if (buf.slice(0, 8).toString("hex") === "89504e470d0a1a0a") return "image/png";
if (buf.slice(0, 4).toString("ascii") === "RIFF") return "image/webp";
return "image/jpeg";
}
private estimateCostUsd(inputTokens: number, outputTokens: number): number {
// Gemini 2.5 Flash pricing (Aug 2025): ~$0.075/1M input, $0.30/1M output.
return inputTokens * 0.075e-6 + outputTokens * 0.30e-6;
}
private async isOverBudget(estimateUsd: number): Promise<boolean> {
if (this.cfg.dailyBudgetUsd <= 0 || !this.cfg.budgetStore) return false;
const current = await this.cfg.budgetStore.getDailySpendUsd();
return current + estimateUsd > this.cfg.dailyBudgetUsd;
}
private async recordSpend(costUsd: number): Promise<void> {
if (this.cfg.budgetStore && costUsd > 0) {
await this.cfg.budgetStore.incrementDailySpendUsd(costUsd);
}
}
}
+1
View File
@@ -1,3 +1,4 @@
export * from "./tasks.js";
export * from "./client.js";
export * from "./gemini.js";
export { mockOutputFor } from "./mock.js";
+154
View File
@@ -0,0 +1,154 @@
import { describe, it, expect } from "vitest";
import { GeminiAamosClient, MemoryBudgetStore } from "../src/gemini.js";
import type { AamosRequestEnvelope } from "../src/tasks.js";
const FIXTURE_RESPONSE = {
candidates: [
{
content: {
parts: [
{
text: JSON.stringify({
items: [
{
produkt: "Mellanmjölk 1,5% fett",
varumarke: "Arla",
kvantitet: 1,
enhet: "liter",
kategori: "Mejeri",
bastaFore: null,
sistaForbruk: null,
konfidens: 0.98,
},
{
produkt: "Smör",
varumarke: "Arla",
kvantitet: "500 g",
enhet: "g",
kategori: "Mejeri",
bastaFore: null,
konfidens: 0.95,
},
],
imageQualityIssues: [],
}),
},
],
},
},
],
usageMetadata: { promptTokenCount: 392, candidatesTokenCount: 121 },
};
function makeFetch(imageBytes: Buffer): typeof fetch {
return async (url: string | URL | Request, init?: RequestInit) => {
const urlStr = url.toString();
if (urlStr.includes("/mock-s3/")) {
return new Response(imageBytes, { status: 200, headers: { "content-type": "image/jpeg" } });
}
if (urlStr.includes("/models/") && init?.method === "POST") {
return new Response(JSON.stringify(FIXTURE_RESPONSE), { status: 200, headers: { "content-type": "application/json" } });
}
return new Response("not found", { status: 404 });
};
}
// A tiny 1x1 JPEG.
const JPEG_BYTES = Buffer.from(
"/9j/4AAQSkZJRgABAQEASABIAAD/2wBDAP//////////////////////////////////////////////////////////////////////////////////////wAALCAABAAEBAREA/8QAFAABAAAAAAAAAAAAAAAAAAAAA//EABQQAQAAAAAAAAAAAAAAAAAAAAD/2gAIAQEAAD8Af//Z",
"base64",
);
describe("GeminiAamosClient", () => {
it("analyzes fridge image with recorded fixture", async () => {
const budget = new MemoryBudgetStore();
const client = new GeminiAamosClient({
apiKey: "test-key",
model: "gemini-2.5-flash",
fetchImpl: makeFetch(JPEG_BYTES),
budgetStore: budget,
dailyBudgetUsd: 10,
});
const result = await client.runTask("ANALYZE_FRIDGE_IMAGE", {
imageUrls: ["http://localhost/v1/mock-s3/fridge.jpg"],
locationType: "fridge",
marketLocale: "sv-SE",
knownItems: [],
});
expect(result.status).toBe("ok");
expect(result.output).not.toBeNull();
expect(result.modelVersion).toBe("gemini-2.5-flash");
expect(result.costUsd).toBeGreaterThan(0);
expect(result.inputTokens).toBe(392);
expect(result.outputTokens).toBe(121);
expect(result.output).not.toBeNull();
const items = result.output!.items;
expect(items).toHaveLength(2);
expect(items[0]?.detectedName).toBe("Mellanmjölk 1,5% fett");
expect(items[0]?.brand).toBe("Arla");
expect(items[0]?.estimatedQuantity).toBe(1);
expect(items[0]?.unit).toBe("LITER");
expect(items[0]?.confidence).toBe(0.98);
expect(items[1]?.detectedName).toBe("Smör");
expect(items[1]?.estimatedQuantity).toBe(500);
expect(items[1]?.unit).toBe("GRAM");
const spend = await budget.getDailySpendUsd();
expect(spend).toBe(result.costUsd);
});
it("blocks call when daily budget exceeded", async () => {
const budget = new MemoryBudgetStore();
const client = new GeminiAamosClient({
apiKey: "test-key",
fetchImpl: makeFetch(JPEG_BYTES),
budgetStore: budget,
dailyBudgetUsd: 0.00001, // essentially zero
});
const result = await client.runTask("ANALYZE_FRIDGE_IMAGE", {
imageUrls: ["http://localhost/v1/mock-s3/fridge.jpg"],
locationType: "fridge",
marketLocale: "sv-SE",
knownItems: [],
});
expect(result.status).toBe("failed");
expect(result.error).toContain("budget");
});
it("rejects unsupported task types", async () => {
const client = new GeminiAamosClient({
apiKey: "test-key",
fetchImpl: makeFetch(JPEG_BYTES),
});
const result = await client.runTask(
"READ_RECEIPT",
{ imageUrls: ["http://localhost/v1/mock-s3/receipt.jpg"], marketLocale: "sv-SE" },
);
expect(result.status).toBe("failed");
expect(result.error).toContain("READ_RECEIPT");
});
it("validates input before calling Gemini", async () => {
const client = new GeminiAamosClient({
apiKey: "test-key",
fetchImpl: makeFetch(JPEG_BYTES),
});
const result = await client.runTask("ANALYZE_FRIDGE_IMAGE", {
imageUrls: [],
locationType: "fridge",
marketLocale: "sv-SE",
knownItems: [],
} as unknown as { imageUrls: string[]; locationType: string; marketLocale: string; knownItems: string[] });
expect(result.status).toBe("failed");
expect(result.error).toContain("Kontraktsfel");
});
});
@@ -1,4 +1,5 @@
import {
bigint,
boolean,
index,
integer,
@@ -121,6 +122,7 @@ export const aiUsageCounters = pgTable(
aiScans: integer("ai_scans").notNull().default(0),
aiTokensIn: integer("ai_tokens_in").notNull().default(0),
aiTokensOut: integer("ai_tokens_out").notNull().default(0),
aiCostUsdMicrocents: bigint("ai_cost_usd_microcents", { mode: "number" }).notNull().default(0),
updatedAt: updatedAt(),
},
(t) => [uniqueIndex("ai_usage_user_month_unique").on(t.userId, t.month)],