Sven (AAMOS AI)
1036 lines
36 KiB
TypeScript
1036 lines
36 KiB
TypeScript
/**
|
||
* Gemini-lärar-tier för AAMOS-task-typer (SKIVA 1).
|
||
*
|
||
* - Implementerar samma AamosClient-interface som HttpAamosClient/MockAamosClient.
|
||
* - App + Food API ser ingen skillnad; workern byter bara adapter.
|
||
* - Nycklar finns serversidan; klienten exponeras aldrig i appen.
|
||
* - Kostnad/tokens rapporteras tillbaka så workern kan bokföra i ai_usage_counters.
|
||
* - Global dagsbudget kan sättas via GEMINI_DAILY_BUDGET_USD.
|
||
*/
|
||
import { z } from "zod";
|
||
import {
|
||
TASK_CONTRACTS,
|
||
type AamosTaskType,
|
||
type TaskInput,
|
||
type TaskOutput,
|
||
detectedItemSchema,
|
||
type DetectedItem,
|
||
generateRecipeCandidatesOutput,
|
||
analyzeMealImageOutput,
|
||
readNutritionLabelOutput,
|
||
readExpiryDateOutput,
|
||
readReceiptOutput,
|
||
} from "./tasks.js";
|
||
import type { AamosCallOptions, AamosClient, AamosResult } from "./client.js";
|
||
import type { LocaleContext } from "@app/shared-types";
|
||
|
||
const GEMINI_API_BASE = "https://generativelanguage.googleapis.com/v1beta";
|
||
|
||
/** Custom fetch using Node https module — works around IPv6 hangs on WSL. Follows redirects. */
|
||
function customNodeFetch(input: string | URL | Request, init?: RequestInit): Promise<Response> {
|
||
return requestOnce(input, init, 5);
|
||
}
|
||
|
||
function requestOnce(
|
||
input: string | URL | Request,
|
||
init: RequestInit | undefined,
|
||
redirectsLeft: number,
|
||
): Promise<Response> {
|
||
return new Promise((resolve, reject) => {
|
||
const url =
|
||
typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url;
|
||
const u = new URL(url);
|
||
const isHttps = u.protocol === "https:";
|
||
const mod = isHttps ? import("node:https") : import("node:http");
|
||
mod
|
||
.then((httpMod) => {
|
||
const postData = init?.body ? String(init.body) : undefined;
|
||
const headers: Record<string, string> = {};
|
||
if (init?.headers) {
|
||
if (init.headers instanceof Headers) {
|
||
init.headers.forEach((v, k) => {
|
||
headers[k] = v;
|
||
});
|
||
} else if (Array.isArray(init.headers)) {
|
||
(init.headers as [string, string][]).forEach(([k, v]) => {
|
||
if (k != null) headers[k] = v;
|
||
});
|
||
} else {
|
||
Object.assign(headers, init.headers as Record<string, string>);
|
||
}
|
||
}
|
||
if (postData && !headers["content-length"]) {
|
||
headers["content-length"] = String(Buffer.byteLength(postData));
|
||
}
|
||
if (!headers["user-agent"]) {
|
||
headers["user-agent"] = "Mozilla/5.0 (compatible; AAMOS-Gateway/1.0)";
|
||
}
|
||
|
||
const req = httpMod.request(
|
||
{
|
||
hostname: u.hostname,
|
||
port: u.port || (isHttps ? 443 : 80),
|
||
path: u.pathname + u.search,
|
||
method: init?.method || "GET",
|
||
headers,
|
||
timeout: 120_000,
|
||
},
|
||
(res) => {
|
||
const status = res.statusCode ?? 200;
|
||
const location = res.headers.location;
|
||
if (status >= 300 && status < 400 && location && redirectsLeft > 0) {
|
||
const nextUrl = new URL(location, url).toString();
|
||
requestOnce(nextUrl, { ...init, method: "GET", body: undefined }, redirectsLeft - 1)
|
||
.then(resolve)
|
||
.catch(reject);
|
||
return;
|
||
}
|
||
|
||
const chunks: Buffer[] = [];
|
||
res.on("data", (chunk: Buffer) => {
|
||
chunks.push(chunk);
|
||
});
|
||
res.on("end", () => {
|
||
const body = Buffer.concat(chunks);
|
||
resolve(
|
||
new Response(body, {
|
||
status,
|
||
statusText: res.statusMessage ?? "OK",
|
||
headers: new Headers(Object.entries(res.headers).map(([k, v]) => [k, String(v)])),
|
||
}),
|
||
);
|
||
});
|
||
},
|
||
);
|
||
req.on("error", (err) => reject(err));
|
||
req.on("timeout", () => {
|
||
req.destroy();
|
||
reject(new Error("Request timeout"));
|
||
});
|
||
if (postData) req.write(postData);
|
||
req.end();
|
||
})
|
||
.catch(reject);
|
||
});
|
||
}
|
||
|
||
/** Estimated max cost per image call in USD (pessimistic). */
|
||
const COST_ESTIMATE_PER_IMAGE_USD = 0.0015;
|
||
|
||
/** Estimated max cost per simple OCR image call in USD. */
|
||
const COST_ESTIMATE_PER_OCR_IMAGE_USD = 0.001;
|
||
|
||
export interface BudgetStore {
|
||
/** Return current daily spend in USD. */
|
||
getDailySpendUsd(): Promise<number>;
|
||
/** Increment daily spend by amountUsd; return new total. */
|
||
incrementDailySpendUsd(amountUsd: number): Promise<number>;
|
||
/**
|
||
* Optional: called when the daily budget is exceeded. Implementations can
|
||
* raise an alert (e.g. write to Redis / DB) so ops-summary/EOC can surface it.
|
||
*/
|
||
raiseBudgetAlarm?(): Promise<void>;
|
||
}
|
||
|
||
/** In-memory budget store for tests / single-process dev. */
|
||
export class MemoryBudgetStore implements BudgetStore {
|
||
private spend = 0;
|
||
private alarmRaised = false;
|
||
async getDailySpendUsd(): Promise<number> {
|
||
return this.spend;
|
||
}
|
||
async incrementDailySpendUsd(amountUsd: number): Promise<number> {
|
||
this.spend += amountUsd;
|
||
return this.spend;
|
||
}
|
||
async raiseBudgetAlarm(): Promise<void> {
|
||
this.alarmRaised = true;
|
||
}
|
||
alarmWasRaised(): boolean {
|
||
return this.alarmRaised;
|
||
}
|
||
reset() {
|
||
this.spend = 0;
|
||
this.alarmRaised = false;
|
||
}
|
||
}
|
||
|
||
export interface GeminiAamosClientConfig {
|
||
apiKey: string;
|
||
model?: string;
|
||
timeoutMs?: number;
|
||
dailyBudgetUsd?: number;
|
||
budgetStore?: BudgetStore;
|
||
fetchImpl?: typeof fetch;
|
||
promptVersion?: string;
|
||
}
|
||
|
||
const geminiFridgeResponseSchema = z.object({
|
||
items: z.array(
|
||
z.object({
|
||
produkt: z.string(),
|
||
varumarke: z.string().nullable(),
|
||
kvantitet: z.union([z.number(), z.string()]).nullable(),
|
||
enhet: z.string().nullable(),
|
||
kategori: z.string().nullable(),
|
||
bastaFore: z.string().nullable().optional(),
|
||
sistaForbruk: z.string().nullable().optional(),
|
||
beskrivning: z.string().nullable().optional(),
|
||
konfidens: z.number().min(0).max(1),
|
||
}),
|
||
),
|
||
imageQualityIssues: z.array(z.enum(["dark", "blurry", "occlusion", "too_far"])).default([]),
|
||
});
|
||
|
||
function parseDate(value: string | null | undefined): string | null {
|
||
if (!value) return null;
|
||
const normalized = value.trim();
|
||
if (!normalized) return null;
|
||
// Accept ISO dates (YYYY-MM-DD) only.
|
||
if (/^\d{4}-\d{2}-\d{2}$/.test(normalized)) return normalized;
|
||
// Try common Swedish/EU formats.
|
||
const parts = normalized.split(/[-/.]/);
|
||
if (parts.length === 3) {
|
||
const [aStr, bStr, cStr] = parts;
|
||
if (!aStr || !bStr || !cStr) return null;
|
||
const a = parseInt(aStr, 10);
|
||
const b = parseInt(bStr, 10);
|
||
const c = parseInt(cStr, 10);
|
||
if (Number.isNaN(a) || Number.isNaN(b) || Number.isNaN(c)) return null;
|
||
// YYYY-MM-DD
|
||
if (a > 2000 && b <= 12 && c <= 31)
|
||
return `${a}-${String(b).padStart(2, "0")}-${String(c).padStart(2, "0")}`;
|
||
// DD-MM-YYYY
|
||
if (c > 2000 && b <= 12 && a <= 31)
|
||
return `${c}-${String(b).padStart(2, "0")}-${String(a).padStart(2, "0")}`;
|
||
}
|
||
return null;
|
||
}
|
||
|
||
function parseQuantity(value: string | number | null | undefined): {
|
||
quantity: number | null;
|
||
unit: string | null;
|
||
} {
|
||
if (value == null) return { quantity: null, unit: null };
|
||
if (typeof value === "number") return { quantity: value, unit: null };
|
||
const text = String(value).trim().replace(/,/g, ".");
|
||
if (!text) return { quantity: null, unit: null };
|
||
const match = text.match(/^(\d+(?:\.\d+)?)\s*(.*)$/);
|
||
if (match && match[1] != null && match[2] != null) {
|
||
const qty = parseFloat(match[1]);
|
||
const unit = match[2].trim();
|
||
return { quantity: Number.isFinite(qty) ? qty : null, unit: unit || null };
|
||
}
|
||
return { quantity: null, unit: text };
|
||
}
|
||
|
||
function normalizeUnit(unit: string | null | undefined): string | null {
|
||
if (unit == null) return null;
|
||
const u = unit.toLowerCase().trim();
|
||
if (u === "g" || u === "gram") return "GRAM";
|
||
if (u === "kg" || u === "kilo" || u === "kilogram") return "KILOGRAM";
|
||
if (u === "ml" || u === "milliliter") return "MILLILITER";
|
||
if (u === "dl" || u === "deciliter") return "DECILITER";
|
||
if (u === "l" || u === "liter") return "LITER";
|
||
if (u === "st" || u === "styck" || u === "piece" || u === "pieces") return "COUNT";
|
||
if (u === "förpackning" || u === "paket" || u === "package") return "PACKAGE";
|
||
if (u === "portion" || u === "portioner") return "PORTION";
|
||
if (u === "klyfta" || u === "klyftor") return "CLOVE";
|
||
if (u === "skiva" || u === "skivor") return "SLICE";
|
||
return null;
|
||
}
|
||
|
||
function toDetectedItems(parsed: z.infer<typeof geminiFridgeResponseSchema>): DetectedItem[] {
|
||
return parsed.items.map((item) => {
|
||
const { quantity, unit } = parseQuantity(item.kvantitet);
|
||
return detectedItemSchema.parse({
|
||
detectedName: item.produkt,
|
||
canonicalIngredientId: null,
|
||
brand: item.varumarke,
|
||
estimatedQuantity: quantity,
|
||
unit: normalizeUnit(unit ?? item.enhet),
|
||
bestBeforeDate: parseDate(item.bastaFore ?? item.sistaForbruk),
|
||
confidence: item.konfidens,
|
||
requiresConfirmation: item.konfidens < 0.92,
|
||
boundingBox: null,
|
||
});
|
||
});
|
||
}
|
||
|
||
export class GeminiAamosClient implements AamosClient {
|
||
private readonly cfg: Required<Omit<GeminiAamosClientConfig, "budgetStore" | "fetchImpl">> & {
|
||
fetchImpl: typeof fetch;
|
||
budgetStore?: BudgetStore;
|
||
};
|
||
|
||
constructor(config: GeminiAamosClientConfig) {
|
||
this.cfg = {
|
||
apiKey: config.apiKey,
|
||
model: config.model ?? "gemini-3.5-flash-lite",
|
||
timeoutMs: config.timeoutMs ?? 60_000,
|
||
dailyBudgetUsd: config.dailyBudgetUsd ?? 0,
|
||
promptVersion: config.promptVersion ?? "gemini-fridge-v1",
|
||
fetchImpl: config.fetchImpl ?? customNodeFetch,
|
||
budgetStore: config.budgetStore,
|
||
};
|
||
}
|
||
|
||
async healthCheck(): Promise<{ ok: boolean; detail?: string }> {
|
||
try {
|
||
const url = `${GEMINI_API_BASE}/models?key=${encodeURIComponent(this.cfg.apiKey)}`;
|
||
const res = await this.cfg.fetchImpl(url, { signal: AbortSignal.timeout(5_000) });
|
||
return { ok: res.ok, detail: `status ${res.status}` };
|
||
} catch (err) {
|
||
return { ok: false, detail: err instanceof Error ? err.message : String(err) };
|
||
}
|
||
}
|
||
|
||
async runTask<T extends AamosTaskType>(
|
||
taskType: T,
|
||
input: TaskInput<T>,
|
||
options: AamosCallOptions = {},
|
||
): Promise<AamosResult<T>> {
|
||
const contract = TASK_CONTRACTS[taskType];
|
||
const parsedInput = contract.input.safeParse(input);
|
||
if (!parsedInput.success) {
|
||
return {
|
||
status: "failed",
|
||
output: null,
|
||
error: `Kontraktsfel (input) för ${taskType}: ${parsedInput.error.message}`,
|
||
};
|
||
}
|
||
|
||
switch (taskType) {
|
||
case "ANALYZE_FRIDGE_IMAGE":
|
||
case "ANALYZE_PANTRY_IMAGE":
|
||
return this.analyzeStorageImage(
|
||
taskType,
|
||
parsedInput.data as TaskInput<"ANALYZE_FRIDGE_IMAGE">,
|
||
options,
|
||
) as Promise<AamosResult<T>>;
|
||
case "GENERATE_RECIPE_CANDIDATES":
|
||
return this.generateRecipeCandidates(
|
||
parsedInput.data as TaskInput<"GENERATE_RECIPE_CANDIDATES">,
|
||
options,
|
||
) as Promise<AamosResult<T>>;
|
||
case "ANALYZE_MEAL_IMAGE":
|
||
return this.analyzeMealImage(
|
||
parsedInput.data as TaskInput<"ANALYZE_MEAL_IMAGE">,
|
||
options,
|
||
) as Promise<AamosResult<T>>;
|
||
case "READ_NUTRITION_LABEL":
|
||
return this.readNutritionLabel(
|
||
parsedInput.data as TaskInput<"READ_NUTRITION_LABEL">,
|
||
options,
|
||
) as Promise<AamosResult<T>>;
|
||
case "READ_EXPIRY_DATE":
|
||
return this.readExpiryDate(
|
||
parsedInput.data as TaskInput<"READ_EXPIRY_DATE">,
|
||
options,
|
||
) as Promise<AamosResult<T>>;
|
||
case "READ_RECEIPT":
|
||
return this.readReceipt(parsedInput.data as TaskInput<"READ_RECEIPT">, options) as Promise<
|
||
AamosResult<T>
|
||
>;
|
||
default:
|
||
return {
|
||
status: "failed",
|
||
output: null,
|
||
error: `${taskType} stöds inte av Gemini-lärar-tier i SKIVA 1.`,
|
||
} as AamosResult<T>;
|
||
}
|
||
}
|
||
|
||
private async fetchOneImage(
|
||
url: string,
|
||
timeoutMs: number,
|
||
): Promise<{ inline_data: { mime_type: string; data: string } }> {
|
||
if (url.startsWith("data:")) {
|
||
const match = url.match(/^data:([^;]+);base64,(.+)$/);
|
||
if (match && match[1] && match[2]) {
|
||
return { inline_data: { mime_type: match[1], data: match[2] } };
|
||
}
|
||
throw new Error(`Ogiltig data-URL: ${url.slice(0, 50)}`);
|
||
}
|
||
const res = await this.cfg.fetchImpl(url, { signal: AbortSignal.timeout(timeoutMs) });
|
||
if (!res.ok) throw new Error(`Kunde inte hämta bild: ${res.status} ${url}`);
|
||
const buf = Buffer.from(await res.arrayBuffer());
|
||
return { inline_data: { mime_type: this.mimeType(buf), data: buf.toString("base64") } };
|
||
}
|
||
|
||
private async analyzeStorageImage(
|
||
taskType: "ANALYZE_FRIDGE_IMAGE" | "ANALYZE_PANTRY_IMAGE",
|
||
input: TaskInput<"ANALYZE_FRIDGE_IMAGE">,
|
||
options: AamosCallOptions,
|
||
): Promise<AamosResult<"ANALYZE_FRIDGE_IMAGE">> {
|
||
const locale = options.localeContext ?? this.defaultLocale();
|
||
const started = Date.now();
|
||
|
||
const imageCount = Math.min(input.imageUrls.length, 6);
|
||
const estimatedCostUsd = imageCount * COST_ESTIMATE_PER_IMAGE_USD;
|
||
if (await this.isOverBudget(estimatedCostUsd)) {
|
||
console.warn(
|
||
"[gemini] Global dagsbudget överskriden — LARM, blockerar INTE, skanning fortsätter.",
|
||
);
|
||
}
|
||
|
||
const imageParts = await Promise.all(
|
||
input.imageUrls.slice(0, 6).map((url) => this.fetchOneImage(url, this.cfg.timeoutMs)),
|
||
);
|
||
|
||
if (imageParts.length === 0) {
|
||
return {
|
||
status: "failed",
|
||
output: null,
|
||
error: "Global Gemini-dagsbudget är förbrukad.",
|
||
};
|
||
}
|
||
|
||
const prompt = this.buildFridgePrompt(taskType, locale, input.locationType);
|
||
const payload = {
|
||
contents: [{ parts: [{ text: prompt }, ...imageParts] }],
|
||
generationConfig: {
|
||
responseMimeType: "application/json",
|
||
temperature: 0.2,
|
||
},
|
||
};
|
||
|
||
const url = `${GEMINI_API_BASE}/models/${this.cfg.model}:generateContent?key=${encodeURIComponent(this.cfg.apiKey)}`;
|
||
const res = await this.cfg.fetchImpl(url, {
|
||
method: "POST",
|
||
headers: { "content-type": "application/json" },
|
||
body: JSON.stringify(payload),
|
||
signal: AbortSignal.timeout(this.cfg.timeoutMs),
|
||
});
|
||
|
||
if (!res.ok) {
|
||
const body = await res.text();
|
||
return {
|
||
status: "failed",
|
||
output: null,
|
||
error: `Gemini ${res.status}: ${body.slice(0, 500)}`,
|
||
};
|
||
}
|
||
|
||
const geminiBody = (await res.json()) as {
|
||
candidates?: Array<{ content?: { parts?: Array<{ text?: string }> } }>;
|
||
usageMetadata?: {
|
||
promptTokenCount?: number;
|
||
candidatesTokenCount?: number;
|
||
totalTokenCount?: number;
|
||
};
|
||
};
|
||
const text = geminiBody.candidates?.[0]?.content?.parts?.[0]?.text ?? "";
|
||
|
||
let parsed: z.infer<typeof geminiFridgeResponseSchema>;
|
||
try {
|
||
const json = JSON.parse(text);
|
||
const safe = geminiFridgeResponseSchema.safeParse(json);
|
||
if (!safe.success) {
|
||
return {
|
||
status: "failed",
|
||
output: null,
|
||
error: `Gemini-svar matchar inte schema: ${safe.error.message}`,
|
||
};
|
||
}
|
||
parsed = safe.data;
|
||
} catch {
|
||
return { status: "failed", output: null, error: "Gemini-svar var inte giltig JSON." };
|
||
}
|
||
|
||
const inputTokens = geminiBody.usageMetadata?.promptTokenCount ?? 0;
|
||
const outputTokens = geminiBody.usageMetadata?.candidatesTokenCount ?? 0;
|
||
const totalTokens = geminiBody.usageMetadata?.totalTokenCount ?? 0;
|
||
const costUsd = this.estimateCostUsd(inputTokens, outputTokens);
|
||
await this.recordSpend(costUsd);
|
||
|
||
const latencyMs = Date.now() - started;
|
||
const items = toDetectedItems(parsed);
|
||
|
||
return {
|
||
status: items.length > 0 ? "ok" : "uncertain",
|
||
output: {
|
||
items,
|
||
imageQualityIssues: parsed.imageQualityIssues,
|
||
} as TaskOutput<"ANALYZE_FRIDGE_IMAGE">,
|
||
modelVersion: this.cfg.model,
|
||
promptVersion: this.cfg.promptVersion,
|
||
latencyMs,
|
||
costUsd,
|
||
inputTokens,
|
||
outputTokens,
|
||
totalTokens,
|
||
};
|
||
}
|
||
|
||
private async runVisionTask<T extends AamosTaskType>(
|
||
taskType: T,
|
||
imageUrls: string[],
|
||
prompt: string,
|
||
outputSchema: z.ZodSchema<unknown>,
|
||
mapOutput: (parsed: unknown) => TaskOutput<T>,
|
||
options: AamosCallOptions,
|
||
promptVersion: string,
|
||
costEstimatePerImage: number,
|
||
): Promise<AamosResult<T>> {
|
||
const locale = options.localeContext ?? this.defaultLocale();
|
||
const started = Date.now();
|
||
|
||
const imageCount = Math.min(imageUrls.length, 6);
|
||
const estimatedCostUsd = imageCount * costEstimatePerImage;
|
||
if (await this.isOverBudget(estimatedCostUsd)) {
|
||
console.warn(
|
||
"[gemini] Global dagsbudget överskriden — LARM, blockerar INTE, skanning fortsätter.",
|
||
);
|
||
}
|
||
|
||
const imageParts = await this.fetchImageParts(imageUrls, this.cfg.timeoutMs);
|
||
if (imageParts.length === 0) {
|
||
return {
|
||
status: "failed",
|
||
output: null,
|
||
error: "Global Gemini-dagsbudget är förbrukad.",
|
||
} as AamosResult<T>;
|
||
}
|
||
|
||
const payload = {
|
||
contents: [{ parts: [{ text: prompt }, ...imageParts] }],
|
||
generationConfig: {
|
||
responseMimeType: "application/json",
|
||
temperature: 0.2,
|
||
},
|
||
};
|
||
|
||
const url = `${GEMINI_API_BASE}/models/${this.cfg.model}:generateContent?key=${encodeURIComponent(this.cfg.apiKey)}`;
|
||
const res = await this.cfg.fetchImpl(url, {
|
||
method: "POST",
|
||
headers: { "content-type": "application/json" },
|
||
body: JSON.stringify(payload),
|
||
signal: AbortSignal.timeout(this.cfg.timeoutMs),
|
||
});
|
||
|
||
if (!res.ok) {
|
||
const body = await res.text();
|
||
return {
|
||
status: "failed",
|
||
output: null,
|
||
error: `Gemini ${res.status}: ${body.slice(0, 500)}`,
|
||
} as AamosResult<T>;
|
||
}
|
||
|
||
const geminiBody = (await res.json()) as {
|
||
candidates?: Array<{ content?: { parts?: Array<{ text?: string }> } }>;
|
||
usageMetadata?: {
|
||
promptTokenCount?: number;
|
||
candidatesTokenCount?: number;
|
||
totalTokenCount?: number;
|
||
};
|
||
};
|
||
const text = geminiBody.candidates?.[0]?.content?.parts?.[0]?.text ?? "";
|
||
|
||
let parsed: unknown;
|
||
try {
|
||
const json = JSON.parse(text);
|
||
const safe = outputSchema.safeParse(json);
|
||
if (!safe.success) {
|
||
return {
|
||
status: "failed",
|
||
output: null,
|
||
error: `Gemini-svar matchar inte schema: ${safe.error.message}`,
|
||
} as AamosResult<T>;
|
||
}
|
||
parsed = safe.data;
|
||
} catch {
|
||
return {
|
||
status: "failed",
|
||
output: null,
|
||
error: "Gemini-svar var inte giltig JSON.",
|
||
} as AamosResult<T>;
|
||
}
|
||
|
||
const inputTokens = geminiBody.usageMetadata?.promptTokenCount ?? 0;
|
||
const outputTokens = geminiBody.usageMetadata?.candidatesTokenCount ?? 0;
|
||
const totalTokens = geminiBody.usageMetadata?.totalTokenCount ?? 0;
|
||
const costUsd = this.estimateCostUsd(inputTokens, outputTokens);
|
||
await this.recordSpend(costUsd);
|
||
|
||
const latencyMs = Date.now() - started;
|
||
const output = mapOutput(parsed) as TaskOutput<T>;
|
||
|
||
return {
|
||
status: "ok",
|
||
output,
|
||
modelVersion: this.cfg.model,
|
||
promptVersion,
|
||
latencyMs,
|
||
costUsd,
|
||
inputTokens,
|
||
outputTokens,
|
||
totalTokens,
|
||
};
|
||
}
|
||
|
||
private async analyzeMealImage(
|
||
input: TaskInput<"ANALYZE_MEAL_IMAGE">,
|
||
options: AamosCallOptions,
|
||
): Promise<AamosResult<"ANALYZE_MEAL_IMAGE">> {
|
||
const locale = options.localeContext ?? this.defaultLocale();
|
||
const prompt = this.buildMealPrompt(locale, input.recipeContext);
|
||
return this.runVisionTask(
|
||
"ANALYZE_MEAL_IMAGE",
|
||
input.imageUrls,
|
||
prompt,
|
||
analyzeMealImageOutput,
|
||
(parsed) => parsed as TaskOutput<"ANALYZE_MEAL_IMAGE">,
|
||
options,
|
||
"gemini-meal-v1",
|
||
COST_ESTIMATE_PER_IMAGE_USD,
|
||
);
|
||
}
|
||
|
||
private async readNutritionLabel(
|
||
input: TaskInput<"READ_NUTRITION_LABEL">,
|
||
options: AamosCallOptions,
|
||
): Promise<AamosResult<"READ_NUTRITION_LABEL">> {
|
||
const locale = options.localeContext ?? this.defaultLocale();
|
||
const prompt = this.buildNutritionPrompt(locale);
|
||
return this.runVisionTask(
|
||
"READ_NUTRITION_LABEL",
|
||
input.imageUrls,
|
||
prompt,
|
||
readNutritionLabelOutput,
|
||
(parsed) => parsed as TaskOutput<"READ_NUTRITION_LABEL">,
|
||
options,
|
||
"gemini-nutrition-v1",
|
||
COST_ESTIMATE_PER_OCR_IMAGE_USD,
|
||
);
|
||
}
|
||
|
||
private async readExpiryDate(
|
||
input: TaskInput<"READ_EXPIRY_DATE">,
|
||
options: AamosCallOptions,
|
||
): Promise<AamosResult<"READ_EXPIRY_DATE">> {
|
||
const locale = options.localeContext ?? this.defaultLocale();
|
||
const prompt = this.buildExpiryPrompt(locale);
|
||
return this.runVisionTask(
|
||
"READ_EXPIRY_DATE",
|
||
input.imageUrls,
|
||
prompt,
|
||
readExpiryDateOutput,
|
||
(parsed) => parsed as TaskOutput<"READ_EXPIRY_DATE">,
|
||
options,
|
||
"gemini-expiry-v1",
|
||
COST_ESTIMATE_PER_OCR_IMAGE_USD,
|
||
);
|
||
}
|
||
|
||
private async readReceipt(
|
||
input: TaskInput<"READ_RECEIPT">,
|
||
options: AamosCallOptions,
|
||
): Promise<AamosResult<"READ_RECEIPT">> {
|
||
const locale = options.localeContext ?? this.defaultLocale();
|
||
const prompt = this.buildReceiptPrompt(locale);
|
||
return this.runVisionTask(
|
||
"READ_RECEIPT",
|
||
input.imageUrls,
|
||
prompt,
|
||
readReceiptOutput,
|
||
(parsed) => parsed as TaskOutput<"READ_RECEIPT">,
|
||
options,
|
||
"gemini-receipt-v1",
|
||
COST_ESTIMATE_PER_IMAGE_USD,
|
||
);
|
||
}
|
||
|
||
private async fetchImageParts(
|
||
imageUrls: string[],
|
||
timeoutMs: number,
|
||
): Promise<Array<{ inline_data: { mime_type: string; data: string } }>> {
|
||
return Promise.all(imageUrls.slice(0, 6).map((url) => this.fetchOneImage(url, timeoutMs)));
|
||
}
|
||
|
||
private async generateRecipeCandidates(
|
||
input: TaskInput<"GENERATE_RECIPE_CANDIDATES">,
|
||
options: AamosCallOptions,
|
||
): Promise<AamosResult<"GENERATE_RECIPE_CANDIDATES">> {
|
||
const locale = options.localeContext ?? this.defaultLocale();
|
||
const started = Date.now();
|
||
|
||
// Estimate cost: ~0.003 USD per candidate (text-only, pessimistic)
|
||
const totalCandidates = input.targetMatrix.reduce((sum, t) => sum + t.count, 0);
|
||
const estimatedCostUsd = totalCandidates * 0.003;
|
||
if (await this.isOverBudget(estimatedCostUsd)) {
|
||
console.warn(
|
||
"[gemini] Global dagsbudget överskriden — LARM, blockerar INTE, skanning fortsätter.",
|
||
);
|
||
}
|
||
|
||
const prompt = this.buildRecipeGenerationPrompt(input, locale);
|
||
const payload = {
|
||
contents: [{ parts: [{ text: prompt }] }],
|
||
generationConfig: {
|
||
responseMimeType: "application/json",
|
||
temperature: 0.3,
|
||
},
|
||
};
|
||
|
||
const url = `${GEMINI_API_BASE}/models/${this.cfg.model}:generateContent?key=${encodeURIComponent(this.cfg.apiKey)}`;
|
||
const res = await this.cfg.fetchImpl(url, {
|
||
method: "POST",
|
||
headers: { "content-type": "application/json" },
|
||
body: JSON.stringify(payload),
|
||
signal: AbortSignal.timeout(this.cfg.timeoutMs),
|
||
});
|
||
|
||
if (!res.ok) {
|
||
const body = await res.text();
|
||
return {
|
||
status: "failed",
|
||
output: null,
|
||
error: `Gemini ${res.status}: ${body.slice(0, 500)}`,
|
||
};
|
||
}
|
||
|
||
const geminiBody = (await res.json()) as {
|
||
candidates?: Array<{ content?: { parts?: Array<{ text?: string }> } }>;
|
||
usageMetadata?: {
|
||
promptTokenCount?: number;
|
||
candidatesTokenCount?: number;
|
||
totalTokenCount?: number;
|
||
};
|
||
};
|
||
const text = geminiBody.candidates?.[0]?.content?.parts?.[0]?.text ?? "";
|
||
|
||
let parsed: z.infer<typeof generateRecipeCandidatesOutput>;
|
||
try {
|
||
const json = JSON.parse(text);
|
||
// Defensiv normalisering: Gemini kan returnera rejectedPrompts som objekt istället för strängar.
|
||
if (Array.isArray(json.rejectedPrompts)) {
|
||
json.rejectedPrompts = json.rejectedPrompts.map((p: unknown) =>
|
||
typeof p === "string" ? p : JSON.stringify(p),
|
||
);
|
||
}
|
||
const safe = generateRecipeCandidatesOutput.safeParse(json);
|
||
if (!safe.success) {
|
||
return {
|
||
status: "failed",
|
||
output: null,
|
||
error: `Gemini-svar matchar inte schema: ${safe.error.message}`,
|
||
};
|
||
}
|
||
parsed = safe.data;
|
||
} catch {
|
||
return { status: "failed", output: null, error: "Gemini-svar var inte giltig JSON." };
|
||
}
|
||
|
||
const inputTokens = geminiBody.usageMetadata?.promptTokenCount ?? 0;
|
||
const outputTokens = geminiBody.usageMetadata?.candidatesTokenCount ?? 0;
|
||
const totalTokens = geminiBody.usageMetadata?.totalTokenCount ?? 0;
|
||
const costUsd = this.estimateCostUsd(inputTokens, outputTokens);
|
||
await this.recordSpend(costUsd);
|
||
|
||
// Normalisera enheter defensivt: Gemini kan ibland returnera "dl"/"msk"/"st".
|
||
for (const c of parsed.candidates) {
|
||
for (const ing of c.ingredients) {
|
||
const normalized = normalizeUnit(String(ing.unit));
|
||
if (normalized) ing.unit = normalized as typeof ing.unit;
|
||
}
|
||
}
|
||
|
||
const latencyMs = Date.now() - started;
|
||
|
||
return {
|
||
status: parsed.candidates.length > 0 ? "ok" : "uncertain",
|
||
output: parsed as TaskOutput<"GENERATE_RECIPE_CANDIDATES">,
|
||
modelVersion: this.cfg.model,
|
||
promptVersion: "gemini-recipe-gen-v1",
|
||
latencyMs,
|
||
costUsd,
|
||
inputTokens,
|
||
outputTokens,
|
||
totalTokens,
|
||
};
|
||
}
|
||
|
||
private buildRecipeGenerationPrompt(
|
||
input: TaskInput<"GENERATE_RECIPE_CANDIDATES">,
|
||
locale: LocaleContext,
|
||
): string {
|
||
const lang = locale.languageTag.startsWith("en") ? "English" : "Swedish";
|
||
const catalog = input.canonicalIngredientsCatalog
|
||
.map(
|
||
(i) =>
|
||
`- ${i.id} (${i.nameSv}, ${i.category}, enhet: ${i.defaultUnit}, vegan: ${i.isVegan}, veg: ${i.isVegetarian}, gluten: ${i.containsGluten}, laktos: ${i.containsLactose})`,
|
||
)
|
||
.join("\n");
|
||
|
||
const targets = input.targetMatrix
|
||
.map((t) => `- ${t.mealType} × ${t.mainIngredientId} × ${t.dietVariant}: ${t.count} st`)
|
||
.join("\n");
|
||
|
||
const constraints = input.constraints;
|
||
|
||
return `Du är en erfaren svensk matskribent som skriver vardagsrecept för svenska hushåll.
|
||
|
||
VIKTIGAST: Använd ENDAST ingredienser från katalogen nedan. Hitta ALDRIG på ingredienser som inte finns i listan. Om ett recept behöver något som saknas, hoppa över det receptet och lägg till en notering i rejectedPrompts.
|
||
|
||
Tillåtna ingredienser (canonical_ingredients):
|
||
${catalog}
|
||
|
||
Önskade recept:
|
||
${targets}
|
||
|
||
Begränsningar:
|
||
- max förberedelsetid: ${constraints.maxPrepTimeMinutes ?? 60} min
|
||
- max koktid: ${constraints.maxCookTimeMinutes ?? 45} min
|
||
- portioner: ${constraints.portions ?? 4}
|
||
- max kryddnivå: ${constraints.spiceLevelMax ?? 3}
|
||
- undvik: ${constraints.avoidIngredients.join(", ") || "(ingen)"}
|
||
|
||
Regler:
|
||
- Recepten ska vara realistiska vardagsrätter för svenska hushåll.
|
||
- Stegen ska vara tydliga, med rimliga tider och temperaturer.
|
||
- MATSÄKERHET – recept som innehåller rå fågel, rått fläsk/köttfärs, rå fisk/skaldjur eller råa ägg MÅSTE innehålla ett explicit steg som beskriver att proteinet blir genomstekt/genomkokt. Använd formuleringar som: "stek tills kycklingen är genomstekt och inte längre rosa i mitten", "koka tills fisken flagnar", "stek tills köttfärsen är genomstekt", eller ange kärn-/innertemperatur i °C. Det räcker INTE med enbart utseende (t.ex. "gyllenbrun") eller enbart tid.
|
||
- Använd ENDAST dessa exakta enhetskoder: GRAM, KILOGRAM, MILLILITER, DECILITER, LITER, TEASPOON, TABLESPOON, CUP_US, COUNT, PORTION, PINCH, SLICE, CLOVE, CAN, PACKAGE. Aldrig "g", "dl", "msk", "tsk", "st".
|
||
- Formuleringen ska vara mjölkprincips-vänlig: ingen svinnskam, inget "släng".
|
||
- Varje ingrediens MÅSTE finnas i katalogen ovan.
|
||
- Svara ENDAST med giltig JSON i exakt detta format:
|
||
|
||
{
|
||
"candidates": [
|
||
{
|
||
"titleSv": "...",
|
||
"descriptionSv": "...",
|
||
"cuisine": "swedish",
|
||
"mealTypes": ["dinner"],
|
||
"prepTimeMinutes": 15,
|
||
"cookTimeMinutes": 30,
|
||
"portions": 4,
|
||
"spiceLevel": 1,
|
||
"ingredients": [
|
||
{
|
||
"canonicalIngredientId": "kycklingfile",
|
||
"displayNameSv": "kycklingfilé",
|
||
"quantity": 500,
|
||
"unit": "GRAM",
|
||
"optional": false,
|
||
"note": null
|
||
}
|
||
],
|
||
"steps": [
|
||
{
|
||
"instructionSv": "...",
|
||
"timerSeconds": null,
|
||
"temperatureC": null,
|
||
"tip": null
|
||
}
|
||
],
|
||
"storageGuidanceSv": "...",
|
||
"mealPrepFriendly": false,
|
||
"freezerFriendly": false,
|
||
"confidence": 0.95
|
||
}
|
||
],
|
||
"rejectedPrompts": []
|
||
}
|
||
|
||
Språk: ${lang}.`;
|
||
}
|
||
|
||
private buildFridgePrompt(
|
||
taskType: "ANALYZE_FRIDGE_IMAGE" | "ANALYZE_PANTRY_IMAGE",
|
||
locale: LocaleContext,
|
||
locationType: string,
|
||
): string {
|
||
const place = taskType === "ANALYZE_FRIDGE_IMAGE" ? "kylskåpet" : "skafferiet";
|
||
const lang = locale.languageTag.startsWith("en") ? "English" : "Swedish";
|
||
return `Du är en noggrann livsmedelsassistent för svenska hushåll. Användaren har fotat ${place} (${locationType}).
|
||
|
||
Identifiera varje livsmedelsprodukt du ser. För varje produkt, svara med:
|
||
- produkt: produktens namn på svenska (t.ex. "Mellanmjölk", "Smör", "Vispgrädde")
|
||
- varumarke: varumärke om det syns tydligt, annars null
|
||
- kvantitet: siffra (t.ex. 1, 500, 1.5)
|
||
- enhet: förkortning som l, g, ml, st, förpackning
|
||
- kategori: t.ex. mejeri, frukt, grönt, kött, skafferi
|
||
- bastaFore: bäst-före-datum om det syns (YYYY-MM-DD), annars null
|
||
- sistaForbruk: sista förbrukningsdatum om det syns (YYYY-MM-DD), annars null
|
||
- konfidens: 0.0–1.0
|
||
|
||
Viktigt:
|
||
- Kvantitet: uppskatta den FAKTISKA synliga/kvarvarande mängden, inte tryckta förpackningsantal. Ett tomt eller öppnat paket ska ha låg kvantitet (nära 0), även om lådan säger t.ex. "24-pack". Är du osäker, gissa lågt och sänk konfidensen.
|
||
- Svenska mjölk- och mejerifärger: röd förpackning = standardmjölk 3%, grön = mellanmjölk 1,5%, blå = lättmjölk 0,5%. Använd färgen för att avgöra fetthalt.
|
||
- Hellre lägre konfidens än en exakt gissning när du är osäker.
|
||
|
||
Svara ENDAST med giltig JSON i exakt detta format:
|
||
{
|
||
"items": [...],
|
||
"imageQualityIssues": []
|
||
}
|
||
imageQualityIssues kan innehålla någon av: "dark", "blurry", "occlusion", "too_far".
|
||
|
||
Språk: ${lang}.`;
|
||
}
|
||
|
||
private buildMealPrompt(locale: LocaleContext, recipeContext: unknown): string {
|
||
const lang = locale.languageTag.startsWith("en") ? "English" : "Swedish";
|
||
const ctx = recipeContext as {
|
||
recipeId?: string;
|
||
titleSv?: string;
|
||
nutritionPerPortion?: Record<string, number>;
|
||
portions?: number;
|
||
} | null;
|
||
|
||
let contextLine = "";
|
||
if (ctx?.titleSv) {
|
||
contextLine = `Användaren säger att måltiden ska matcha receptet "${ctx.titleSv}" (${ctx.portions ?? "?"} portioner). `;
|
||
}
|
||
|
||
return `Du är en svensk näringsassistent. ${contextLine}Bilden visar en serverad måltid.
|
||
|
||
Uppskatta:
|
||
- matchesRecipeContext: true/false/null – matchar bilden receptet användaren nämnde?
|
||
- portionFractionEstimate: hur stor del av en normalportion ser det ut att vara? 0.0–5.0, eller null.
|
||
- kcalRange: { min, max, mostLikely } för hela måltiden, eller null.
|
||
- components: lista med uppskattade komponenter (namn, ev. canonicalIngredientId som null, estimatedGrams, confidence).
|
||
- confidence: 0.0–1.0 för hela tolkningen.
|
||
|
||
Svara ENDAST med giltig JSON som matchar detta schema:
|
||
{
|
||
"matchesRecipeContext": boolean | null,
|
||
"portionFractionEstimate": number | null,
|
||
"kcalRange": { "min": number, "max": number, "mostLikely": number } | null,
|
||
"components": [
|
||
{ "name": string, "canonicalIngredientId": string | null, "estimatedGrams": number | null, "confidence": number }
|
||
],
|
||
"confidence": number
|
||
}
|
||
|
||
Språk: ${lang}.`;
|
||
}
|
||
|
||
private buildNutritionPrompt(locale: LocaleContext): string {
|
||
const lang = locale.languageTag.startsWith("en") ? "English" : "Swedish";
|
||
return `Du är en OCR-assistent för svenska livsmedelsförpackningar. Bilden visar en näringsdeklaration/ingredienslista.
|
||
|
||
Läs av exakt det som står på etiketten. Gissa aldrig värden som inte syns. Returnera:
|
||
- basis: "per_100_g", "per_100_ml" eller "per_portion" – vad gäller tabellen?
|
||
- values: { kcal, proteinG, carbsG, fatG, saturatedFatG, fiberG, sugarG, saltG } – null om ej synligt.
|
||
- ingredientsText: rå text med ingredienser, eller null.
|
||
- allergensDeclared: lista med allergener som explicit nämns (t.ex. ["gluten", "mjölk", "nötter"]).
|
||
- gtin: streckkod/EAN om synlig, annars null.
|
||
- productName: produktnamn om synligt, annars null.
|
||
- brand: varumärke om synligt, annars null.
|
||
- confidence: 0.0–1.0.
|
||
|
||
Svara ENDAST med giltig JSON som matchar detta schema:
|
||
{
|
||
"basis": "per_100_g" | "per_100_ml" | "per_portion" | null,
|
||
"values": { "kcal": number|null, "proteinG": number|null, "carbsG": number|null, "fatG": number|null, "saturatedFatG": number|null, "fiberG": number|null, "sugarG": number|null, "saltG": number|null } | null,
|
||
"ingredientsText": string | null,
|
||
"allergensDeclared": string[],
|
||
"gtin": string | null,
|
||
"productName": string | null,
|
||
"brand": string | null,
|
||
"confidence": number
|
||
}
|
||
|
||
Språk: ${lang}.`;
|
||
}
|
||
|
||
private buildExpiryPrompt(locale: LocaleContext): string {
|
||
const lang = locale.languageTag.startsWith("en") ? "English" : "Swedish";
|
||
return `Du är en OCR-assistent för svenska livsmedelsförpackningar. Bilden visar ett bäst-före-datum eller sista förbrukningsdatum.
|
||
|
||
Returnera:
|
||
- date: datum i format YYYY-MM-DD, eller null om ej läsbart.
|
||
- dateKind: "best_before" för "bäst före", "use_by" för "sista förbrukningsdag", annars null.
|
||
- confidence: 0.0–1.0.
|
||
|
||
Svara ENDAST med giltig JSON:
|
||
{
|
||
"date": "YYYY-MM-DD" | null,
|
||
"dateKind": "best_before" | "use_by" | null,
|
||
"confidence": number
|
||
}
|
||
|
||
Språk: ${lang}.`;
|
||
}
|
||
|
||
private buildReceiptPrompt(locale: LocaleContext): string {
|
||
const lang = locale.languageTag.startsWith("en") ? "English" : "Swedish";
|
||
return `Du är en OCR-assistent för svenska kvitton. Bilden visar ett butikskvitto.
|
||
|
||
Läs av:
|
||
- storeName: butikens namn, eller null.
|
||
- purchaseDate: datum i format YYYY-MM-DD, eller null.
|
||
- lines: varje rad med { rawText (exakt text), normalizedName (förenklat livsmedelsnamn eller null), canonicalIngredientId (alltid null), quantity (antal/enhet om går att utläsa, annars null), unit (enhetskod eller null), unitPriceMinor (ören per styck om synligt, annars null), totalPriceMinor (radens totalpris i ören om synligt, annars null), isDiscount (true om det är en rabatt/avdrag-rad), confidence (0.0–1.0) }.
|
||
- totalMinor: kvittots totalsumma i ören, eller null.
|
||
- discountTotalMinor: total rabatt i ören, eller null.
|
||
- confidence: 0.0–1.0 för hela tolkningen.
|
||
|
||
Svara ENDAST med giltig JSON som matchar detta schema:
|
||
{
|
||
"storeName": string | null,
|
||
"purchaseDate": "YYYY-MM-DD" | null,
|
||
"lines": [
|
||
{
|
||
"rawText": string,
|
||
"normalizedName": string | null,
|
||
"canonicalIngredientId": null,
|
||
"quantity": number | null,
|
||
"unit": string | null,
|
||
"unitPriceMinor": number | null,
|
||
"totalPriceMinor": number | null,
|
||
"isDiscount": boolean,
|
||
"confidence": number
|
||
}
|
||
],
|
||
"totalMinor": number | null,
|
||
"discountTotalMinor": number | null,
|
||
"confidence": number
|
||
}
|
||
|
||
Använd endast de enhetskoder du kan utläsa (GRAM, KILOGRAM, LITER, etc.), annars null. quantity är alltid ett decimaltal.
|
||
Språk: ${lang}.`;
|
||
}
|
||
|
||
private defaultLocale(): LocaleContext {
|
||
return {
|
||
languageTag: "sv-SE",
|
||
regionCode: "SE",
|
||
timeZone: "Europe/Stockholm",
|
||
measurementSystem: "METRIC",
|
||
temperatureUnit: "CELSIUS",
|
||
currencyCode: "SEK",
|
||
};
|
||
}
|
||
|
||
private mimeType(buf: Buffer): string {
|
||
if (buf[0] === 0xff && buf[1] === 0xd8) return "image/jpeg";
|
||
if (buf.slice(0, 8).toString("hex") === "89504e470d0a1a0a") return "image/png";
|
||
if (buf.slice(0, 4).toString("ascii") === "RIFF") return "image/webp";
|
||
return "image/jpeg";
|
||
}
|
||
|
||
private estimateCostUsd(inputTokens: number, outputTokens: number): number {
|
||
// Gemini 2.5 Flash pricing (Aug 2025): ~$0.075/1M input, $0.30/1M output.
|
||
return inputTokens * 0.075e-6 + outputTokens * 0.3e-6;
|
||
}
|
||
|
||
private async isOverBudget(estimateUsd: number): Promise<boolean> {
|
||
if (this.cfg.dailyBudgetUsd <= 0 || !this.cfg.budgetStore) return false;
|
||
const current = await this.cfg.budgetStore.getDailySpendUsd();
|
||
const over = current + estimateUsd > this.cfg.dailyBudgetUsd;
|
||
if (over && this.cfg.budgetStore.raiseBudgetAlarm) {
|
||
await this.cfg.budgetStore.raiseBudgetAlarm();
|
||
}
|
||
return over;
|
||
}
|
||
|
||
private async recordSpend(costUsd: number): Promise<void> {
|
||
if (this.cfg.budgetStore && costUsd > 0) {
|
||
await this.cfg.budgetStore.incrementDailySpendUsd(costUsd);
|
||
}
|
||
}
|
||
}
|