Initial commit (unpacked platform)

2026-08-05 19:21:11 +07:00
commit ac5340195a
314 changed files with 57584 additions and 0 deletions
@@ -0,0 +1,193 @@
+import type { AamosTaskType, TaskInput, TaskOutput } from "@app/ai-contracts";
+import { numbersPreserved } from "../processors/translation.js";
+
+/**
+ * Utvärderingssvit för AAMOS (spec §40 – "ingen modell-/promptändring i
+ * produktion utan godkänd utvärdering").
+ *
+ * Golden cases med SEMANTISKA kontroller (inte exakt-matchning): riktiga
+ * modellsvar varierar, men egenskaperna nedan måste alltid hålla. Samma svit
+ * körs mot mocken i dev/CI och mot riktiga AAMOS före varje produktionsbyte –
+ * bara AAMOS_MODE skiljer.
+ */
+
+export interface EvalCheck {
+  name: string;
+  passed: boolean;
+}
+
+export interface EvalCase<T extends AamosTaskType = AamosTaskType> {
+  id: string;
+  taskType: T;
+  descriptionSv: string;
+  input: TaskInput<T>;
+  /** Returnerar kontrollresultat – varje kontroll är en produktegenskap. */
+  verify(output: TaskOutput<T>): EvalCheck[];
+}
+
+const check = (name: string, passed: boolean): EvalCheck => ({ name, passed });
+
+function confidenceInRange(values: number[]): boolean {
+  return values.every((c) => c >= 0 && c <= 1);
+}
+
+export const EVAL_CASES: EvalCase[] = [
+  {
+    id: "fridge-basic",
+    taskType: "ANALYZE_FRIDGE_IMAGE",
+    descriptionSv: "Kylskåpsbild: objekt hittas, osäkerhet uttrycks ärligt, enheter är koder",
+    input: {
+      imageUrls: ["https://example.com/eval/fridge-1.jpg"],
+      locationType: "fridge",
+      marketLocale: "sv-SE",
+      knownItems: [],
+    },
+    verify(output) {
+      const o = output as TaskOutput<"ANALYZE_FRIDGE_IMAGE">;
+      return [
+        check("minst ett objekt detekteras", o.items.length >= 1),
+        check("alla confidence i [0,1]", confidenceInRange(o.items.map((i) => i.confidence))),
+        check(
+          "osäkra objekt kräver bekräftelse (spec §61.4)",
+          o.items.every((i) => i.confidence >= 0.7 || i.requiresConfirmation),
+        ),
+        check(
+          "okänd vara tillåts vara okänd – aldrig påhittat id med hög confidence",
+          o.items.every((i) => i.canonicalIngredientId !== null || i.confidence <= 0.9),
+        ),
+        check(
+          "kvantiteter är positiva när de anges",
+          o.items.every((i) => i.estimatedQuantity == null || i.estimatedQuantity > 0),
+        ),
+      ];
+    },
+  },
+  {
+    id: "receipt-consistency",
+    taskType: "READ_RECEIPT",
+    descriptionSv: "Kvitto: belopp i minor units, radsumma stämmer med totalen",
+    input: { imageUrls: ["https://example.com/eval/receipt-1.jpg"], marketLocale: "sv-SE" },
+    verify(output) {
+      const o = output as TaskOutput<"READ_RECEIPT">;
+      const lineSum = o.lines
+        .filter((l) => !l.isDiscount)
+        .reduce((s, l) => s + (l.totalPriceMinor ?? 0), 0);
+      const total = o.totalMinor ?? 0;
+      return [
+        check("minst en kvittorad", o.lines.length >= 1),
+        check(
+          "alla belopp är heltal (minor units, i18n §20)",
+          o.lines.every(
+            (l) =>
+              (l.unitPriceMinor == null || Number.isInteger(l.unitPriceMinor)) &&
+              (l.totalPriceMinor == null || Number.isInteger(l.totalPriceMinor)),
+          ) && Number.isInteger(total),
+        ),
+        check(
+          "radsumma ≈ total (±5 % för avrundning/pant/rabatter)",
+          total === 0 || Math.abs(lineSum - (o.discountTotalMinor ?? 0) - total) / total <= 0.05,
+        ),
+        check("rad-confidence i [0,1]", confidenceInRange(o.lines.map((l) => l.confidence))),
+      ];
+    },
+  },
+  {
+    id: "craving-swedish",
+    taskType: "PARSE_CRAVING",
+    descriptionSv: "Fritext-sug på svenska blir strukturerade filter",
+    input: { text: "något krämigt med kyckling under 500 kcal", marketLocale: "sv-SE" },
+    verify(output) {
+      const o = output as TaskOutput<"PARSE_CRAVING">;
+      return [
+        check(
+          "tolkningen är inte tom (taggar/kök/kcal)",
+          o.tags.length > 0 || o.cuisine != null || o.maxKcal != null,
+        ),
+        check("maxKcal rimlig när satt", o.maxKcal == null || (o.maxKcal > 0 && o.maxKcal <= 3000)),
+        check("confidence i [0,1]", confidenceInRange([o.confidence])),
+      ];
+    },
+  },
+  {
+    id: "translate-preserves-structure",
+    taskType: "TRANSLATE_RECIPE",
+    descriptionSv: "Översättning: stegantal och alla tal bevaras (M3-verifieringen)",
+    input: {
+      sourceLanguageTag: "sv",
+      targetLanguageTag: "en",
+      title: "Ugnspannkaka med 225 g fläsk",
+      description: "Klassiker i 200 grader.",
+      storageGuidance: null,
+      steps: [
+        { stepNumber: 1, instruction: "Sätt ugnen på 200 grader.", tip: null },
+        { stepNumber: 2, instruction: "Vispa 4 ägg med 6 dl mjölk i 2 minuter.", tip: null },
+      ],
+    },
+    verify(output) {
+      const o = output as TaskOutput<"TRANSLATE_RECIPE">;
+      const sourceTexts = [
+        "Ugnspannkaka med 225 g fläsk",
+        "Klassiker i 200 grader.",
+        "Sätt ugnen på 200 grader.",
+        "Vispa 4 ägg med 6 dl mjölk i 2 minuter.",
+      ];
+      const targetTexts = [o.title, o.description ?? "", ...o.steps.map((s) => s.instruction)];
+      return [
+        check("stegantal bevarat", o.steps.length === 2),
+        check("stegnummer bevarade", o.steps.map((s) => s.stepNumber).join(",") === "1,2"),
+        check(
+          "alla tal bevarade (aldrig konverterade)",
+          numbersPreserved(sourceTexts, targetTexts),
+        ),
+        check("titel ej tom", o.title.trim().length > 0),
+      ];
+    },
+  },
+  {
+    id: "nutrition-label",
+    taskType: "READ_NUTRITION_LABEL",
+    descriptionSv: "Näringsetikett: värden per 100 med rimliga intervall",
+    input: { imageUrls: ["https://example.com/eval/label-1.jpg"], marketLocale: "sv-SE" },
+    verify(output) {
+      const o = output as TaskOutput<"READ_NUTRITION_LABEL">;
+      const n = o.values ?? { kcal: null, proteinG: null, carbsG: null, fatG: null };
+      return [
+        check("basis identifierad", o.basis != null),
+        check("kcal rimlig när satt (0–900/100)", n.kcal == null || (n.kcal >= 0 && n.kcal <= 900)),
+        check(
+          "makron icke-negativa när satta",
+          [n.proteinG, n.carbsG, n.fatG].every((v) => v == null || v >= 0),
+        ),
+        check(
+          "makron ryms i 100 g när alla satta",
+          n.proteinG == null || n.carbsG == null || n.fatG == null
+            ? true
+            : n.proteinG + n.carbsG + n.fatG <= 110,
+        ),
+        check("confidence i [0,1]", confidenceInRange([o.confidence])),
+      ];
+    },
+  },
+  {
+    id: "moderation-safe-recipe",
+    taskType: "MODERATE_RECIPE",
+    descriptionSv: "Ofarligt recept flaggas inte som reject",
+    input: {
+      titleSv: "Havregrynsgröt",
+      descriptionSv: "Enkel frukostgröt.",
+      ingredients: ["1 DECILITER havregryn", "2 DECILITER vatten", "1 PINCH salt"],
+      steps: ["Koka upp vatten.", "Rör ner gryn och sjud 3 minuter."],
+    },
+    verify(output) {
+      const o = output as TaskOutput<"MODERATE_RECIPE">;
+      return [
+        check("rekommendationen är inte reject", o.recommendation !== "reject"),
+        check(
+          "inga reject-flaggor",
+          o.flags.every((f) => f.severity !== "reject"),
+        ),
+        check("confidence i [0,1]", confidenceInRange([o.confidence])),
+      ];
+    },
+  },
+];
@@ -0,0 +1,93 @@
+import { config as loadDotenv } from "dotenv";
+import { existsSync } from "node:fs";
+import path from "node:path";
+for (const candidate of [".env", "../.env", "../../.env"]) {
+  const p = path.resolve(process.cwd(), candidate);
+  if (existsSync(p)) {
+    loadDotenv({ path: p });
+    break;
+  }
+}
+
+import { schema } from "@app/database";
+import { createContext } from "../context.js";
+import { EVAL_CASES } from "./cases.js";
+
+/**
+ * Kör AAMOS-utvärderingssviten (spec §40).
+ *
+ *   pnpm --filter @app/worker eval          # mot AAMOS_MODE i .env (mock i dev)
+ *   AAMOS_MODE=http AAMOS_API_URL=… pnpm --filter @app/worker eval   # mot riktig AAMOS
+ *
+ * Resultatet skrivs till ai_eval_runs (syns i adminpanelen) och processen
+ * avslutas med exitkod 1 om någon kontroll faller – körbar som CI-grind
+ * inför varje modell-/promptbyte i AAMOS.
+ */
+async function main() {
+  const ctx = createContext();
+  const mode = process.env.AAMOS_MODE ?? "mock";
+  console.log(`[eval] AAMOS-utvärdering – läge: ${mode}, ${EVAL_CASES.length} fall\n`);
+
+  let totalChecks = 0;
+  let failedChecks = 0;
+
+  for (const evalCase of EVAL_CASES) {
+    const started = Date.now();
+    let checks: { name: string; passed: boolean }[] = [];
+    let error: string | null = null;
+    try {
+      const result = await ctx.aamos.runTask(evalCase.taskType, evalCase.input as never);
+      if (result.status !== "ok" || !result.output) {
+        error = `status=${result.status}`;
+        checks = [{ name: "AAMOS svarade ok", passed: false }];
+      } else {
+        checks = [
+          { name: "AAMOS svarade ok", passed: true },
+          ...evalCase.verify(result.output as never),
+        ];
+      }
+    } catch (err) {
+      error = (err as Error).message;
+      checks = [{ name: "AAMOS svarade ok", passed: false }];
+    }
+    const ms = Date.now() - started;
+    const passed = checks.every((c) => c.passed);
+    totalChecks += checks.length;
+    failedChecks += checks.filter((c) => !c.passed).length;
+
+    console.log(`${passed ? "✓" : "✗"} ${evalCase.id} (${evalCase.taskType}, ${ms} ms)`);
+    for (const c of checks) if (!c.passed) console.log(`    ✗ ${c.name}`);
+    if (error) console.log(`    fel: ${error}`);
+
+    await ctx.db.insert(schema.aiEvalRuns).values({
+      taskType: evalCase.taskType,
+      modelVersion: `aamos-${mode}`,
+      promptVersion: "eval-suite-v1",
+      metrics: {
+        caseId: evalCase.id,
+        durationMs: ms,
+        checks,
+        ...(error ? { error } : {}),
+      },
+      passed,
+      notes: evalCase.descriptionSv,
+    });
+  }
+
+  const failedCases = failedChecks > 0;
+  console.log(
+    `\n[eval] ${EVAL_CASES.length} fall, ${totalChecks} kontroller, ${failedChecks} fallerade.`,
+  );
+  console.log(
+    failedCases
+      ? "[eval] UNDERKÄND – åtgärda innan modell-/promptbytet tas i produktion (spec §40)."
+      : "[eval] GODKÄND – resultatet är loggat i ai_eval_runs.",
+  );
+  await ctx.close();
+  process.exit(failedCases ? 1 : 0);
+}
+
+main().catch((err) => {
+  console.error("[eval] KRASCH:", err);
+  process.exit(1);
+});