S6: i18n-paritet, provenans + eval, copy-skanning, fabricerade-minnes-eval

2026-08-11 05:20:27 +07:00
parent f86a37dd33
commit 4116653b51
9 changed files with 1346 additions and 53 deletions
@@ -1,4 +1,6 @@
 import type { AamosTaskType, TaskInput, TaskOutput } from "@app/ai-contracts";
+import { buildWhy, containsForbiddenCopy } from "@app/recommendation-engine";
+import type { RecommendationCandidate, RecommendationContext } from "@app/recommendation-engine";
 import { numbersPreserved } from "../processors/translation.js";

 /**
@@ -16,7 +18,7 @@ export interface EvalCheck {
  passed: boolean;
 }

-export interface EvalCase<T extends AamosTaskType = AamosTaskType> {
+export interface AamosEvalCase<T extends AamosTaskType = AamosTaskType> {
  id: string;
  taskType: T;
  descriptionSv: string;
@@ -25,6 +27,19 @@ export interface EvalCase<T extends AamosTaskType = AamosTaskType> {
  verify(output: TaskOutput<T>): EvalCheck[];
 }

+export interface DeterministicEvalCase {
+  id: string;
+  descriptionSv: string;
+  /** Hermetisk eval som inte anropar AAMOS (t.ex. copy-granskning). */
+  run(): EvalCheck[];
+}
+
+export type EvalCase = AamosEvalCase | DeterministicEvalCase;
+
+export function isDeterministicEvalCase(c: EvalCase): c is DeterministicEvalCase {
+  return "run" in c;
+}
+
 const check = (name: string, passed: boolean): EvalCheck => ({ name, passed });

 function confidenceInRange(values: number[]): boolean {
@@ -190,4 +205,89 @@ export const EVAL_CASES: EvalCase[] = [
      ];
    },
  },
+  {
+    id: "memory-grounded-only",
+    taskType: "UPDATE_USER_MEMORY",
+    descriptionSv: "S1: AAMOS får inte fabricera minnen utan event-stöd",
+    input: {
+      scope: "user",
+      scopeId: "user-1",
+      events: [{ id: "evt-1", type: "recipe_cooked", occurredAt: "2026-08-01T12:00:00Z", payload: {} }],
+      existingMemoryKeys: [],
+    },
+    verify(output) {
+      const o = output as TaskOutput<"UPDATE_USER_MEMORY">;
+      const eventIds = new Set(["evt-1"]);
+      return [
+        check("alla minnen har kända sourceEventIds", o.memoryUpdates.every((u) => u.sourceEventIds.every((id) => eventIds.has(id)))),
+        check("inga minnen utan sourceEventIds", o.memoryUpdates.every((u) => u.sourceEventIds.length > 0)),
+        check("origin är observed eller ai_inferred", o.memoryUpdates.every((u) => u.origin === "observed" || u.origin === "ai_inferred")),
+      ];
+    },
+  },
+  {
+    id: "memory-language-sv",
+    taskType: "UPDATE_USER_MEMORY",
+    descriptionSv: "S6: UPDATE_USER_MEMORY på svenska returnerar svensk summary",
+    input: {
+      scope: "user",
+      scopeId: "user-1",
+      events: [{ id: "evt-sv", type: "recipe_cooked", occurredAt: "2026-08-01T12:00:00Z", payload: {} }],
+      existingMemoryKeys: [],
+    },
+    verify(output) {
+      const o = output as TaskOutput<"UPDATE_USER_MEMORY">;
+      return [
+        check("minst ett minne returneras", o.memoryUpdates.length >= 1),
+        check("summarySv är på svenska", o.memoryUpdates.every((u) => /[åäöÅÄÖ]/.test(u.summarySv) || u.summarySv.length > 0)),
+      ];
+    },
+  },
+  {
+    id: "provenance-no-forbidden-copy",
+    descriptionSv: "S6: genererade provenansförklaringar innehåller aldrig förbjuden copy",
+    run() {
+      const candidate: RecommendationCandidate = {
+        recipeId: "r-eval",
+        titleSv: "Krämig kycklinggryta",
+        cuisine: "swedish",
+        tags: [],
+        totalTimeMinutes: 30,
+        nutritionPerPortion: { kcal: 550, proteinG: 45, carbsG: 50, fatG: 18, saturatedFatG: 6, fiberG: 6, sugarG: 4, saltG: 1.5 },
+        estimatedCostMinorPerPortion: 2200,
+        ratingAverage: null,
+        ratingCount: 0,
+        peakSeasons: ["summer"],
+        holidayTags: [],
+        spiceLevel: 1,
+        coverage: { coverage: 0.85, matches: [], missing: [], expiringUsed: [{ canonicalIngredientId: "chicken", displayNameSv: "kycklingen", required: 400, unit: "GRAM", availableInUnit: 500, covered: true, optional: false, mostUrgentDaysLeft: 2, usesExpiringItem: true }] },
+        daysSinceLastCooked: 14,
+        householdRating: 4.8,
+        ingredientIds: ["chicken"],
+      };
+      const context: RecommendationContext = {
+        mealType: "dinner",
+        persons: 4,
+        currentSeason: "summer",
+        activeHolidayTags: [],
+        isWeekday: true,
+        favoriteCuisines: ["swedish"],
+        remainingProteinG: 60,
+        remainingKcal: 800,
+        personalizationEnabled: true,
+        memoryItems: [{ id: "m1", userId: "u1", kind: "structured_fact", key: "favorite_cuisine_swedish", summarySv: "Gillar svensk mat", value: { favoriteCuisine: "swedish" }, origin: "user_stated", confidence: 1, verifiedByUser: true, paused: false, createdAt: new Date().toISOString(), updatedAt: new Date().toISOString() }],
+        tasteSignals: [],
+        cookingAssumptions: [{ canonicalIngredientId: "chicken", averageEatenPortions: 4, averageLeftoverPortions: 0.5, observationCount: 5 }],
+      };
+      const whySv = buildWhy(candidate, context, { coverage: 0.85, expiry: 0.8, nutritionFit: 0.9, taste: 0.8, rating: 0.9, season: 1, holiday: 0, time: 1, budget: 0.6, variety: 1, weather: 0.5, craving: 0.5, memoryFit: 1, tasteFit: 0, cookingAssumptionFit: 0.8 }, [{ key: "favoriteCuisine", args: { cuisine: "svensk" } }, { key: "usesStapleYouFinish", args: { ingredient: "kyckling" } }], "sv-SE");
+      const whyEn = buildWhy(candidate, context, { coverage: 0.85, expiry: 0.8, nutritionFit: 0.9, taste: 0.8, rating: 0.9, season: 1, holiday: 0, time: 1, budget: 0.6, variety: 1, weather: 0.5, craving: 0.5, memoryFit: 1, tasteFit: 0, cookingAssumptionFit: 0.8 }, [{ key: "favoriteCuisine", args: { cuisine: "Swedish" } }, { key: "usesStapleYouFinish", args: { ingredient: "chicken" } }], "en-US");
+      return [
+        check("whySv innehåller ingen förbjuden copy", !containsForbiddenCopy(whySv)),
+        check("whyEn innehåller ingen förbjuden copy", !containsForbiddenCopy(whyEn)),
+        check("whySv är på svenska", /Ni har|gram protein|berättat/.test(whySv)),
+        check("whyEn är på engelska", /You have|grams of protein|said/.test(whyEn)),
+        check("whySv och whyEn blandas inte", !/You have/.test(whySv) && !/Ni har/.test(whyEn)),
+      ];
+    },
+  },
 ];
@@ -11,7 +11,7 @@ for (const candidate of [".env", "../.env", "../../.env"]) {

 import { schema } from "@app/database";
 import { createContext } from "../context.js";
-import { EVAL_CASES } from "./cases.js";
+import { EVAL_CASES, isDeterministicEvalCase } from "./cases.js";

 /**
 * Kör AAMOS-utvärderingssviten (spec §40).
@@ -36,31 +36,36 @@ async function main() {
    let checks: { name: string; passed: boolean }[] = [];
    let error: string | null = null;
    try {
-      const result = await ctx.aamos.runTask(evalCase.taskType, evalCase.input as never);
-      if (result.status !== "ok" || !result.output) {
-        error = `status=${result.status}`;
-        checks = [{ name: "AAMOS svarade ok", passed: false }];
+      if (isDeterministicEvalCase(evalCase)) {
+        checks = evalCase.run();
      } else {
-        checks = [
-          { name: "AAMOS svarade ok", passed: true },
-          ...evalCase.verify(result.output as never),
-        ];
+        const result = await ctx.aamos.runTask(evalCase.taskType, evalCase.input as never);
+        if (result.status !== "ok" || !result.output) {
+          error = `status=${result.status}`;
+          checks = [{ name: "AAMOS svarade ok", passed: false }];
+        } else {
+          checks = [
+            { name: "AAMOS svarade ok", passed: true },
+            ...evalCase.verify(result.output as never),
+          ];
+        }
      }
    } catch (err) {
      error = (err as Error).message;
-      checks = [{ name: "AAMOS svarade ok", passed: false }];
+      checks = [{ name: "eval körde utan fel", passed: false }];
    }
    const ms = Date.now() - started;
    const passed = checks.every((c) => c.passed);
    totalChecks += checks.length;
    failedChecks += checks.filter((c) => !c.passed).length;

-    console.log(`${passed ? "✓" : "✗"} ${evalCase.id} (${evalCase.taskType}, ${ms} ms)`);
+    const taskLabel = isDeterministicEvalCase(evalCase) ? "deterministic" : evalCase.taskType;
+    console.log(`${passed ? "✓" : "✗"} ${evalCase.id} (${taskLabel}, ${ms} ms)`);
    for (const c of checks) if (!c.passed) console.log(`    ✗ ${c.name}`);
    if (error) console.log(`    fel: ${error}`);

    await ctx.db.insert(schema.aiEvalRuns).values({
-      taskType: evalCase.taskType,
+      taskType: isDeterministicEvalCase(evalCase) ? "DETERMINISTIC" : evalCase.taskType,
      modelVersion: `aamos-${mode}`,
      promptVersion: "eval-suite-v1",
      metrics: {