feat(storage,deploy): delad S3/mock-lagring, SSM .env-sync, Gemini readyz
- @app/storage med MockStorage + AwsStorage delas mellan API och worker - Workerns readUrl() ger presignerad S3-URL i aws-läge (fixar mock-s3 404) - sync-env-from-ssm.py hämtar /cibello/prod/* och skriver .env; first-deploy.sh kör det i prod - /readyz returnerar 503 om app.aamos.healthCheck() misslyckas - docs/SECRETS.md med rotations- och SSM-regler
This commit is contained in:
@@ -25,12 +25,12 @@
|
||||
"@app/recipe-engine": "workspace:*",
|
||||
"@app/recommendation-engine": "workspace:*",
|
||||
"@app/shared-types": "workspace:*",
|
||||
"@app/storage": "workspace:*",
|
||||
"@app/subscriptions": "workspace:*",
|
||||
"@aws-sdk/client-s3": "^3.750.0",
|
||||
"@aws-sdk/client-s3": "^3.1102.0",
|
||||
"bullmq": "^6.0.0",
|
||||
"dotenv": "^16.4.0",
|
||||
"drizzle-orm": "^0.45.0",
|
||||
"@aws-sdk/client-s3": "^3.750.0",
|
||||
"ioredis": "^6.0.0",
|
||||
"pg": "^8.13.0"
|
||||
},
|
||||
@@ -50,6 +50,7 @@
|
||||
"@app/recipe-engine",
|
||||
"@app/recommendation-engine",
|
||||
"@app/shared-types",
|
||||
"@app/storage",
|
||||
"@app/subscriptions"
|
||||
]
|
||||
}
|
||||
|
||||
+27
-12
@@ -10,9 +10,9 @@ for (const candidate of [".env", "../.env", "../../.env"]) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
import { createHmac } from "node:crypto";
|
||||
import { createDatabase, type Database } from "@app/database";
|
||||
import { createAamosClient, type AamosClient, type BudgetStore } from "@app/ai-contracts";
|
||||
import { createStorageService, type StorageService } from "@app/storage";
|
||||
import type { Redis } from "ioredis";
|
||||
|
||||
export interface WorkerContext {
|
||||
@@ -21,8 +21,10 @@ export interface WorkerContext {
|
||||
redis?: Redis;
|
||||
/** Delad dagsbudget-store så att memory-jobbet kan spegla skanningens budgetkoll. */
|
||||
budgetStore?: BudgetStore;
|
||||
/** Bygger läs-URL för lagrade bilder (samma signaturlogik som API:ts mock-S3). */
|
||||
readUrl: (key: string) => string;
|
||||
/** Lagringstjänst för att läsa/skriva bilder (S3 eller mock). */
|
||||
storage: StorageService;
|
||||
/** Bygger läs-URL för lagrade bilder (presignerad S3 i aws-läge, mock-URL lokalt). */
|
||||
readUrl: (key: string) => Promise<string>;
|
||||
apiBaseUrl: string;
|
||||
close: () => Promise<void>;
|
||||
}
|
||||
@@ -64,29 +66,42 @@ class RedisBudgetStore implements BudgetStore {
|
||||
}
|
||||
}
|
||||
|
||||
function requireEnv(name: string): string {
|
||||
const value = process.env[name];
|
||||
if (!value) throw new Error(`Missing ${name}. Worker must be configured explicitly.`);
|
||||
return value;
|
||||
}
|
||||
|
||||
export function createContext(redis?: Redis): WorkerContext {
|
||||
const databaseUrl = process.env.DATABASE_URL;
|
||||
if (!databaseUrl) {
|
||||
throw new Error("Missing DATABASE_URL. Worker must connect explicitly to the app database.");
|
||||
}
|
||||
const databaseUrl = requireEnv("DATABASE_URL");
|
||||
const { db, pool } = createDatabase(databaseUrl);
|
||||
const budgetStore = redis
|
||||
? new RedisBudgetStore(redis, `gemini:daily:budget:${new Date().toISOString().slice(0, 10)}`)
|
||||
: undefined;
|
||||
const aamos = createAamosClient(undefined, { budgetStore });
|
||||
const apiBaseUrl = process.env.API_BASE_URL ?? "http://localhost:4000";
|
||||
const secret = process.env.ENTITLEMENT_SIGNING_SECRET ?? "dev-only-change-me-three";
|
||||
const signingSecret = requireEnv("ENTITLEMENT_SIGNING_SECRET");
|
||||
|
||||
const s3Mode = process.env.S3_MODE === "aws" ? "aws" : "mock";
|
||||
const storage = createStorageService({
|
||||
mode: s3Mode,
|
||||
baseUrl: apiBaseUrl,
|
||||
signingSecret,
|
||||
s3Bucket: process.env.S3_BUCKET ?? "",
|
||||
s3Region: process.env.S3_REGION ?? "",
|
||||
s3Endpoint: process.env.S3_ENDPOINT || undefined,
|
||||
s3AccessKeyId: process.env.S3_ACCESS_KEY_ID || undefined,
|
||||
s3SecretAccessKey: process.env.S3_SECRET_ACCESS_KEY || undefined,
|
||||
});
|
||||
|
||||
return {
|
||||
db,
|
||||
aamos,
|
||||
redis,
|
||||
budgetStore,
|
||||
storage,
|
||||
apiBaseUrl,
|
||||
readUrl: (key: string) => {
|
||||
const sig = createHmac("sha256", secret).update(key).digest("hex").slice(0, 32);
|
||||
return `${apiBaseUrl}/v1/mock-s3/${encodeURIComponent(key)}?sig=${sig}`;
|
||||
},
|
||||
readUrl: (key: string) => storage.getReadUrl(key),
|
||||
close: async () => {
|
||||
await pool.end();
|
||||
},
|
||||
|
||||
@@ -47,7 +47,7 @@ async function main() {
|
||||
anonymizedImprovement: true,
|
||||
imageTraining: true,
|
||||
},
|
||||
readUrl: (key: string) => {
|
||||
readUrl: async (key: string) => {
|
||||
if (key === "eval/fridge-1.jpg") return fridgeDataUrl;
|
||||
return `http://localhost/v1/mock-s3/${encodeURIComponent(key)}?sig=dummy`;
|
||||
},
|
||||
|
||||
@@ -39,7 +39,7 @@ export interface CaptureInput {
|
||||
inputTokens?: number | null;
|
||||
outputTokens?: number | null;
|
||||
consentFlags: CaptureConsentFlags;
|
||||
readUrl: (key: string) => string;
|
||||
readUrl: (key: string) => Promise<string>;
|
||||
/** Override for tests; otherwise built from env. */
|
||||
storage?: CaptureStorage;
|
||||
}
|
||||
@@ -138,7 +138,7 @@ export async function captureTrainingSample(input: CaptureInput): Promise<Captur
|
||||
if (!originalKey) continue;
|
||||
try {
|
||||
const imageKey = `${prefix}/images/${String(i).padStart(3, "0")}.jpg`;
|
||||
const imageData = await fetchImageData(input.readUrl(originalKey));
|
||||
const imageData = await fetchImageData(await input.readUrl(originalKey));
|
||||
if (imageData) {
|
||||
await storage.put(imageKey, imageData, "image/jpeg");
|
||||
trainingImageKeys.push(imageKey);
|
||||
|
||||
@@ -27,7 +27,7 @@ export async function processScanJob(ctx: WorkerContext, scanJobId: string): Pro
|
||||
.set({ status: "running", attempts: job.attempts + 1, updatedAt: new Date() })
|
||||
.where(eq(schema.scanJobs.id, scanJobId));
|
||||
|
||||
const imageUrls = job.s3Keys.map((k) => ctx.readUrl(k));
|
||||
const imageUrls = await Promise.all(job.s3Keys.map((k) => ctx.readUrl(k)));
|
||||
const localeContext = await getLocaleContext(ctx, job.userId);
|
||||
const consentFlags = await loadConsentFlags(ctx, job.userId);
|
||||
|
||||
|
||||
@@ -86,7 +86,8 @@ describe("UPDATE_USER_MEMORY hardening", () => {
|
||||
aamos,
|
||||
budgetStore,
|
||||
apiBaseUrl: "http://localhost:4000",
|
||||
readUrl: (key: string) => `http://localhost:4000/mock-s3/${key}`,
|
||||
storage: {} as any,
|
||||
readUrl: async (key: string) => `http://localhost:4000/mock-s3/${key}`,
|
||||
close: async () => {},
|
||||
};
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user