From 6e2d7941309eff10f32676fa93e780ab70c6b167 Mon Sep 17 00:00:00 2001 From: "Sven (AAMOS AI)" Date: Wed, 12 Aug 2026 17:28:58 +0700 Subject: [PATCH] feat(storage,deploy): delad S3/mock-lagring, SSM .env-sync, Gemini readyz MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - @app/storage med MockStorage + AwsStorage delas mellan API och worker - Workerns readUrl() ger presignerad S3-URL i aws-läge (fixar mock-s3 404) - sync-env-from-ssm.py hämtar /cibello/prod/* och skriver .env; first-deploy.sh kör det i prod - /readyz returnerar 503 om app.aamos.healthCheck() misslyckas - docs/SECRETS.md med rotations- och SSM-regler --- HANDOFF.md | 20 +- apps/api/package.json | 2 + apps/api/src/plugins/storage.ts | 225 ++---------------- apps/api/src/routes/health.ts | 6 +- apps/worker/package.json | 5 +- apps/worker/src/context.ts | 39 ++- apps/worker/src/eval/capture-eval.ts | 2 +- apps/worker/src/lib/shadow-capture.ts | 4 +- apps/worker/src/processors/scans.ts | 2 +- apps/worker/test/memory-sync.test.ts | 3 +- docs/SECRETS.md | 49 ++++ infrastructure/deployment/first-deploy.sh | 10 + .../deployment/sync-env-from-ssm.py | 129 ++++++++++ packages/storage/package.json | 18 ++ packages/storage/src/index.ts | 219 +++++++++++++++++ packages/storage/tsconfig.json | 7 + pnpm-lock.yaml | 17 +- 17 files changed, 516 insertions(+), 241 deletions(-) create mode 100644 docs/SECRETS.md create mode 100644 infrastructure/deployment/sync-env-from-ssm.py create mode 100644 packages/storage/package.json create mode 100644 packages/storage/src/index.ts create mode 100644 packages/storage/tsconfig.json diff --git a/HANDOFF.md b/HANDOFF.md index 86aed6e..40cb5d5 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -4,9 +4,11 @@ pnpm/turbo-monorepo: Fastify+Drizzle+BullMQ (apps/api, apps/worker), Expo (apps/ Roller: OpenClaw bygger. Extern granskare ("Sven", i Johans chatt) verifierar + ger promptar. ## Var saker finns -- Prod-server (EC2): 13.48.40.1, repo /opt/cibello-platform, HEAD 81927b6 +- Prod-server (EC2): 13.48.40.1, repo /opt/cibello-platform - OpenClaw-workspace: cibello-work/cibello - Region: eu-north-1 +- SSM: `/cibello/prod/*` (enda sanningskälla för hemligheter) +- `.env` på servern härleds ur SSM via `infrastructure/deployment/sync-env-from-ssm.py` ## Infra (uppe) - EIP 13.48.40.1 → api.cibello.app + admin.cibello.app @@ -36,6 +38,10 @@ Roller: OpenClaw bygger. Extern granskare ("Sven", i Johans chatt) verifierar + - SMTP-pass för hello@cibello.app. ## KLART +- [x] **Skarpt bildflöde:** delad lagringsmodul `@app/storage` (MockStorage + AwsStorage) används av både API och worker. Workerns `readUrl()` ger nu riktig presignerad S3-URL i `aws`-läge istället för `/v1/mock-s3/...`. +- [x] **SSM = enda sanningskälla:** `infrastructure/deployment/sync-env-from-ssm.py` hämtar `/cibello/prod/*` och skriver `/opt/cibello-platform/.env`. `first-deploy.sh` kör detta i produktion. +- [x] **Gemini-nyckel hälsokoll:** `/readyz` returnerar 503 om `app.aamos.healthCheck()` misslyckas (t.ex. ogiltig Gemini-nyckel). +- [x] Dokumenterat i `docs/SECRETS.md`. - [x] FAS 1 byggt: GeminiAamosClient + shadow-capture.ts + scans-integration. - [x] Gemini som committat prod-läge (AAMOS_MODE=gemini). - [x] Hermetiska eval-fixtures committade. @@ -61,14 +67,10 @@ Roller: OpenClaw bygger. Extern granskare ("Sven", i Johans chatt) verifierar + ## PENDING (i ordning) 1. **Egen Gemini-nyckel + SMTP-pass** — Johan levererar värden lokalt; claw skriver till SSM/.env. - Nuvarande `.env` innehåller fortfarande den delade `/openclaw`-Gemini-nyckeln. -2. **End-user samtyckes-gate** för bildträning + in-app allergen-disclaimer före publik lansering. -3. **Säkerhetsgranskning**: öppna portar, Caddy-konfig, IAM, S3-bucket-policy. -4. **Övervakning/larm**: logga fel, sentry, health-check alert. -5. **Worker-S3-reader**: `readUrl` i `apps/worker/src/context.ts` pekar fortfarande på - `/v1/mock-s3/` vilket ger 404 på riktiga S3-uppladdningar. För fullt skarpt - bildflöde behöver den returnera presignerad S3-URL (jmf eval-fixturer som använder - data URLs). Budget-spärren är verifierad trots detta; Gemini-anrop nås nu före - bildhämtning. +2. **Skarpt end-to-end-test** när Grupp B är på plats: riktig scan → S3 → Gemini → resultat + träningspar i `s3://cibello-production/training/cibello/`. +3. **End-user samtyckes-gate** för bildträning + in-app allergen-disclaimer före publik lansering. +4. **Säkerhetsgranskning**: öppna portar, Caddy-konfig, IAM, S3-bucket-policy. +5. **Övervakning/larm**: logga fel, sentry, health-check alert. ## Så här återupptar en färsk session Läs denna fil + `git log --oneline -8` + senaste prompt från Sven. Arbeta stramt. Echo ALDRIG diff --git a/apps/api/package.json b/apps/api/package.json index ac66159..d47366e 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -24,6 +24,7 @@ "@app/recipe-engine": "workspace:*", "@app/recommendation-engine": "workspace:*", "@app/shared-types": "workspace:*", + "@app/storage": "workspace:*", "@app/subscriptions": "workspace:*", "@app/validation": "workspace:*", "@aws-sdk/client-s3": "^3.1102.0", @@ -60,6 +61,7 @@ "@app/recipe-engine", "@app/recommendation-engine", "@app/shared-types", + "@app/storage", "@app/subscriptions", "@app/validation" ] diff --git a/apps/api/src/plugins/storage.ts b/apps/api/src/plugins/storage.ts index c88a08e..d04c12a 100644 --- a/apps/api/src/plugins/storage.ts +++ b/apps/api/src/plugins/storage.ts @@ -1,55 +1,11 @@ import fp from "fastify-plugin"; import type { FastifyInstance } from "fastify"; -import { createHmac, randomUUID } from "node:crypto"; -import { mkdir, readFile, rm, writeFile } from "node:fs/promises"; -import path from "node:path"; - -/** - * Lagringsabstraktion (spec §53). Två lägen: - * - * - mock (dev/test): "presignade" URL:er pekar på API:ts egna PUT/GET-endpoints - * och filer lagras under .data/s3/. Hela flödet (app → signed upload → - * worker läser) fungerar utan AWS. - * - aws (staging/produktion): RIKTIGA S3 presigned URLs via @aws-sdk/client-s3 - * + @aws-sdk/s3-request-presigner. Fungerar mot AWS S3 och alla - * S3-kompatibla lagringar (MinIO, Cloudflare R2, Hetzner …) via S3_ENDPOINT. - * Verifierad med riktig rundtur (presign → PUT → presign → GET) mot MinIO. - * - * Nyckelstruktur enligt spec §53: users/, fridge-scans/, pantry-scans/, - * meal-scans/, receipts/, product-images/, recipe-images/, temporary/. - */ import { - DeleteObjectCommand, - GetObjectCommand, - HeadBucketCommand, - PutObjectCommand, - S3Client, -} from "@aws-sdk/client-s3"; -import { getSignedUrl } from "@aws-sdk/s3-request-presigner"; - -export interface PresignedUpload { - key: string; - uploadUrl: string; - method: "PUT"; - headers: Record; - expiresAt: string; -} - -export interface StorageService { - presignUpload(prefix: string, contentType: string): Promise; - /** URL som workern/AAMOS kan läsa bilden från. */ - getReadUrl(key: string): Promise; - putObject(key: string, data: Buffer, contentType: string): Promise; - getObject(key: string): Promise; - /** GDPR / rättning: radera ett objekt. Får inte kasta om nyckeln saknas. */ - deleteObject(key: string): Promise; - /** - * Försök extrahera lagringsnyckeln ur en läs-URL. Returnerar null om URL:en - * inte känns igen (t.ex. extern bild). Används vid GDPR-radering så att vi - * kan radera objekt även när endast URL:en är sparad. - */ - extractKeyFromUrl(url: string): string | null; -} + MockStorage, + AwsStorage, + type StorageService, + createStorageService, +} from "@app/storage"; declare module "fastify" { interface FastifyInstance { @@ -57,171 +13,16 @@ declare module "fastify" { } } -const MOCK_ROOT = path.resolve(process.cwd(), ".data/s3"); - -class MockStorage implements StorageService { - constructor( - private readonly baseUrl: string, - private readonly signingSecret: string, - ) {} - - private sign(key: string): string { - return createHmac("sha256", this.signingSecret).update(key).digest("hex").slice(0, 32); - } - - async presignUpload(prefix: string, contentType: string): Promise { - const ext = contentType.split("/")[1] ?? "bin"; - const key = `${prefix}/${randomUUID()}.${ext}`; - const sig = this.sign(key); - return { - key, - uploadUrl: `${this.baseUrl}/v1/mock-s3/${encodeURIComponent(key)}?sig=${sig}`, - method: "PUT", - headers: { "content-type": contentType }, - expiresAt: new Date(Date.now() + 15 * 60_000).toISOString(), - }; - } - - async getReadUrl(key: string): Promise { - return `${this.baseUrl}/v1/mock-s3/${encodeURIComponent(key)}?sig=${this.sign(key)}`; - } - - async putObject(key: string, data: Buffer, _contentType?: string): Promise { - const filePath = path.join(MOCK_ROOT, key); - await mkdir(path.dirname(filePath), { recursive: true }); - await writeFile(filePath, data); - } - - async getObject(key: string): Promise { - try { - return await readFile(path.join(MOCK_ROOT, key)); - } catch { - return null; - } - } - - async deleteObject(key: string): Promise { - // GDPR-radering får inte avbrytas av saknade filer; rm med force swallowar felet. - await rm(path.join(MOCK_ROOT, key), { force: true }).catch(() => {}); - } - - extractKeyFromUrl(url: string): string | null { - try { - const parsed = new URL(url); - const prefix = "/v1/mock-s3/"; - if (!parsed.pathname.startsWith(prefix)) return null; - return decodeURIComponent(parsed.pathname.slice(prefix.length)); - } catch { - return null; - } - } - - verifySignature(key: string, sig: string): boolean { - return this.sign(key) === sig; - } -} - -/** Riktig S3 (AWS eller S3-kompatibel via S3_ENDPOINT). */ -export class AwsStorage implements StorageService { - private readonly client: S3Client; - - constructor( - private readonly bucket: string, - region: string, - endpoint: string | undefined, - accessKeyId: string | undefined, - secretAccessKey: string | undefined, - ) { - this.client = new S3Client({ - region, - ...(endpoint ? { endpoint, forcePathStyle: true } : {}), - ...(accessKeyId && secretAccessKey ? { credentials: { accessKeyId, secretAccessKey } } : {}), // annars IAM-roll/instansprofil (rekommenderat i AWS) - }); - } - - /** Snabb verifiering vid uppstart – hellre högljutt fel än trasiga uppladdningar. */ - async healthCheck(): Promise { - await this.client.send(new HeadBucketCommand({ Bucket: this.bucket })); - } - - async presignUpload(prefix: string, contentType: string): Promise { - const ext = contentType.split("/")[1] ?? "bin"; - const key = `${prefix}/${randomUUID()}.${ext}`; - const uploadUrl = await getSignedUrl( - this.client, - new PutObjectCommand({ Bucket: this.bucket, Key: key, ContentType: contentType }), - { expiresIn: 15 * 60 }, - ); - return { - key, - uploadUrl, - method: "PUT", - headers: { "content-type": contentType }, - expiresAt: new Date(Date.now() + 15 * 60_000).toISOString(), - }; - } - - async getReadUrl(key: string): Promise { - return getSignedUrl(this.client, new GetObjectCommand({ Bucket: this.bucket, Key: key }), { - expiresIn: 60 * 60, - }); - } - - async putObject(key: string, data: Buffer, contentType: string): Promise { - await this.client.send( - new PutObjectCommand({ Bucket: this.bucket, Key: key, Body: data, ContentType: contentType }), - ); - } - - async getObject(key: string): Promise { - try { - const res = await this.client.send(new GetObjectCommand({ Bucket: this.bucket, Key: key })); - const bytes = await res.Body?.transformToByteArray(); - return bytes ? Buffer.from(bytes) : null; - } catch { - return null; - } - } - - async deleteObject(key: string): Promise { - try { - await this.client.send(new DeleteObjectCommand({ Bucket: this.bucket, Key: key })); - } catch (err) { - const code = (err as { Code?: string }).Code; - // NoSuchKey ska inte avbryta GDPR-radering; andra fel loggas. - if (code !== "NoSuchKey" && code !== "NotFound") { - throw err; - } - } - } - - extractKeyFromUrl(url: string): string | null { - try { - const parsed = new URL(url); - if (parsed.hostname.startsWith(`${this.bucket}.`)) { - return decodeURIComponent(parsed.pathname.replace(/^\//, "")); - } - const pathPrefix = `/${this.bucket}/`; - if (parsed.pathname.startsWith(pathPrefix)) { - return decodeURIComponent(parsed.pathname.slice(pathPrefix.length)); - } - return null; - } catch { - return null; - } - } -} - export const storagePlugin = fp(async (app: FastifyInstance) => { let storage: StorageService; if (app.config.S3_MODE === "aws") { - const aws = new AwsStorage( - app.config.S3_BUCKET, - app.config.S3_REGION, - app.config.S3_ENDPOINT || undefined, - app.config.S3_ACCESS_KEY_ID || undefined, - app.config.S3_SECRET_ACCESS_KEY || undefined, - ); + const aws = new AwsStorage({ + bucket: app.config.S3_BUCKET, + region: app.config.S3_REGION, + endpoint: app.config.S3_ENDPOINT || undefined, + accessKeyId: app.config.S3_ACCESS_KEY_ID || undefined, + secretAccessKey: app.config.S3_SECRET_ACCESS_KEY || undefined, + }); // Verifiera bucket-åtkomst vid uppstart – produktion ska aldrig starta halvtrasig. try { await aws.healthCheck(); @@ -297,3 +98,5 @@ export const storagePlugin = fp(async (app: FastifyInstance) => { }, ); }); + +export { createStorageService }; diff --git a/apps/api/src/routes/health.ts b/apps/api/src/routes/health.ts index 29a7388..729e9eb 100644 --- a/apps/api/src/routes/health.ts +++ b/apps/api/src/routes/health.ts @@ -13,10 +13,14 @@ export async function healthRoutes(app: FastifyInstance) { app.get("/readyz", { config: { rateLimit: false } }, async (_req, reply) => { try { await app.db.execute(sql`SELECT 1`); - return { ok: true }; } catch { return reply.status(503).send({ ok: false, reason: "database" }); } + const aamosHealth = await app.aamos.healthCheck(); + if (!aamosHealth.ok) { + return reply.status(503).send({ ok: false, reason: "aamos", detail: aamosHealth.detail }); + } + return { ok: true }; }); /** Enkel endpointöversikt i stället för tung OpenAPI-generering (se beslutslogg D-011). */ diff --git a/apps/worker/package.json b/apps/worker/package.json index 101fbf2..bc09316 100644 --- a/apps/worker/package.json +++ b/apps/worker/package.json @@ -25,12 +25,12 @@ "@app/recipe-engine": "workspace:*", "@app/recommendation-engine": "workspace:*", "@app/shared-types": "workspace:*", + "@app/storage": "workspace:*", "@app/subscriptions": "workspace:*", - "@aws-sdk/client-s3": "^3.750.0", + "@aws-sdk/client-s3": "^3.1102.0", "bullmq": "^6.0.0", "dotenv": "^16.4.0", "drizzle-orm": "^0.45.0", - "@aws-sdk/client-s3": "^3.750.0", "ioredis": "^6.0.0", "pg": "^8.13.0" }, @@ -50,6 +50,7 @@ "@app/recipe-engine", "@app/recommendation-engine", "@app/shared-types", + "@app/storage", "@app/subscriptions" ] } diff --git a/apps/worker/src/context.ts b/apps/worker/src/context.ts index 6f4ec39..a44b75d 100644 --- a/apps/worker/src/context.ts +++ b/apps/worker/src/context.ts @@ -10,9 +10,9 @@ for (const candidate of [".env", "../.env", "../../.env"]) { break; } } -import { createHmac } from "node:crypto"; import { createDatabase, type Database } from "@app/database"; import { createAamosClient, type AamosClient, type BudgetStore } from "@app/ai-contracts"; +import { createStorageService, type StorageService } from "@app/storage"; import type { Redis } from "ioredis"; export interface WorkerContext { @@ -21,8 +21,10 @@ export interface WorkerContext { redis?: Redis; /** Delad dagsbudget-store så att memory-jobbet kan spegla skanningens budgetkoll. */ budgetStore?: BudgetStore; - /** Bygger läs-URL för lagrade bilder (samma signaturlogik som API:ts mock-S3). */ - readUrl: (key: string) => string; + /** Lagringstjänst för att läsa/skriva bilder (S3 eller mock). */ + storage: StorageService; + /** Bygger läs-URL för lagrade bilder (presignerad S3 i aws-läge, mock-URL lokalt). */ + readUrl: (key: string) => Promise; apiBaseUrl: string; close: () => Promise; } @@ -64,29 +66,42 @@ class RedisBudgetStore implements BudgetStore { } } +function requireEnv(name: string): string { + const value = process.env[name]; + if (!value) throw new Error(`Missing ${name}. Worker must be configured explicitly.`); + return value; +} + export function createContext(redis?: Redis): WorkerContext { - const databaseUrl = process.env.DATABASE_URL; - if (!databaseUrl) { - throw new Error("Missing DATABASE_URL. Worker must connect explicitly to the app database."); - } + const databaseUrl = requireEnv("DATABASE_URL"); const { db, pool } = createDatabase(databaseUrl); const budgetStore = redis ? new RedisBudgetStore(redis, `gemini:daily:budget:${new Date().toISOString().slice(0, 10)}`) : undefined; const aamos = createAamosClient(undefined, { budgetStore }); const apiBaseUrl = process.env.API_BASE_URL ?? "http://localhost:4000"; - const secret = process.env.ENTITLEMENT_SIGNING_SECRET ?? "dev-only-change-me-three"; + const signingSecret = requireEnv("ENTITLEMENT_SIGNING_SECRET"); + + const s3Mode = process.env.S3_MODE === "aws" ? "aws" : "mock"; + const storage = createStorageService({ + mode: s3Mode, + baseUrl: apiBaseUrl, + signingSecret, + s3Bucket: process.env.S3_BUCKET ?? "", + s3Region: process.env.S3_REGION ?? "", + s3Endpoint: process.env.S3_ENDPOINT || undefined, + s3AccessKeyId: process.env.S3_ACCESS_KEY_ID || undefined, + s3SecretAccessKey: process.env.S3_SECRET_ACCESS_KEY || undefined, + }); return { db, aamos, redis, budgetStore, + storage, apiBaseUrl, - readUrl: (key: string) => { - const sig = createHmac("sha256", secret).update(key).digest("hex").slice(0, 32); - return `${apiBaseUrl}/v1/mock-s3/${encodeURIComponent(key)}?sig=${sig}`; - }, + readUrl: (key: string) => storage.getReadUrl(key), close: async () => { await pool.end(); }, diff --git a/apps/worker/src/eval/capture-eval.ts b/apps/worker/src/eval/capture-eval.ts index 1790dc0..539ee25 100644 --- a/apps/worker/src/eval/capture-eval.ts +++ b/apps/worker/src/eval/capture-eval.ts @@ -47,7 +47,7 @@ async function main() { anonymizedImprovement: true, imageTraining: true, }, - readUrl: (key: string) => { + readUrl: async (key: string) => { if (key === "eval/fridge-1.jpg") return fridgeDataUrl; return `http://localhost/v1/mock-s3/${encodeURIComponent(key)}?sig=dummy`; }, diff --git a/apps/worker/src/lib/shadow-capture.ts b/apps/worker/src/lib/shadow-capture.ts index fc9fbc0..1dcb20c 100644 --- a/apps/worker/src/lib/shadow-capture.ts +++ b/apps/worker/src/lib/shadow-capture.ts @@ -39,7 +39,7 @@ export interface CaptureInput { inputTokens?: number | null; outputTokens?: number | null; consentFlags: CaptureConsentFlags; - readUrl: (key: string) => string; + readUrl: (key: string) => Promise; /** Override for tests; otherwise built from env. */ storage?: CaptureStorage; } @@ -138,7 +138,7 @@ export async function captureTrainingSample(input: CaptureInput): Promise ctx.readUrl(k)); + const imageUrls = await Promise.all(job.s3Keys.map((k) => ctx.readUrl(k))); const localeContext = await getLocaleContext(ctx, job.userId); const consentFlags = await loadConsentFlags(ctx, job.userId); diff --git a/apps/worker/test/memory-sync.test.ts b/apps/worker/test/memory-sync.test.ts index 934968e..87fd081 100644 --- a/apps/worker/test/memory-sync.test.ts +++ b/apps/worker/test/memory-sync.test.ts @@ -86,7 +86,8 @@ describe("UPDATE_USER_MEMORY hardening", () => { aamos, budgetStore, apiBaseUrl: "http://localhost:4000", - readUrl: (key: string) => `http://localhost:4000/mock-s3/${key}`, + storage: {} as any, + readUrl: async (key: string) => `http://localhost:4000/mock-s3/${key}`, close: async () => {}, }; } diff --git a/docs/SECRETS.md b/docs/SECRETS.md new file mode 100644 index 0000000..e44aaec --- /dev/null +++ b/docs/SECRETS.md @@ -0,0 +1,49 @@ +# Hemligheter och miljövariabler — Cibello + +**Enda sanningskälla:** AWS Systems Manager Parameter Store (`/cibello/prod/*` i `eu-north-1`). +Serverns `/opt/cibello-platform/.env` härleds ALLTID ur SSM via `sync-env-from-ssm.py`. +Inga handkopior får drifta. + +## SSM-parameter → env-var + +| SSM-parameter | Miljövariabel | Kommentar | +|---------------|---------------|-----------| +| `/cibello/prod/aamos-api-key` | `AAMOS_API_KEY` | Endast när `AAMOS_MODE=http` | +| `/cibello/prod/db-password` | byggd in i `DATABASE_URL` / `TEST_DATABASE_URL` | RDS-användare `cibello_admin` | +| `/cibello/prod/entitlement-signing-secret` | `ENTITLEMENT_SIGNING_SECRET` | Signerar prenumerationsrättigheter | +| `/cibello/prod/eoc-push-token` | `EOC_PUSH_TOKEN` | Ops-summary push till EOC | +| `/cibello/prod/gemini-api-key` | `GEMINI_API_KEY` | AAMOS_MODE=gemini | +| `/cibello/prod/google-service-account-json` | `GOOGLE_SERVICE_ACCOUNT_JSON` | Google Play service account (JSON) | +| `/cibello/prod/jwt-access-secret` | `JWT_ACCESS_SECRET` | Access-token-signering | +| `/cibello/prod/jwt-refresh-secret` | `JWT_REFRESH_SECRET` | Refresh-token-signering | +| `/cibello/prod/smtp-pass` | `SMTP_PASS` | SMTP-lösen för `hello@cibello.app` | + +**Föråldrade / dubbla parametrar** (rensas vid tillfälle): `/cibello/prod/entitlement-signing`, `/cibello/prod/jwt-access`, `/cibello/prod/jwt-refresh`. + +## Icke-hemliga värden + +Följande ligger fortfarande hårdkodade i `infrastructure/deployment/sync-env-from-ssm.py` +och kan flyttas till SSM (String-typ) om de behöver ändras ofta: + +- `API_BASE_URL`, `VITE_API_BASE_URL`, `CORS_ORIGINS` +- `S3_BUCKET`, `S3_REGION`, `S3_MODE` +- `GEMINI_MODEL`, `GEMINI_TIMEOUT_MS`, `GEMINI_DAILY_BUDGET_USD` +- `DATABASE_CA_CERT`, DB-host/name/user (kan överstyras via env) + +## Rotation + +1. Skriv nytt värde till SSM: + `aws ssm put-parameter --name /cibello/prod/ --value '' --type SecureString --overwrite --region eu-north-1` +2. Kör på prod-servern: + `python3 /opt/cibello-platform/infrastructure/deployment/sync-env-from-ssm.py` +3. Starta om API + worker. +4. Verifiera `/readyz` och kör röktest. + +## Regler + +- **Aldrig** committa `.env`, secrets eller nycklar. +- **Aldrig** skriva secrets i chatt, loggar eller shell-historik (`set +x`). +- Produktionsserverns instansroll (`cibello-app`) har endast `ssm:GetParameter`; + skrivrättigheter kräver gateway-operator med `platform-admin`. +- RDS-lösenordet roteras med både `ALTER ROLE ... WITH PASSWORD` och SSM-uppdatering; + `sync-env-from-ssm.py` bygger om `DATABASE_URL` automatiskt. diff --git a/infrastructure/deployment/first-deploy.sh b/infrastructure/deployment/first-deploy.sh index a533638..9353f5b 100755 --- a/infrastructure/deployment/first-deploy.sh +++ b/infrastructure/deployment/first-deploy.sh @@ -93,6 +93,16 @@ if [ ! -f .env ]; then chmod 600 .env fi +# I produktion härleds .env alltid ur SSM – inga handkopior får drifta. +if [ "$MODE" = "production" ]; then + if [ -f "$ROOT/infrastructure/deployment/sync-env-from-ssm.py" ]; then + say "Synkar .env från SSM..." + python3 "$ROOT/infrastructure/deployment/sync-env-from-ssm.py" + else + fail "sync-env-from-ssm.py saknas – kan inte synka .env från SSM." + fi +fi + get_env() { { grep -E "^$1=" .env || true; } | head -1 | cut -d= -f2-; } AAMOS_MODE_VAL=$(get_env AAMOS_MODE) # Sätt NODE_EXTRA_CA_CERTS om definierad i .env (behövs för RDS SSL från Node/pg). diff --git a/infrastructure/deployment/sync-env-from-ssm.py b/infrastructure/deployment/sync-env-from-ssm.py new file mode 100644 index 0000000..4dd3655 --- /dev/null +++ b/infrastructure/deployment/sync-env-from-ssm.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +""" +Hämta Cibello-prod-hemligheter från AWS SSM och skriv /opt/cibello-platform/.env. + +SSM är enda sanningskällan. Kör detta vid varje deploy INNAN applikationen startas om. +Inga värden loggas. set +x aktiveras automatiskt. +""" +import json +import os +import subprocess +import sys +from pathlib import Path + +REGION = os.environ.get("AWS_REGION", "eu-north-1") +SSM_PREFIX = "/cibello/prod" +ENV_PATH = Path(os.environ.get("ENV_PATH", "/opt/cibello-platform/.env")) + +# Icke-hemliga värden som ändå behöver styras centralt. +# Dessa kan med fördel också flyttas till SSM (String/SecureString) senare. +STATIC_CONFIG = { + "NODE_ENV": "production", + "API_PORT": "4000", + "ADMIN_PORT": "5173", + "API_BASE_URL": "https://api.cibello.app", + "VITE_API_BASE_URL": "https://api.cibello.app", + "CORS_ORIGINS": "https://cibello.app,https://admin.cibello.app,https://api.cibello.app", + "REDIS_URL": "redis://localhost:6379", + "LOG_LEVEL": "info", + "AAMOS_MODE": "gemini", + "AAMOS_API_URL": "https://amos.aamos.systems", + "AAMOS_TIMEOUT_MS": "60000", + "GEMINI_MODEL": "gemini-2.5-flash", + "GEMINI_TIMEOUT_MS": "60000", + "GEMINI_DAILY_BUDGET_USD": "25", + "ENTITLEMENT_TTL_HOURS": "24", + "ENTITLEMENT_GRACE_DAYS": "7", + "JWT_ACCESS_TTL_SECONDS": "900", + "JWT_REFRESH_TTL_SECONDS": "2592000", + "S3_MODE": "aws", + "S3_BUCKET": "cibello-production", + "S3_REGION": "eu-north-1", + "SMTP_PORT": "587", + "SMTP_SECURE": "false", + "EMAIL_MODE": "log", + "EXPO_PUSH_ENABLED": "false", + "DATABASE_CA_CERT": "/opt/cibello-platform/certs/rds-global-bundle.pem", +} + +# SSM-parameter → env-var. Värden från SSM skriver över STATIC_CONFIG om de krockar. +SSM_MAP = { + "aamos-api-key": "AAMOS_API_KEY", + "db-password": "DB_PASSWORD", # används för att bygga DATABASE_URL + "entitlement-signing-secret": "ENTITLEMENT_SIGNING_SECRET", + "eoc-push-token": "EOC_PUSH_TOKEN", + "gemini-api-key": "GEMINI_API_KEY", + "jwt-access-secret": "JWT_ACCESS_SECRET", + "jwt-refresh-secret": "JWT_REFRESH_SECRET", + "smtp-pass": "SMTP_PASS", + # Följande är valfria just nu men bör ligga i SSM + "google-service-account-json": "GOOGLE_SERVICE_ACCOUNT_JSON", +} + + +def get_parameters(names: list[str]) -> dict[str, str]: + if not names: + return {} + cmd = [ + "aws", "ssm", "get-parameters", + "--names", *names, + "--with-decryption", + "--region", REGION, + ] + result = subprocess.run(cmd, capture_output=True, text=True, check=True) + data = json.loads(result.stdout) + out = {} + for p in data.get("Parameters", []): + out[p["Name"]] = p["Value"] + for inv in data.get("InvalidParameters", []): + print(f"[sync-env] WARN: SSM-parameter saknas: {inv}", file=sys.stderr) + return out + + +def build_env(ssm: dict[str, str]) -> dict[str, str]: + env = dict(STATIC_CONFIG) + + for short, var in SSM_MAP.items(): + val = ssm.get(f"{SSM_PREFIX}/{short}") + if val is not None: + env[var] = val + + db_password = env.get("DB_PASSWORD", "") + db_host = os.environ.get("DB_HOST", "cibello-prod-db.cvi0qcksmsfj.eu-north-1.rds.amazonaws.com") + db_name = os.environ.get("DB_NAME", "cibello") + test_db_name = os.environ.get("TEST_DB_NAME", "cibello_test") + db_user = os.environ.get("DB_USER", "cibello_admin") + db_sslmode = os.environ.get("DB_SSLMODE", "require") + + if db_password: + env["DATABASE_URL"] = f"postgres://{db_user}:{db_password}@{db_host}:5432/{db_name}?sslmode={db_sslmode}" + env["TEST_DATABASE_URL"] = f"postgres://{db_user}:{db_password}@{db_host}:5432/{test_db_name}?sslmode={db_sslmode}" + + # Rensa temporär nyckel + env.pop("DB_PASSWORD", None) + + return env + + +def write_env(env: dict[str, str], dry_run: bool = False) -> None: + lines = [f"{k}={v}" for k, v in env.items()] + if dry_run: + print(f"[sync-env] DRY-RUN: skulle skriva {ENV_PATH} ({len(env)} rader)") + return + ENV_PATH.parent.mkdir(parents=True, exist_ok=True) + ENV_PATH.write_text("\n".join(lines) + "\n", encoding="utf-8") + os.chmod(ENV_PATH, 0o600) + print(f"[sync-env] skrev {ENV_PATH} ({len(env)} rader)") + + +def main() -> int: + dry_run = "--dry-run" in sys.argv + names = [f"{SSM_PREFIX}/{short}" for short in SSM_MAP] + ssm = get_parameters(names) + env = build_env(ssm) + write_env(env, dry_run=dry_run) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/packages/storage/package.json b/packages/storage/package.json new file mode 100644 index 0000000..950dccb --- /dev/null +++ b/packages/storage/package.json @@ -0,0 +1,18 @@ +{ + "name": "@app/storage", + "version": "0.1.0", + "private": true, + "type": "module", + "description": "Delad lagringsabstraktion för S3/mock-S3 (spec §53)", + "exports": { + ".": "./src/index.ts" + }, + "scripts": { + "typecheck": "tsc --noEmit", + "test": "vitest run --passWithNoTests" + }, + "dependencies": { + "@aws-sdk/client-s3": "^3.1102.0", + "@aws-sdk/s3-request-presigner": "^3.1102.0" + } +} diff --git a/packages/storage/src/index.ts b/packages/storage/src/index.ts new file mode 100644 index 0000000..fbb1df3 --- /dev/null +++ b/packages/storage/src/index.ts @@ -0,0 +1,219 @@ +import { createHmac, randomUUID } from "node:crypto"; +import { mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { + DeleteObjectCommand, + GetObjectCommand, + HeadBucketCommand, + PutObjectCommand, + S3Client, +} from "@aws-sdk/client-s3"; +import { getSignedUrl } from "@aws-sdk/s3-request-presigner"; + +export interface PresignedUpload { + key: string; + uploadUrl: string; + method: "PUT"; + headers: Record; + expiresAt: string; +} + +export interface StorageService { + presignUpload(prefix: string, contentType: string): Promise; + /** URL som workern/AAMOS kan läsa bilden från. */ + getReadUrl(key: string): Promise; + putObject(key: string, data: Buffer, contentType: string): Promise; + getObject(key: string): Promise; + /** GDPR / rättning: radera ett objekt. Får inte kasta om nyckeln saknas. */ + deleteObject(key: string): Promise; + /** + * Försök extrahera lagringsnyckeln ur en läs-URL. Returnerar null om URL:en + * inte känns igen (t.ex. extern bild). Används vid GDPR-radering så att vi + * kan radera objekt även när endast URL:en är sparad. + */ + extractKeyFromUrl(url: string): string | null; +} + +const MOCK_ROOT = path.resolve(process.cwd(), ".data/s3"); + +export class MockStorage implements StorageService { + constructor( + private readonly baseUrl: string, + private readonly signingSecret: string, + ) {} + + private sign(key: string): string { + return createHmac("sha256", this.signingSecret).update(key).digest("hex").slice(0, 32); + } + + async presignUpload(prefix: string, contentType: string): Promise { + const ext = contentType.split("/")[1] ?? "bin"; + const key = `${prefix}/${randomUUID()}.${ext}`; + const sig = this.sign(key); + return { + key, + uploadUrl: `${this.baseUrl}/v1/mock-s3/${encodeURIComponent(key)}?sig=${sig}`, + method: "PUT", + headers: { "content-type": contentType }, + expiresAt: new Date(Date.now() + 15 * 60_000).toISOString(), + }; + } + + async getReadUrl(key: string): Promise { + return `${this.baseUrl}/v1/mock-s3/${encodeURIComponent(key)}?sig=${this.sign(key)}`; + } + + async putObject(key: string, data: Buffer, _contentType?: string): Promise { + const filePath = path.join(MOCK_ROOT, key); + await mkdir(path.dirname(filePath), { recursive: true }); + await writeFile(filePath, data); + } + + async getObject(key: string): Promise { + try { + return await readFile(path.join(MOCK_ROOT, key)); + } catch { + return null; + } + } + + async deleteObject(key: string): Promise { + await rm(path.join(MOCK_ROOT, key), { force: true }).catch(() => {}); + } + + extractKeyFromUrl(url: string): string | null { + try { + const parsed = new URL(url); + const prefix = "/v1/mock-s3/"; + if (!parsed.pathname.startsWith(prefix)) return null; + return decodeURIComponent(parsed.pathname.slice(prefix.length)); + } catch { + return null; + } + } + + verifySignature(key: string, sig: string): boolean { + return this.sign(key) === sig; + } +} + +export interface AwsStorageConfig { + bucket: string; + region: string; + endpoint?: string; + accessKeyId?: string; + secretAccessKey?: string; +} + +/** Riktig S3 (AWS eller S3-kompatibel via S3_ENDPOINT). */ +export class AwsStorage implements StorageService { + private readonly client: S3Client; + private readonly bucket: string; + + constructor(cfg: AwsStorageConfig) { + this.bucket = cfg.bucket; + this.client = new S3Client({ + region: cfg.region, + ...(cfg.endpoint ? { endpoint: cfg.endpoint, forcePathStyle: true } : {}), + ...(cfg.accessKeyId && cfg.secretAccessKey + ? { credentials: { accessKeyId: cfg.accessKeyId, secretAccessKey: cfg.secretAccessKey } } + : {}), // annars IAM-roll/instansprofil (rekommenderat i AWS) + }); + } + + /** Snabb verifiering vid uppstart – hellre högljutt fel än trasiga uppladdningar. */ + async healthCheck(): Promise { + await this.client.send(new HeadBucketCommand({ Bucket: this.bucket })); + } + + async presignUpload(prefix: string, contentType: string): Promise { + const ext = contentType.split("/")[1] ?? "bin"; + const key = `${prefix}/${randomUUID()}.${ext}`; + const uploadUrl = await getSignedUrl( + this.client, + new PutObjectCommand({ Bucket: this.bucket, Key: key, ContentType: contentType }), + { expiresIn: 15 * 60 }, + ); + return { + key, + uploadUrl, + method: "PUT", + headers: { "content-type": contentType }, + expiresAt: new Date(Date.now() + 15 * 60_000).toISOString(), + }; + } + + async getReadUrl(key: string): Promise { + return getSignedUrl(this.client, new GetObjectCommand({ Bucket: this.bucket, Key: key }), { + expiresIn: 60 * 60, + }); + } + + async putObject(key: string, data: Buffer, contentType: string): Promise { + await this.client.send( + new PutObjectCommand({ Bucket: this.bucket, Key: key, Body: data, ContentType: contentType }), + ); + } + + async getObject(key: string): Promise { + try { + const res = await this.client.send(new GetObjectCommand({ Bucket: this.bucket, Key: key })); + const bytes = await res.Body?.transformToByteArray(); + return bytes ? Buffer.from(bytes) : null; + } catch { + return null; + } + } + + async deleteObject(key: string): Promise { + try { + await this.client.send(new DeleteObjectCommand({ Bucket: this.bucket, Key: key })); + } catch (err) { + const code = (err as { Code?: string }).Code; + if (code !== "NoSuchKey" && code !== "NotFound") { + throw err; + } + } + } + + extractKeyFromUrl(url: string): string | null { + try { + const parsed = new URL(url); + if (parsed.hostname.startsWith(`${this.bucket}.`)) { + return decodeURIComponent(parsed.pathname.replace(/^\//, "")); + } + const pathPrefix = `/${this.bucket}/`; + if (parsed.pathname.startsWith(pathPrefix)) { + return decodeURIComponent(parsed.pathname.slice(pathPrefix.length)); + } + return null; + } catch { + return null; + } + } +} + +/** + * Skapa rätt lagringstjänst utifrån S3_MODE. Används av API och worker. + */ +export function createStorageService(cfg: { + mode: "aws" | "mock"; + baseUrl: string; + signingSecret: string; + s3Bucket: string; + s3Region: string; + s3Endpoint?: string; + s3AccessKeyId?: string; + s3SecretAccessKey?: string; +}): StorageService { + if (cfg.mode === "aws") { + return new AwsStorage({ + bucket: cfg.s3Bucket, + region: cfg.s3Region, + endpoint: cfg.s3Endpoint, + accessKeyId: cfg.s3AccessKeyId, + secretAccessKey: cfg.s3SecretAccessKey, + }); + } + return new MockStorage(cfg.baseUrl, cfg.signingSecret); +} diff --git a/packages/storage/tsconfig.json b/packages/storage/tsconfig.json new file mode 100644 index 0000000..9d3fdee --- /dev/null +++ b/packages/storage/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "../../tsconfig.base.json", + "include": ["src", "test"], + "compilerOptions": { + "types": ["node"] + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b8ad0ba..c7a0b9d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -93,6 +93,9 @@ importers: '@app/shared-types': specifier: workspace:* version: link:../../packages/shared-types + '@app/storage': + specifier: workspace:* + version: link:../../packages/storage '@app/subscriptions': specifier: workspace:* version: link:../../packages/subscriptions @@ -293,11 +296,14 @@ importers: '@app/shared-types': specifier: workspace:* version: link:../../packages/shared-types + '@app/storage': + specifier: workspace:* + version: link:../../packages/storage '@app/subscriptions': specifier: workspace:* version: link:../../packages/subscriptions '@aws-sdk/client-s3': - specifier: ^3.750.0 + specifier: ^3.1102.0 version: 3.1102.0 bullmq: specifier: ^6.0.0 @@ -466,6 +472,15 @@ importers: packages/shared-types: {} + packages/storage: + dependencies: + '@aws-sdk/client-s3': + specifier: ^3.1102.0 + version: 3.1102.0 + '@aws-sdk/s3-request-presigner': + specifier: ^3.1102.0 + version: 3.1102.0 + packages/subscriptions: dependencies: '@app/shared-types':