Fas 1a punkt 3+4: worker-härdning (DLQ, larm, multi-instance), AAMOS training export
This commit is contained in:
Sven (AAMOS AI)
@@ -10,6 +10,7 @@ import { SubscriptionsPage } from "./pages/Subscriptions.js";
|
||||
import { AuditLogsPage } from "./pages/AuditLogs.js";
|
||||
import { AnalyticsPage } from "./pages/Analytics.js";
|
||||
import { ReleaseGatesPage } from "./pages/ReleaseGates.js";
|
||||
import { WorkersPage } from "./pages/Workers.js";
|
||||
|
||||
export function App() {
|
||||
const [authed, setAuthed] = useState(hasToken());
|
||||
@@ -32,6 +33,7 @@ export function App() {
|
||||
<NavLink to="/audit">Audit logs</NavLink>
|
||||
<NavLink to="/analytics">Analytics</NavLink>
|
||||
<NavLink to="/release-gates">Release gates</NavLink>
|
||||
<NavLink to="/workers">Workers</NavLink>
|
||||
<button
|
||||
onClick={() => {
|
||||
setToken(null);
|
||||
@@ -51,6 +53,7 @@ export function App() {
|
||||
<Route path="/audit" element={<AuditLogsPage />} />
|
||||
<Route path="/analytics" element={<AnalyticsPage />} />
|
||||
<Route path="/release-gates" element={<ReleaseGatesPage />} />
|
||||
<Route path="/workers" element={<WorkersPage />} />
|
||||
<Route path="*" element={<Navigate to="/" replace />} />
|
||||
</Routes>
|
||||
</main>
|
||||
|
||||
@@ -0,0 +1,204 @@
|
||||
import { useEffect, useState } from "react";
|
||||
import { api } from "../api.js";
|
||||
|
||||
interface JobCounts {
|
||||
wait: number;
|
||||
active: number;
|
||||
delayed: number;
|
||||
completed: number;
|
||||
failed: number;
|
||||
paused?: number;
|
||||
}
|
||||
|
||||
interface WorkerInfo {
|
||||
id: string;
|
||||
addr: string;
|
||||
concurrency: number;
|
||||
lastSeenAt: number;
|
||||
}
|
||||
|
||||
interface SchedulerInfo {
|
||||
id: string;
|
||||
pattern?: string;
|
||||
every?: number;
|
||||
}
|
||||
|
||||
interface HealthResponse {
|
||||
ok: boolean;
|
||||
queues: {
|
||||
jobs: JobCounts;
|
||||
deadLetter: JobCounts;
|
||||
};
|
||||
workers: WorkerInfo[];
|
||||
schedulers: SchedulerInfo[];
|
||||
}
|
||||
|
||||
interface DlqJob {
|
||||
id: string;
|
||||
name: string;
|
||||
data: unknown;
|
||||
failedReason?: string;
|
||||
attemptsMade: number;
|
||||
timestamp: number;
|
||||
}
|
||||
|
||||
export function WorkersPage() {
|
||||
const [health, setHealth] = useState<HealthResponse | null>(null);
|
||||
const [dlq, setDlq] = useState<DlqJob[]>([]);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [loading, setLoading] = useState(false);
|
||||
const [actionId, setActionId] = useState<string | null>(null);
|
||||
|
||||
const load = async () => {
|
||||
setLoading(true);
|
||||
try {
|
||||
const [h, d] = await Promise.all([
|
||||
api<HealthResponse>("/admin/v1/workers/health"),
|
||||
api<{ jobs: DlqJob[] }>("/admin/v1/workers/dead-letter?limit=50"),
|
||||
]);
|
||||
setHealth(h);
|
||||
setDlq(d.jobs);
|
||||
setError(null);
|
||||
} catch (e) {
|
||||
setError(String((e as Error).message));
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
load();
|
||||
const t = setInterval(load, 10_000);
|
||||
return () => clearInterval(t);
|
||||
}, []);
|
||||
|
||||
const retry = async (id: string) => {
|
||||
setActionId(id);
|
||||
try {
|
||||
await api(`/admin/v1/workers/dead-letter/${id}/retry`, { method: "POST" });
|
||||
await load();
|
||||
} catch (e) {
|
||||
setError(String((e as Error).message));
|
||||
} finally {
|
||||
setActionId(null);
|
||||
}
|
||||
};
|
||||
|
||||
const discard = async (id: string) => {
|
||||
if (!confirm("Ta bort jobbet permanent?")) return;
|
||||
setActionId(id);
|
||||
try {
|
||||
await api(`/admin/v1/workers/dead-letter/${id}/discard`, { method: "POST" });
|
||||
await load();
|
||||
} catch (e) {
|
||||
setError(String((e as Error).message));
|
||||
} finally {
|
||||
setActionId(null);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="page">
|
||||
<h2>Workers & köer</h2>
|
||||
{error && <div className="alert err">{error}</div>}
|
||||
{loading && !health && <div>Laddar …</div>}
|
||||
|
||||
{health && (
|
||||
<>
|
||||
<section className="card">
|
||||
<h3>Jobbkö</h3>
|
||||
<div className="metrics">
|
||||
<Metric label="Väntar" value={health.queues.jobs.wait} />
|
||||
<Metric label="Aktiva" value={health.queues.jobs.active} />
|
||||
<Metric label="Fördröjda" value={health.queues.jobs.delayed} />
|
||||
<Metric label="Misslyckade" value={health.queues.jobs.failed} />
|
||||
<Metric label="Klara" value={health.queues.jobs.completed} />
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section className="card">
|
||||
<h3>Dead letter ({health.queues.deadLetter.wait + health.queues.deadLetter.failed})</h3>
|
||||
{dlq.length === 0 ? (
|
||||
<p>Inga jobb i dead letter.</p>
|
||||
) : (
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Jobb</th>
|
||||
<th>Fel</th>
|
||||
<th>Försök</th>
|
||||
<th>Tid</th>
|
||||
<th>Åtgärder</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{dlq.map((job) => (
|
||||
<tr key={job.id}>
|
||||
<td>{job.name}</td>
|
||||
<td className="ellipsis" title={job.failedReason}>
|
||||
{job.failedReason ?? "–"}
|
||||
</td>
|
||||
<td>{job.attemptsMade}</td>
|
||||
<td>{new Date(job.timestamp).toLocaleString("sv-SE")}</td>
|
||||
<td>
|
||||
<button
|
||||
onClick={() => retry(job.id)}
|
||||
disabled={actionId === job.id}
|
||||
className="btn small"
|
||||
>
|
||||
Kör igen
|
||||
</button>{" "}
|
||||
<button
|
||||
onClick={() => discard(job.id)}
|
||||
disabled={actionId === job.id}
|
||||
className="btn small danger"
|
||||
>
|
||||
Kasta
|
||||
</button>
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
)}
|
||||
</section>
|
||||
|
||||
<section className="card">
|
||||
<h3>Aktiva worker-instanser</h3>
|
||||
{health.workers.length === 0 ? (
|
||||
<p className="err">Inga aktiva workers!</p>
|
||||
) : (
|
||||
<ul>
|
||||
{health.workers.map((w) => (
|
||||
<li key={w.id}>
|
||||
{w.id} @ {w.addr} (concurrency {w.concurrency})
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
)}
|
||||
</section>
|
||||
|
||||
<section className="card">
|
||||
<h3>Scheduler</h3>
|
||||
<ul>
|
||||
{health.schedulers.map((s) => (
|
||||
<li key={s.id}>
|
||||
{s.id}: {s.pattern ?? `every ${s.every}ms`}
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
</section>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function Metric({ label, value }: { label: string; value: number }) {
|
||||
return (
|
||||
<div className="metric">
|
||||
<div className="value">{value}</div>
|
||||
<div className="label">{label}</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -124,3 +124,51 @@ input, select {
|
||||
.stat .value { font-size: 1.6rem; font-weight: 700; }
|
||||
.stat .label { color: var(--muted); font-size: 0.8rem; }
|
||||
.row { display: flex; gap: 0.5rem; align-items: center; flex-wrap: wrap; }
|
||||
|
||||
.card {
|
||||
background: var(--panel);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 12px;
|
||||
padding: 1.25rem;
|
||||
margin-bottom: 1.25rem;
|
||||
}
|
||||
.card h3 { margin-top: 0; }
|
||||
|
||||
.metrics { display: flex; gap: 1rem; flex-wrap: wrap; }
|
||||
.metric {
|
||||
background: var(--bg);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 10px;
|
||||
padding: 0.75rem 1rem;
|
||||
min-width: 90px;
|
||||
}
|
||||
.metric .value { font-size: 1.4rem; font-weight: 700; }
|
||||
.metric .label { color: var(--muted); font-size: 0.8rem; }
|
||||
|
||||
.btn {
|
||||
background: var(--brand);
|
||||
color: #fff;
|
||||
border: none;
|
||||
padding: 0.45rem 0.9rem;
|
||||
border-radius: 8px;
|
||||
cursor: pointer;
|
||||
font-size: 0.9rem;
|
||||
}
|
||||
.btn:hover { background: var(--brand-dark); }
|
||||
.btn:disabled { opacity: 0.6; cursor: not-allowed; }
|
||||
.btn.small { padding: 0.25rem 0.55rem; font-size: 0.8rem; }
|
||||
.btn.danger { background: var(--danger); }
|
||||
|
||||
.alert {
|
||||
padding: 0.75rem 1rem;
|
||||
border-radius: 8px;
|
||||
margin-bottom: 1rem;
|
||||
}
|
||||
.alert.err { background: #fee2e2; color: var(--danger); }
|
||||
|
||||
.ellipsis {
|
||||
max-width: 280px;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
import type { FastifyInstance } from "fastify";
|
||||
import { DEAD_LETTER_QUEUE_NAME, JOB_QUEUE_NAME } from "@app/shared-types";
|
||||
import { Queue, Job } from "bullmq";
|
||||
|
||||
interface JobSummary {
|
||||
id: string | undefined;
|
||||
name: string;
|
||||
data: unknown;
|
||||
failedReason: string | undefined;
|
||||
attemptsMade: number;
|
||||
timestamp: number;
|
||||
}
|
||||
|
||||
export async function adminWorkersRoutes(app: FastifyInstance) {
|
||||
const admin = { preHandler: [app.requireAdmin] };
|
||||
const dlq = new Queue(DEAD_LETTER_QUEUE_NAME, { connection: app.redis });
|
||||
|
||||
app.addHook("onClose", async () => {
|
||||
await dlq.close();
|
||||
});
|
||||
|
||||
app.get("/admin/v1/workers/health", admin, async (_req, reply) => {
|
||||
const queue = new Queue(JOB_QUEUE_NAME, { connection: app.redis });
|
||||
const [jobCounts, dlqCounts, workers, schedulers] = await Promise.all([
|
||||
queue.getJobCounts("wait", "active", "delayed", "completed", "failed"),
|
||||
dlq.getJobCounts("wait", "active", "completed", "failed"),
|
||||
queue.getWorkers(),
|
||||
queue.getJobSchedulers(),
|
||||
]);
|
||||
await queue.close();
|
||||
|
||||
return reply.send({
|
||||
ok: true,
|
||||
queues: {
|
||||
jobs: jobCounts,
|
||||
deadLetter: dlqCounts,
|
||||
},
|
||||
workers: workers.map((w) => ({
|
||||
id: w.id,
|
||||
addr: w.addr,
|
||||
concurrency: w.concurrency,
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
lastSeenAt: (w as any).ts,
|
||||
})),
|
||||
schedulers: schedulers.map((s) => ({ id: s.id, pattern: s.pattern, every: s.every })),
|
||||
});
|
||||
});
|
||||
|
||||
app.get("/admin/v1/workers/dead-letter", admin, async (req, reply) => {
|
||||
const limit = Math.min(Number((req.query as { limit?: string }).limit ?? 50), 200);
|
||||
const jobs = await dlq.getJobs(["wait", "failed"], 0, limit - 1, true);
|
||||
const summary: JobSummary[] = jobs.map((job) => ({
|
||||
id: job.id,
|
||||
name: job.name,
|
||||
data: job.data,
|
||||
failedReason: job.failedReason,
|
||||
attemptsMade: job.attemptsMade,
|
||||
timestamp: job.timestamp,
|
||||
}));
|
||||
return reply.send({ jobs: summary });
|
||||
});
|
||||
|
||||
app.post("/admin/v1/workers/dead-letter/:id/retry", admin, async (req, reply) => {
|
||||
const { id } = req.params as { id: string };
|
||||
const job = await Job.fromId(dlq, id);
|
||||
if (!job) {
|
||||
return reply.status(404).send({ error: { code: "NOT_FOUND", message: "Jobbet finns inte i dead letter queue." } });
|
||||
}
|
||||
|
||||
const mainQueue = new Queue(JOB_QUEUE_NAME, { connection: app.redis });
|
||||
await mainQueue.add(job.name, job.data, {
|
||||
attempts: 3,
|
||||
backoff: { type: "exponential", delay: 2_000 },
|
||||
});
|
||||
await job.remove();
|
||||
await mainQueue.close();
|
||||
|
||||
return reply.send({ ok: true, message: "Jobbet har lagts tillbaka i kön." });
|
||||
});
|
||||
|
||||
app.post("/admin/v1/workers/dead-letter/:id/discard", admin, async (req, reply) => {
|
||||
const { id } = req.params as { id: string };
|
||||
const job = await Job.fromId(dlq, id);
|
||||
if (!job) {
|
||||
return reply.status(404).send({ error: { code: "NOT_FOUND", message: "Jobbet finns inte i dead letter queue." } });
|
||||
}
|
||||
await job.remove();
|
||||
return reply.send({ ok: true, message: "Jobbet har tagits bort från dead letter queue." });
|
||||
});
|
||||
}
|
||||
@@ -23,6 +23,7 @@ import { communityRoutes } from "./routes/community.js";
|
||||
import { adminRoutes } from "./routes/admin.js";
|
||||
import { adminAnalyticsRoutes } from "./routes/admin-analytics.js";
|
||||
import { adminReleaseGateRoutes } from "./routes/admin-release-gates.js";
|
||||
import { adminWorkersRoutes } from "./routes/admin-workers.js";
|
||||
import { analyticsRoutes } from "./routes/analytics.js";
|
||||
|
||||
declare module "fastify" {
|
||||
@@ -85,6 +86,7 @@ export async function buildServer(config: AppConfig) {
|
||||
await app.register(adminRoutes);
|
||||
await app.register(adminAnalyticsRoutes);
|
||||
await app.register(adminReleaseGateRoutes);
|
||||
await app.register(adminWorkersRoutes);
|
||||
await app.register(analyticsRoutes);
|
||||
|
||||
return app;
|
||||
|
||||
+114
-20
@@ -1,6 +1,7 @@
|
||||
import "dotenv/config";
|
||||
import { Queue, Worker, type Job } from "bullmq";
|
||||
import IORedis from "ioredis";
|
||||
import { createServer } from "node:http";
|
||||
import { createContext } from "./context.js";
|
||||
import { processScanJob } from "./processors/scans.js";
|
||||
import { processModerateRecipe } from "./processors/moderation.js";
|
||||
@@ -17,19 +18,29 @@ import {
|
||||
processTrainingExport,
|
||||
} from "./processors/maintenance.js";
|
||||
|
||||
import {
|
||||
DEAD_LETTER_QUEUE_NAME,
|
||||
JOB_QUEUE_NAME as QUEUE_NAME,
|
||||
WORKER_DEFAULT_ATTEMPTS,
|
||||
WORKER_DEFAULT_BACKOFF_MS,
|
||||
} from "@app/shared-types";
|
||||
|
||||
/**
|
||||
* workern. Konsumerar kön "<slug>-jobs" (spec §50):
|
||||
* App → signed S3 upload → API job → worker → AAMOS → result → app
|
||||
* workern. Konsumerar kön "<slug>-jobs" (spec §50).
|
||||
*
|
||||
* Jobbnamn = jobbtyp (spec §54). Deterministiska jobb rör aldrig AAMOS;
|
||||
* AI-jobb går alltid via de typade kontrakten i @app/ai-contracts.
|
||||
* Flera instanser kan köra samtidigt mot samma Redis- och DB-anslutning
|
||||
* utan kodändring (BullMQ hanterar fördelning). Varje instans exponerar
|
||||
* en liten HTTP-healthcheck så att lastbalansering kan se om den lever.
|
||||
*/
|
||||
|
||||
import { JOB_QUEUE_NAME as QUEUE_NAME } from "@app/shared-types";
|
||||
const connection = new IORedis(process.env.REDIS_URL ?? "redis://localhost:6379", {
|
||||
maxRetriesPerRequest: null,
|
||||
});
|
||||
const ctx = createContext();
|
||||
const instanceId = process.env.WORKER_INSTANCE_ID ?? `worker-${process.pid}`;
|
||||
|
||||
const deadLetterQueue = new Queue(DEAD_LETTER_QUEUE_NAME, { connection });
|
||||
const queue = new Queue(QUEUE_NAME, { connection });
|
||||
|
||||
const worker = new Worker(
|
||||
QUEUE_NAME,
|
||||
@@ -116,13 +127,48 @@ const worker = new Worker(
|
||||
{
|
||||
connection,
|
||||
concurrency: Number(process.env.WORKER_CONCURRENCY ?? 5),
|
||||
maxStalledCount: 2,
|
||||
stalledInterval: 30_000,
|
||||
limiter: {
|
||||
max: Number(process.env.WORKER_RATE_LIMIT_MAX ?? 60),
|
||||
duration: Number(process.env.WORKER_RATE_LIMIT_DURATION_MS ?? 60_000),
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
worker.on("completed", (job) => log(`✓ ${job.name} (${job.id})`));
|
||||
worker.on("failed", (job, err) => {
|
||||
console.error(`✗ ${job?.name} (${job?.id}): ${err.message}`);
|
||||
void alertWebhook(`Worker-jobb misslyckades: ${job?.name} (${job?.id}): ${err.message}`);
|
||||
worker.on("failed", async (job, err) => {
|
||||
const jobType = (job?.data?.jobType as string) ?? job?.name ?? "unknown";
|
||||
const attempts = job?.opts?.attempts ?? WORKER_DEFAULT_ATTEMPTS;
|
||||
const attemptsMade = job?.attemptsMade ?? 0;
|
||||
console.error(`✗ ${jobType} (${job?.id}): ${err.message} (attempt ${attemptsMade}/${attempts})`);
|
||||
|
||||
if (job && attemptsMade >= attempts) {
|
||||
try {
|
||||
await deadLetterQueue.add(
|
||||
job.name,
|
||||
{
|
||||
...job.data,
|
||||
_deadLetteredAt: new Date().toISOString(),
|
||||
_instanceId: instanceId,
|
||||
_failureReason: err.message,
|
||||
_failedAt: new Date().toISOString(),
|
||||
},
|
||||
{
|
||||
jobId: `${instanceId}:${job.id}`,
|
||||
removeOnComplete: { age: 30 * 24 * 60 * 60 }, // 30 dagar
|
||||
removeOnFail: { age: 90 * 24 * 60 * 60 }, // 90 dagar
|
||||
},
|
||||
);
|
||||
log(`→ dead letter: ${jobType} (${job.id})`);
|
||||
} catch (dlqErr) {
|
||||
console.error("Kunde inte skriva till dead letter queue:", dlqErr);
|
||||
}
|
||||
}
|
||||
|
||||
await alertWebhook(
|
||||
`Worker-jobb misslyckades: ${jobType} (${job?.id}): ${err.message} (attempt ${attemptsMade}/${attempts})`,
|
||||
);
|
||||
});
|
||||
|
||||
/** Larm till valfri webhook (Slack/Discord/Teams …) – fire-and-forget. */
|
||||
@@ -142,50 +188,98 @@ async function alertWebhook(text: string): Promise<void> {
|
||||
}
|
||||
|
||||
// --- Återkommande jobb via BullMQ Job Schedulers (spec §54) ---
|
||||
const queue = new Queue(QUEUE_NAME, { connection });
|
||||
async function registerRepeatableJobs() {
|
||||
const baseOpts = {
|
||||
attempts: WORKER_DEFAULT_ATTEMPTS,
|
||||
backoff: { type: "exponential" as const, delay: WORKER_DEFAULT_BACKOFF_MS },
|
||||
removeOnComplete: { count: 100 },
|
||||
removeOnFail: { count: 100 },
|
||||
};
|
||||
|
||||
await queue.upsertJobScheduler(
|
||||
"scheduler-outbox",
|
||||
{ every: 30_000 },
|
||||
{ name: "PUBLISH_OUTBOX", data: { jobType: "PUBLISH_OUTBOX" } },
|
||||
{ name: "PUBLISH_OUTBOX", data: { jobType: "PUBLISH_OUTBOX" }, opts: baseOpts },
|
||||
);
|
||||
await queue.upsertJobScheduler(
|
||||
"scheduler-expiry",
|
||||
{ pattern: "0 7 * * *", tz: "Europe/Stockholm" },
|
||||
{ name: "SEND_EXPIRY_NOTIFICATION", data: { jobType: "SEND_EXPIRY_NOTIFICATION" } },
|
||||
{ name: "SEND_EXPIRY_NOTIFICATION", data: { jobType: "SEND_EXPIRY_NOTIFICATION" }, opts: baseOpts },
|
||||
);
|
||||
await queue.upsertJobScheduler(
|
||||
"scheduler-memory",
|
||||
{ pattern: "30 3 * * *", tz: "Europe/Stockholm" },
|
||||
{ name: "UPDATE_USER_MEMORY", data: { jobType: "UPDATE_USER_MEMORY" } },
|
||||
{ name: "UPDATE_USER_MEMORY", data: { jobType: "UPDATE_USER_MEMORY" }, opts: baseOpts },
|
||||
);
|
||||
await queue.upsertJobScheduler(
|
||||
"scheduler-subs",
|
||||
{ pattern: "15 4 * * *", tz: "Europe/Stockholm" },
|
||||
{ name: "VERIFY_SUBSCRIPTION", data: { jobType: "VERIFY_SUBSCRIPTION" } },
|
||||
{ name: "VERIFY_SUBSCRIPTION", data: { jobType: "VERIFY_SUBSCRIPTION" }, opts: baseOpts },
|
||||
);
|
||||
await queue.upsertJobScheduler(
|
||||
"scheduler-retention",
|
||||
{ pattern: "45 4 * * *", tz: "Europe/Stockholm" },
|
||||
{ name: "RUN_RETENTION", data: { jobType: "RUN_RETENTION" } },
|
||||
{ name: "RUN_RETENTION", data: { jobType: "RUN_RETENTION" }, opts: baseOpts },
|
||||
);
|
||||
await queue.upsertJobScheduler(
|
||||
"scheduler-training",
|
||||
{ pattern: "45 2 * * 0", tz: "Europe/Stockholm" },
|
||||
{ name: "BUILD_TRAINING_SAMPLE", data: { jobType: "BUILD_TRAINING_SAMPLE" } },
|
||||
{ name: "BUILD_TRAINING_SAMPLE", data: { jobType: "BUILD_TRAINING_SAMPLE" }, opts: baseOpts },
|
||||
);
|
||||
}
|
||||
|
||||
registerRepeatableJobs()
|
||||
.then(() => log("Worker igång. Väntar på jobb …"))
|
||||
.catch((err) => {
|
||||
console.error("Kunde inte registrera återkommande jobb:", err);
|
||||
// --- Minimal healthcheck-server så att flera instanser kan övervakas ---
|
||||
const healthPort = Number(process.env.WORKER_HEALTH_PORT ?? 4001);
|
||||
let processedCount = 0;
|
||||
let failedCount = 0;
|
||||
worker.on("completed", () => processedCount++);
|
||||
worker.on("failed", () => failedCount++);
|
||||
|
||||
const healthServer = createServer(async (req, res) => {
|
||||
if (req.url === "/healthz") {
|
||||
const [queueCount, dlqCount, workers] = await Promise.all([
|
||||
queue.getJobCounts("wait", "active", "delayed", "completed", "failed"),
|
||||
deadLetterQueue.getJobCounts("wait", "active", "completed", "failed"),
|
||||
queue.getWorkers(),
|
||||
]);
|
||||
res.writeHead(200, { "content-type": "application/json" });
|
||||
res.end(
|
||||
JSON.stringify({
|
||||
ok: true,
|
||||
instanceId,
|
||||
uptimeSeconds: Math.floor(process.uptime()),
|
||||
queue: queueCount,
|
||||
deadLetter: dlqCount,
|
||||
activeWorkers: workers.length,
|
||||
processedSinceStart: processedCount,
|
||||
failedSinceStart: failedCount,
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
res.writeHead(404);
|
||||
res.end("not found");
|
||||
});
|
||||
|
||||
async function start() {
|
||||
await registerRepeatableJobs();
|
||||
log("Worker igång. Väntar på jobb …");
|
||||
healthServer.listen(healthPort, () => {
|
||||
log(`Healthcheck på :${healthPort}`);
|
||||
});
|
||||
}
|
||||
|
||||
start().catch((err) => {
|
||||
console.error("Kunde inte starta worker:", err);
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
const shutdown = async () => {
|
||||
log("Stänger ner …");
|
||||
healthServer.close();
|
||||
await worker.close();
|
||||
await queue.close();
|
||||
await deadLetterQueue.close();
|
||||
connection.disconnect();
|
||||
await ctx.close();
|
||||
process.exit(0);
|
||||
@@ -194,5 +288,5 @@ process.on("SIGTERM", () => void shutdown());
|
||||
process.on("SIGINT", () => void shutdown());
|
||||
|
||||
function log(msg: string) {
|
||||
console.log(`[worker ${new Date().toISOString()}] ${msg}`);
|
||||
console.log(`[worker ${instanceId} ${new Date().toISOString()}] ${msg}`);
|
||||
}
|
||||
|
||||
@@ -273,7 +273,7 @@ export async function processMemorySync(ctx: WorkerContext): Promise<number> {
|
||||
return updates;
|
||||
}
|
||||
|
||||
/** BUILD_TRAINING_SAMPLE (spec §33): exportera korrigeringar MED samtycke. */
|
||||
/** BUILD_TRAINING_SAMPLE (spec §33): exportera korrigeringar MED samtycke till AAMOS. */
|
||||
export async function processTrainingExport(ctx: WorkerContext): Promise<number> {
|
||||
const corrections = await ctx.db
|
||||
.select()
|
||||
@@ -281,19 +281,48 @@ export async function processTrainingExport(ctx: WorkerContext): Promise<number>
|
||||
.where(isNull(schema.aiCorrections.exportedToTraining))
|
||||
.limit(100);
|
||||
|
||||
let exported = 0;
|
||||
for (const correction of corrections) {
|
||||
const snapshot = correction.consentSnapshot as Record<string, string>;
|
||||
// Endast korrigeringar där anonymiserad förbättring var beviljad vid tillfället.
|
||||
if (snapshot.anonymized_improvement !== "granted") continue;
|
||||
// Här skulle exporten till AAMOS training-pipeline ske (avidentifierad).
|
||||
const eligible = corrections.filter((c) => {
|
||||
const snapshot = c.consentSnapshot as Record<string, string>;
|
||||
return snapshot.anonymized_improvement === "granted";
|
||||
});
|
||||
|
||||
if (eligible.length === 0) return 0;
|
||||
|
||||
const result = await ctx.aamos.runTask(
|
||||
"EXPORT_TRAINING_SAMPLE",
|
||||
{
|
||||
marketLocale: "sv-SE",
|
||||
samples: eligible.map((c) => ({
|
||||
taskType: c.taskType,
|
||||
aiOutput: c.aiOutput as Record<string, unknown>,
|
||||
userCorrection: c.userCorrection as Record<string, unknown>,
|
||||
modelVersion: c.modelVersion ?? null,
|
||||
promptVersion: c.promptVersion ?? null,
|
||||
})),
|
||||
},
|
||||
{
|
||||
correlationId: `training-export-${Date.now()}`,
|
||||
consentFlags: {
|
||||
personalization: false,
|
||||
anonymizedImprovement: true,
|
||||
imageTraining: false,
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
if (result.status !== "ok" || !result.output) {
|
||||
throw new Error(`AAMOS training export failed: ${result.error ?? "unknown"}`);
|
||||
}
|
||||
|
||||
const batchId = result.output.batchId;
|
||||
for (const correction of eligible) {
|
||||
await ctx.db
|
||||
.update(schema.aiCorrections)
|
||||
.set({ exportedToTraining: new Date() })
|
||||
.set({ exportedToTraining: new Date(), trainingBatchId: batchId })
|
||||
.where(eq(schema.aiCorrections.id, correction.id));
|
||||
exported++;
|
||||
}
|
||||
return exported;
|
||||
|
||||
return eligible.length;
|
||||
}
|
||||
|
||||
/** VERIFY_SUBSCRIPTION: flagga prenumerationer som passerat expiry. */
|
||||
|
||||
Reference in New Issue
Block a user