Files
Tom Moor 2679093af3 chore: Vendorize throng as ClusterManager (#14198)
* chore: Vendorize throng as ClusterManager

* fix: Force kill workers after grace period, skip fork if shutting down

* fix: Give workers their full shutdown timeout before the primary exits

* chore: Use a named export for ShutdownHelper
2026-10-21 18:20:03 -04:00

95 lines
2.8 KiB
TypeScript

import Queue from "bull";
import { snakeCase } from "es-toolkit/compat";
import { toError } from "@shared/utils/error";
import { Second } from "@shared/utils/time";
import env from "@server/env";
import Logger from "@server/logging/Logger";
import Metrics from "@server/logging/Metrics";
import Redis from "@server/storage/redis";
import { ShutdownHelper, ShutdownOrder } from "@server/utils/ShutdownHelper";
export function createQueue(
name: string,
defaultJobOptions?: Partial<Queue.JobOptions>
) {
const prefix = `queue.${snakeCase(name)}`;
// Notes on reusing Redis connections for Bull:
// https://github.com/OptimalBits/bull/blob/b6d530f72a774be0fd4936ddb4ad9df3b183f4b6/PATTERNS.md#reusing-redis-connections
const queue = new Queue(name, {
createClient(type) {
switch (type) {
case "client":
return Redis.defaultClient;
case "subscriber":
return Redis.defaultSubscriber;
case "bclient":
return new Redis(env.REDIS_URL, {
maxRetriesPerRequest: null,
connectionNameSuffix: "bull",
});
default:
throw new Error(`Unexpected connection type: ${String(type)}`);
}
},
defaultJobOptions: {
removeOnComplete: true,
removeOnFail: true,
...defaultJobOptions,
},
});
queue.on("stalled", () => {
Metrics.increment(`${prefix}.jobs.stalled`);
});
queue.on("completed", () => {
Metrics.increment(`${prefix}.jobs.completed`);
});
queue.on("error", () => {
Metrics.increment(`${prefix}.jobs.errored`);
});
queue.on("failed", (job, err) => {
Metrics.increment(`${prefix}.jobs.failed`);
// Report on the final attempt to avoid noise from intermediate retries.
const attempts = job?.opts?.attempts ?? 1;
if ((job?.attemptsMade ?? 0) + 1 >= attempts) {
Logger.error(`Job failed in ${name} queue`, toError(err), {
jobId: job?.id,
attemptsMade: job?.attemptsMade,
data: job?.data,
});
}
});
let metricsTimer: NodeJS.Timeout | undefined;
if (env.ENVIRONMENT !== "test") {
metricsTimer = setInterval(async () => {
try {
Metrics.gauge(`${prefix}.count`, await queue.count());
Metrics.gauge(`${prefix}.delayed_count`, await queue.getDelayedCount());
} catch (err) {
// A transient error querying the queue (eg Redis blip or a queue closing
// during shutdown) should not crash the process with an unhandled rejection.
Logger.warn("Failed to gather queue metrics", {
queue: name,
error: err,
});
}
}, 5 * Second.ms);
metricsTimer.unref();
}
ShutdownHelper.add(name, ShutdownOrder.normal, async () => {
if (metricsTimer) {
clearInterval(metricsTimer);
}
await queue.close();
});
return queue;
}