mirror of
https://github.com/outline/outline.git
synced 2026-11-20 21:23:06 +03:00
* chore: Vendorize throng as ClusterManager * fix: Force kill workers after grace period, skip fork if shutting down * fix: Give workers their full shutdown timeout before the primary exits * chore: Use a named export for ShutdownHelper
95 lines
2.8 KiB
TypeScript
95 lines
2.8 KiB
TypeScript
import Queue from "bull";
|
|
import { snakeCase } from "es-toolkit/compat";
|
|
import { toError } from "@shared/utils/error";
|
|
import { Second } from "@shared/utils/time";
|
|
import env from "@server/env";
|
|
import Logger from "@server/logging/Logger";
|
|
import Metrics from "@server/logging/Metrics";
|
|
import Redis from "@server/storage/redis";
|
|
import { ShutdownHelper, ShutdownOrder } from "@server/utils/ShutdownHelper";
|
|
|
|
export function createQueue(
|
|
name: string,
|
|
defaultJobOptions?: Partial<Queue.JobOptions>
|
|
) {
|
|
const prefix = `queue.${snakeCase(name)}`;
|
|
|
|
// Notes on reusing Redis connections for Bull:
|
|
// https://github.com/OptimalBits/bull/blob/b6d530f72a774be0fd4936ddb4ad9df3b183f4b6/PATTERNS.md#reusing-redis-connections
|
|
const queue = new Queue(name, {
|
|
createClient(type) {
|
|
switch (type) {
|
|
case "client":
|
|
return Redis.defaultClient;
|
|
|
|
case "subscriber":
|
|
return Redis.defaultSubscriber;
|
|
|
|
case "bclient":
|
|
return new Redis(env.REDIS_URL, {
|
|
maxRetriesPerRequest: null,
|
|
connectionNameSuffix: "bull",
|
|
});
|
|
|
|
default:
|
|
throw new Error(`Unexpected connection type: ${String(type)}`);
|
|
}
|
|
},
|
|
defaultJobOptions: {
|
|
removeOnComplete: true,
|
|
removeOnFail: true,
|
|
...defaultJobOptions,
|
|
},
|
|
});
|
|
queue.on("stalled", () => {
|
|
Metrics.increment(`${prefix}.jobs.stalled`);
|
|
});
|
|
queue.on("completed", () => {
|
|
Metrics.increment(`${prefix}.jobs.completed`);
|
|
});
|
|
queue.on("error", () => {
|
|
Metrics.increment(`${prefix}.jobs.errored`);
|
|
});
|
|
queue.on("failed", (job, err) => {
|
|
Metrics.increment(`${prefix}.jobs.failed`);
|
|
|
|
// Report on the final attempt to avoid noise from intermediate retries.
|
|
const attempts = job?.opts?.attempts ?? 1;
|
|
if ((job?.attemptsMade ?? 0) + 1 >= attempts) {
|
|
Logger.error(`Job failed in ${name} queue`, toError(err), {
|
|
jobId: job?.id,
|
|
attemptsMade: job?.attemptsMade,
|
|
data: job?.data,
|
|
});
|
|
}
|
|
});
|
|
|
|
let metricsTimer: NodeJS.Timeout | undefined;
|
|
if (env.ENVIRONMENT !== "test") {
|
|
metricsTimer = setInterval(async () => {
|
|
try {
|
|
Metrics.gauge(`${prefix}.count`, await queue.count());
|
|
Metrics.gauge(`${prefix}.delayed_count`, await queue.getDelayedCount());
|
|
} catch (err) {
|
|
// A transient error querying the queue (eg Redis blip or a queue closing
|
|
// during shutdown) should not crash the process with an unhandled rejection.
|
|
Logger.warn("Failed to gather queue metrics", {
|
|
queue: name,
|
|
error: err,
|
|
});
|
|
}
|
|
}, 5 * Second.ms);
|
|
|
|
metricsTimer.unref();
|
|
}
|
|
|
|
ShutdownHelper.add(name, ShutdownOrder.normal, async () => {
|
|
if (metricsTimer) {
|
|
clearInterval(metricsTimer);
|
|
}
|
|
await queue.close();
|
|
});
|
|
|
|
return queue;
|
|
}
|