import Queue from "bull"; import { snakeCase } from "es-toolkit/compat"; import { toError } from "@shared/utils/error"; import { Second } from "@shared/utils/time"; import env from "@server/env"; import Logger from "@server/logging/Logger"; import Metrics from "@server/logging/Metrics"; import Redis from "@server/storage/redis"; import { ShutdownHelper, ShutdownOrder } from "@server/utils/ShutdownHelper"; export function createQueue( name: string, defaultJobOptions?: Partial ) { const prefix = `queue.${snakeCase(name)}`; // Notes on reusing Redis connections for Bull: // https://github.com/OptimalBits/bull/blob/b6d530f72a774be0fd4936ddb4ad9df3b183f4b6/PATTERNS.md#reusing-redis-connections const queue = new Queue(name, { createClient(type) { switch (type) { case "client": return Redis.defaultClient; case "subscriber": return Redis.defaultSubscriber; case "bclient": return new Redis(env.REDIS_URL, { maxRetriesPerRequest: null, connectionNameSuffix: "bull", }); default: throw new Error(`Unexpected connection type: ${String(type)}`); } }, defaultJobOptions: { removeOnComplete: true, removeOnFail: true, ...defaultJobOptions, }, }); queue.on("stalled", () => { Metrics.increment(`${prefix}.jobs.stalled`); }); queue.on("completed", () => { Metrics.increment(`${prefix}.jobs.completed`); }); queue.on("error", () => { Metrics.increment(`${prefix}.jobs.errored`); }); queue.on("failed", (job, err) => { Metrics.increment(`${prefix}.jobs.failed`); // Report on the final attempt to avoid noise from intermediate retries. const attempts = job?.opts?.attempts ?? 1; if ((job?.attemptsMade ?? 0) + 1 >= attempts) { Logger.error(`Job failed in ${name} queue`, toError(err), { jobId: job?.id, attemptsMade: job?.attemptsMade, data: job?.data, }); } }); let metricsTimer: NodeJS.Timeout | undefined; if (env.ENVIRONMENT !== "test") { metricsTimer = setInterval(async () => { try { Metrics.gauge(`${prefix}.count`, await queue.count()); Metrics.gauge(`${prefix}.delayed_count`, await queue.getDelayedCount()); } catch (err) { // A transient error querying the queue (eg Redis blip or a queue closing // during shutdown) should not crash the process with an unhandled rejection. Logger.warn("Failed to gather queue metrics", { queue: name, error: err, }); } }, 5 * Second.ms); metricsTimer.unref(); } ShutdownHelper.add(name, ShutdownOrder.normal, async () => { if (metricsTimer) { clearInterval(metricsTimer); } await queue.close(); }); return queue; }