mirror of
https://github.com/outline/outline.git
synced 2026-11-21 13:34:02 +03:00
126 lines
3.7 KiB
TypeScript
126 lines
3.7 KiB
TypeScript
import type { Queue } from "bull";
|
|
import Logger from "@server/logging/Logger";
|
|
import Redis from "@server/storage/redis";
|
|
import HealthMonitor from "./HealthMonitor";
|
|
|
|
/**
|
|
* Minimal Bull queue stub exposing only the event emitter and key helper that the monitor
|
|
* relies on.
|
|
*/
|
|
function buildQueue(name: string) {
|
|
return {
|
|
name,
|
|
on() {
|
|
return this;
|
|
},
|
|
toKey(type: string) {
|
|
return `bull:${name}:${type}`;
|
|
},
|
|
};
|
|
}
|
|
|
|
/** Fills the waiting list of a queue with the given number of jobs. */
|
|
async function addWaitingJobs(name: string, count: number) {
|
|
await Redis.defaultClient.rpush(
|
|
`bull:${name}:wait`,
|
|
...Array.from({ length: count }, (_value, index) => `job-${index}`)
|
|
);
|
|
}
|
|
|
|
describe("HealthMonitor", () => {
|
|
let exit: ReturnType<typeof vi.spyOn>;
|
|
let fatal: ReturnType<typeof vi.spyOn>;
|
|
|
|
beforeEach(() => {
|
|
exit = vi
|
|
.spyOn(process, "exit")
|
|
.mockImplementation(() => undefined as never);
|
|
fatal = vi.spyOn(Logger, "fatal").mockImplementation(() => undefined);
|
|
});
|
|
|
|
afterEach(() => {
|
|
vi.useRealTimers();
|
|
vi.restoreAllMocks();
|
|
});
|
|
|
|
it("should exit when an inactive queue has jobs waiting", async () => {
|
|
const queue = buildQueue("stalled");
|
|
await addWaitingJobs("stalled", HealthMonitor.maxWaitingJobs + 1);
|
|
|
|
vi.useFakeTimers();
|
|
HealthMonitor.start(queue as unknown as Queue);
|
|
|
|
// A single unhealthy check only warns.
|
|
await vi.advanceTimersByTimeAsync(HealthMonitor.checkInterval);
|
|
expect(fatal).not.toHaveBeenCalled();
|
|
|
|
await vi.advanceTimersByTimeAsync(HealthMonitor.checkInterval);
|
|
|
|
expect(fatal).toHaveBeenCalledWith(
|
|
"Queue has stopped processing jobs",
|
|
expect.any(Error),
|
|
expect.objectContaining({
|
|
queue: "stalled",
|
|
waiting: HealthMonitor.maxWaitingJobs + 1,
|
|
})
|
|
);
|
|
|
|
expect(exit).not.toHaveBeenCalled();
|
|
await vi.advanceTimersByTimeAsync(HealthMonitor.shutdownTimeout);
|
|
expect(exit).toHaveBeenCalledWith(1);
|
|
});
|
|
|
|
it("should exit when the check does not complete in time", async () => {
|
|
const queue = buildQueue("wedged");
|
|
|
|
// Stand in for a connection that accepts commands but never answers them.
|
|
vi.spyOn(HealthMonitor, "client", "get").mockReturnValue({
|
|
llen: () => new Promise<number>(() => {}),
|
|
} as unknown as Redis);
|
|
|
|
vi.useFakeTimers();
|
|
HealthMonitor.start(queue as unknown as Queue);
|
|
|
|
const round = HealthMonitor.checkInterval + HealthMonitor.checkTimeout;
|
|
|
|
await vi.advanceTimersByTimeAsync(round);
|
|
expect(fatal).not.toHaveBeenCalled();
|
|
|
|
await vi.advanceTimersByTimeAsync(round);
|
|
|
|
expect(fatal).toHaveBeenCalledWith(
|
|
"Queue health check failed",
|
|
expect.any(Error),
|
|
expect.objectContaining({ queue: "wedged" })
|
|
);
|
|
|
|
await vi.advanceTimersByTimeAsync(HealthMonitor.shutdownTimeout);
|
|
expect(exit).toHaveBeenCalledWith(1);
|
|
});
|
|
|
|
it("should not exit when failed checks are not consecutive", async () => {
|
|
const queue = buildQueue("recovered");
|
|
let failing = true;
|
|
|
|
// Fails the way a momentary loss of the Redis connection would.
|
|
vi.spyOn(HealthMonitor, "client", "get").mockReturnValue({
|
|
llen: () =>
|
|
failing
|
|
? Promise.reject(new Error("Connection is closed."))
|
|
: Promise.resolve(0),
|
|
} as unknown as Redis);
|
|
|
|
vi.useFakeTimers();
|
|
HealthMonitor.start(queue as unknown as Queue);
|
|
|
|
await vi.advanceTimersByTimeAsync(HealthMonitor.checkInterval);
|
|
failing = false;
|
|
await vi.advanceTimersByTimeAsync(HealthMonitor.checkInterval);
|
|
failing = true;
|
|
await vi.advanceTimersByTimeAsync(HealthMonitor.checkInterval);
|
|
|
|
expect(fatal).not.toHaveBeenCalled();
|
|
expect(exit).not.toHaveBeenCalled();
|
|
});
|
|
});
|