Files
Tom Moor eff1a43d9b fix: Keep serving requests when Redis is unavailable (#14457)
* fix: Fail fast on Redis for cache and rate limits, lock migrations in Postgres

* fix: Use the fail-fast Redis client for locks

* fix: Require a database pool of at least two connections

* chore: Rename cache Redis client to ephemeralClient
2026-11-03 08:45:56 -05:00

272 lines
8.4 KiB
TypeScript

import { sleep } from "@shared/utils/timers";
import Redis from "@server/storage/redis";
import type * as MutexLockModule from "./MutexLock";
const { MutexLock } =
await vi.importActual<typeof MutexLockModule>("./MutexLock");
const noRetry = { retry: { retryCount: 0 } };
/**
* Make the next SET apply in Redis and then reject, as when the reply is lost.
*/
function rejectAfterSet() {
const client = Redis.ephemeralClient;
const set = client.set.bind(client);
return vi
.spyOn(client, "set")
.mockImplementationOnce(async (...args: Parameters<typeof set>) => {
await set(...args);
throw new Error("Connection lost");
});
}
/**
* Make the next SET apply in Redis and then reply after a delay.
*
* @param ms The delay in milliseconds.
*/
function delaySetReply(ms: number) {
const client = Redis.ephemeralClient;
const set = client.set.bind(client);
return vi
.spyOn(client, "set")
.mockImplementationOnce(async (...args: Parameters<typeof set>) => {
const result = await set(...args);
await sleep(ms);
return result;
});
}
describe("MutexLock", () => {
beforeEach(async () => {
await Redis.ephemeralClient.flushdb();
});
describe("acquire", () => {
it("should set the key with an expiry", async () => {
const lock = await MutexLock.acquire("test:lock", 1000);
expect(await Redis.ephemeralClient.get("test:lock")).toEqual(lock.value);
expect(await Redis.ephemeralClient.pttl("test:lock")).toBeGreaterThan(0);
expect(lock.expiration).toBeGreaterThan(Date.now());
});
it("should throw when the lock is held elsewhere", async () => {
await MutexLock.acquire("test:lock", 1000);
await expect(
MutexLock.acquire("test:lock", 1000, noRetry)
).rejects.toThrow();
});
it("should retry until the lock is released", async () => {
const first = await MutexLock.acquire("test:lock", 1000);
setTimeout(() => void MutexLock.release(first), 50);
const second = await MutexLock.acquire("test:lock", 1000, {
retry: { retryCount: 10, retryDelay: 20, retryJitter: 0 },
});
expect(await Redis.ephemeralClient.get("test:lock")).toEqual(
second.value
);
});
it("should retry after a Redis error", async () => {
const spy = vi
.spyOn(Redis.ephemeralClient, "set")
.mockRejectedValueOnce(new Error("READONLY"));
const lock = await MutexLock.acquire("test:lock", 1000, {
retry: { retryCount: 1, retryDelay: 10, retryJitter: 0 },
});
spy.mockRestore();
expect(await Redis.ephemeralClient.get("test:lock")).toEqual(lock.value);
});
it("should throw the Redis error when all attempts fail", async () => {
const spy = vi
.spyOn(Redis.ephemeralClient, "set")
.mockRejectedValue(new Error("READONLY"));
await expect(
MutexLock.acquire("test:lock", 1000, {
retry: { retryCount: 1, retryDelay: 10, retryJitter: 0 },
})
).rejects.toThrow("READONLY");
spy.mockRestore();
});
it("should recover its own lock after a SET reply is lost", async () => {
const spy = rejectAfterSet();
const lock = await MutexLock.acquire("test:lock", 1000, {
retry: { retryCount: 1, retryDelay: 10, retryJitter: 0 },
});
spy.mockRestore();
expect(await Redis.ephemeralClient.get("test:lock")).toEqual(lock.value);
});
it("should discard its own lock when giving up after a lost reply", async () => {
const spy = rejectAfterSet();
await expect(
MutexLock.acquire("test:lock", 1000, noRetry)
).rejects.toThrow("Connection lost");
spy.mockRestore();
expect(await Redis.ephemeralClient.get("test:lock")).toBeNull();
});
it("should not return a lock that expired before the reply", async () => {
const spy = delaySetReply(150);
await expect(
MutexLock.acquire("test:lock", 100, noRetry)
).rejects.toThrow();
spy.mockRestore();
});
});
describe("release", () => {
it("should delete the key", async () => {
const lock = await MutexLock.acquire("test:lock", 1000);
expect(await MutexLock.release(lock)).toEqual(true);
expect(await Redis.ephemeralClient.get("test:lock")).toBeNull();
});
it("should not delete a lock owned by another holder", async () => {
const lock = await MutexLock.acquire("test:lock", 1000);
await Redis.ephemeralClient.set("test:lock", "other");
expect(await MutexLock.release(lock)).toEqual(false);
expect(await Redis.ephemeralClient.get("test:lock")).toEqual("other");
});
});
describe("using", () => {
it("should hold the lock while the routine runs", async () => {
const result = await MutexLock.using("test:lock", 1000, async () => {
await expect(
MutexLock.acquire("test:lock", 1000, noRetry)
).rejects.toThrow();
return "done";
});
expect(result).toEqual("done");
expect(await Redis.ephemeralClient.get("test:lock")).toBeNull();
});
it("should release the lock when the routine throws", async () => {
await expect(
MutexLock.using("test:lock", 1000, async () => {
throw new Error("failed");
})
).rejects.toThrow("failed");
expect(await Redis.ephemeralClient.get("test:lock")).toBeNull();
});
it("should extend the lock past its initial duration", async () => {
await MutexLock.using("test:lock", 200, async (signal) => {
await new Promise((resolve) => setTimeout(resolve, 500));
expect(signal.aborted).toEqual(false);
expect(await Redis.ephemeralClient.get("test:lock")).not.toBeNull();
});
});
it("should extend the lock in time when acquisition is slow", async () => {
const spy = delaySetReply(600);
await MutexLock.using("test:lock", 1000, async () => {
spy.mockRestore();
await sleep(500);
await expect(
MutexLock.acquire("test:lock", 1000, noRetry)
).rejects.toThrow();
});
});
it("should abort the signal when an extension stays pending past expiry", async () => {
const client = Redis.ephemeralClient;
await MutexLock.using("test:lock", 200, async (signal) => {
const spy = vi.spyOn(client, "eval").mockImplementation(async () => {
await sleep(300);
return 1;
});
await sleep(250);
spy.mockRestore();
expect(signal.aborted).toEqual(true);
expect(() => signal.throwIfAborted()).toThrow("expired");
});
});
it("should abort the signal when the lock is lost", async () => {
await MutexLock.using("test:lock", 200, async (signal) => {
await Redis.ephemeralClient.set("test:lock", "other");
await new Promise((resolve) => setTimeout(resolve, 200));
expect(signal.aborted).toEqual(true);
expect(() => signal.throwIfAborted()).toThrow("was lost");
});
expect(await Redis.ephemeralClient.get("test:lock")).toEqual("other");
});
});
describe("tryUsing", () => {
it("should run the routine when the lock is free", async () => {
const result = await MutexLock.tryUsing(
"test:lock",
1000,
async () => "done"
);
expect(result).toEqual("done");
expect(await Redis.ephemeralClient.get("test:lock")).toBeNull();
});
it("should skip the routine when the lock is held elsewhere", async () => {
await MutexLock.acquire("test:lock", 1000);
const routine = vi.fn();
const result = await MutexLock.tryUsing("test:lock", 1000, routine);
expect(result).toBeUndefined();
expect(routine).not.toHaveBeenCalled();
});
it("should skip the routine when the reply arrives after expiry", async () => {
const spy = delaySetReply(150);
const routine = vi.fn();
const result = await MutexLock.tryUsing("test:lock", 100, routine);
spy.mockRestore();
expect(result).toBeUndefined();
expect(routine).not.toHaveBeenCalled();
});
it("should discard its own lock when the SET reply is lost", async () => {
const spy = rejectAfterSet();
const routine = vi.fn();
await expect(
MutexLock.tryUsing("test:lock", 1000, routine)
).rejects.toThrow("Connection lost");
spy.mockRestore();
expect(routine).not.toHaveBeenCalled();
expect(await Redis.ephemeralClient.get("test:lock")).toBeNull();
});
});
});