fix: Keep serving requests when Redis is unavailable (#14457)

* fix: Fail fast on Redis for cache and rate limits, lock migrations in Postgres

* fix: Use the fail-fast Redis client for locks

* fix: Require a database pool of at least two connections

* chore: Rename cache Redis client to ephemeralClient
This commit is contained in:
Tom Moor authored and GitHub committed 2026-11-03 08:45:56 -05:00
1 parent fb2f0e76d1
commit eff1a43d9b
10 files changed
+121 -51

No files matched your search

+2 -1
View File
@@ -60,7 +60,8 @@ DEFAULT_LANGUAGE=en_US
DATABASE_URL=postgres://user:pass@postgres:5432/outline
# The in-memory database pool per-process settings. Ensure that the pool size that will not exceed
# the maximum number of connections allowed by your database. Defaults to 0 and 5.
# the maximum number of connections allowed by your database. Defaults to 0 and 5, the maximum
# must be at least 2.
DATABASE_CONNECTION_POOL_MIN=
DATABASE_CONNECTION_POOL_MAX=
+15 -1
View File
@@ -167,10 +167,12 @@ export class Environment {
);
/**
* Database connection pool configuration.
* Database connection pool configuration. At least two connections are
* needed, as some operations query outside of a transaction they hold open.
*/
@IsNumber()
@IsOptional()
@Min(2)
public DATABASE_CONNECTION_POOL_MAX = this.toOptionalNumber(
environment.DATABASE_CONNECTION_POOL_MAX
);
@@ -232,6 +234,18 @@ export class Environment {
10
);
/**
* The timeout in milliseconds for a command on the redis connection used for
* ephemeral data such as caches, locks and rate limits, after which the
* command fails.
*/
@IsNumber()
@Min(100)
public REDIS_EPHEMERAL_COMMAND_TIMEOUT = parseInt(
environment.REDIS_EPHEMERAL_COMMAND_TIMEOUT || "500",
10
);
/**
* The fully qualified, external facing domain name of the server.
* If not set, will be derived from HEROKU_APP_NAME for Heroku deployments.
+1 -1
View File
@@ -174,7 +174,7 @@ export function rateLimiter(config: RateLimiterConfig) {
duration: 60,
points: env.RATE_LIMITER_REQUESTS,
keyPrefix: RateLimiter.RATE_LIMITER_REDIS_KEY_PREFIX,
storeClient: Redis.defaultClient,
storeClient: Redis.ephemeralClient,
}
)
);
+10 -9
View File
@@ -1,8 +1,8 @@
import "./bootstrap";
import { styleText } from "node:util";
import type { Transaction } from "sequelize";
import { isEmpty } from "es-toolkit/compat";
import { toError, errToString } from "@shared/utils/error";
import { Minute } from "@shared/utils/time";
import env from "@server/env";
import Logger from "@server/logging/Logger";
import AuthenticationProvider from "@server/models/AuthenticationProvider";
@@ -12,8 +12,8 @@ import {
migrations,
sequelize,
} from "@server/storage/database";
import { LockHelper } from "@server/storage/LockHelper";
import { getArg } from "@server/utils/args";
import { MutexLock } from "@server/utils/MutexLock";
import { ShutdownHelper } from "@server/utils/ShutdownHelper";
/**
@@ -21,11 +21,12 @@ import { ShutdownHelper } from "@server/utils/ShutdownHelper";
* --no-migrate flag was passed in which case the process exits with an error.
*/
async function checkPendingMigrations() {
let lock;
let lock: Transaction | undefined;
try {
lock = await MutexLock.acquire("migrations", 10 * Minute.ms, {
releaseOnShutdown: true,
});
// The advisory lock is held by this transaction until it ends, while the
// migrations themselves run on other connections.
lock = await sequelize.transaction();
await LockHelper.acquire(sequelize, "migrations", lock);
const pending = await migrations.pending();
if (!isEmpty(pending)) {
@@ -58,9 +59,9 @@ async function checkPendingMigrations() {
Logger.fatal(styleText("red", message), error);
}
} finally {
if (lock) {
await MutexLock.release(lock);
}
await lock?.rollback().catch((err) => {
Logger.warn("Failed to release migrations lock", toError(err));
});
}
}
+18
View File
@@ -210,6 +210,7 @@ export default class RedisAdapter extends Redis {
private static client: RedisAdapter;
private static subscriber: RedisAdapter;
private static ephemeral: RedisAdapter;
private static collabClient: RedisAdapter;
private static collabSubscriber: RedisAdapter;
@@ -232,6 +233,23 @@ export default class RedisAdapter extends Redis {
);
}
/**
* A Redis adapter for ephemeral data that can be lost, such as caches, locks
* and rate limits. Commands fail at once when there is no connection, and
* after a short timeout when Redis does not reply, so that callers can
* continue without Redis.
*/
public static get ephemeralClient(): RedisAdapter {
return (
this.ephemeral ||
(this.ephemeral = new this(env.REDIS_URL, {
commandTimeout: env.REDIS_EPHEMERAL_COMMAND_TIMEOUT,
enableOfflineQueue: false,
connectionNameSuffix: "ephemeral",
}))
);
}
/**
* A Redis adapter for collaboration-related operations.
*/
+30
View File
@@ -1,5 +1,6 @@
import Redis from "@server/storage/redis";
import { CacheHelper } from "./CacheHelper";
import { MutexLock } from "./MutexLock";
describe("CacheHelper", () => {
beforeEach(async () => {
@@ -112,6 +113,35 @@ describe("CacheHelper", () => {
});
});
describe("getDataOrSet when Redis is unavailable", () => {
const { status } = Redis.ephemeralClient;
afterEach(() => {
Redis.ephemeralClient.status = status;
vi.restoreAllMocks();
});
it("should return the callback result without taking a lock", async () => {
Redis.ephemeralClient.status = "reconnecting";
vi.spyOn(Redis.ephemeralClient, "get").mockRejectedValue(
new Error("Stream isn't writeable")
);
vi.spyOn(Redis.ephemeralClient, "set").mockRejectedValue(
new Error("Stream isn't writeable")
);
const acquire = vi.spyOn(MutexLock, "acquire");
const result = await CacheHelper.getDataOrSet(
"test:unavailable",
async () => "value",
60
);
expect(result).toEqual("value");
expect(acquire).not.toHaveBeenCalled();
});
});
describe("removeData", () => {
it("should remove a single key", async () => {
await CacheHelper.setData("test:key", "value", 60);
+14 -10
View File
@@ -91,12 +91,15 @@ export class CacheHelper {
let lock;
const lockKey = `lock:${key}`;
try {
try {
lock = await MutexLock.acquire(lockKey, lockTimeout, {
retry: MutexLock.cacheRetrySettings,
});
} catch (err) {
Logger.error(`Could not acquire lock for ${key}`, toError(err));
// Skip the lock when Redis is unavailable, as it would only add latency.
if (Redis.ephemeralClient.status === "ready") {
try {
lock = await MutexLock.acquire(lockKey, lockTimeout, {
retry: MutexLock.cacheRetrySettings,
});
} catch (err) {
Logger.error(`Could not acquire lock for ${key}`, toError(err));
}
}
cache = await this.getData<T>(key);
if (cache !== undefined) {
@@ -139,7 +142,7 @@ export class CacheHelper {
*/
public static async getData<T>(key: string): Promise<T | undefined> {
try {
const data = await Redis.defaultClient.get(key);
const data = await Redis.ephemeralClient.get(key);
if (data !== null) {
return JSON.parse(data);
}
@@ -162,7 +165,7 @@ export class CacheHelper {
*/
public static async setData<T>(key: string, data: T, expiry?: number) {
try {
await Redis.defaultClient.set(
await Redis.ephemeralClient.set(
key,
JSON.stringify(data),
"EX",
@@ -181,7 +184,7 @@ export class CacheHelper {
*/
public static async removeData(key: string) {
try {
await Redis.defaultClient.del(key);
await Redis.ephemeralClient.del(key);
} catch (err) {
Logger.error(
`Could not remove cached entry against ${key}`,
@@ -194,7 +197,8 @@ export class CacheHelper {
* Clears all cache data with the given prefix. Keys are discovered with an
* incremental SCAN rather than KEYS and each batch is removed with UNLINK
* as the scan progresses, so neither discovery nor deletion blocks the
* Redis event loop and matched keys are never buffered in full.
* Redis event loop and matched keys are never buffered in full. This uses
* the default client, which waits for Redis, as it runs in the background.
*
* @param prefix Prefix to clear cache data
*/
+24 -22
View File
@@ -11,7 +11,7 @@ const noRetry = { retry: { retryCount: 0 } };
* Make the next SET apply in Redis and then reject, as when the reply is lost.
*/
function rejectAfterSet() {
const client = Redis.defaultClient;
const client = Redis.ephemeralClient;
const set = client.set.bind(client);
return vi
.spyOn(client, "set")
@@ -27,7 +27,7 @@ function rejectAfterSet() {
* @param ms The delay in milliseconds.
*/
function delaySetReply(ms: number) {
const client = Redis.defaultClient;
const client = Redis.ephemeralClient;
const set = client.set.bind(client);
return vi
.spyOn(client, "set")
@@ -40,15 +40,15 @@ function delaySetReply(ms: number) {
describe("MutexLock", () => {
beforeEach(async () => {
await Redis.defaultClient.flushdb();
await Redis.ephemeralClient.flushdb();
});
describe("acquire", () => {
it("should set the key with an expiry", async () => {
const lock = await MutexLock.acquire("test:lock", 1000);
expect(await Redis.defaultClient.get("test:lock")).toEqual(lock.value);
expect(await Redis.defaultClient.pttl("test:lock")).toBeGreaterThan(0);
expect(await Redis.ephemeralClient.get("test:lock")).toEqual(lock.value);
expect(await Redis.ephemeralClient.pttl("test:lock")).toBeGreaterThan(0);
expect(lock.expiration).toBeGreaterThan(Date.now());
});
@@ -68,12 +68,14 @@ describe("MutexLock", () => {
retry: { retryCount: 10, retryDelay: 20, retryJitter: 0 },
});
expect(await Redis.defaultClient.get("test:lock")).toEqual(second.value);
expect(await Redis.ephemeralClient.get("test:lock")).toEqual(
second.value
);
});
it("should retry after a Redis error", async () => {
const spy = vi
.spyOn(Redis.defaultClient, "set")
.spyOn(Redis.ephemeralClient, "set")
.mockRejectedValueOnce(new Error("READONLY"));
const lock = await MutexLock.acquire("test:lock", 1000, {
@@ -81,12 +83,12 @@ describe("MutexLock", () => {
});
spy.mockRestore();
expect(await Redis.defaultClient.get("test:lock")).toEqual(lock.value);
expect(await Redis.ephemeralClient.get("test:lock")).toEqual(lock.value);
});
it("should throw the Redis error when all attempts fail", async () => {
const spy = vi
.spyOn(Redis.defaultClient, "set")
.spyOn(Redis.ephemeralClient, "set")
.mockRejectedValue(new Error("READONLY"));
await expect(
@@ -105,7 +107,7 @@ describe("MutexLock", () => {
});
spy.mockRestore();
expect(await Redis.defaultClient.get("test:lock")).toEqual(lock.value);
expect(await Redis.ephemeralClient.get("test:lock")).toEqual(lock.value);
});
it("should discard its own lock when giving up after a lost reply", async () => {
@@ -116,7 +118,7 @@ describe("MutexLock", () => {
).rejects.toThrow("Connection lost");
spy.mockRestore();
expect(await Redis.defaultClient.get("test:lock")).toBeNull();
expect(await Redis.ephemeralClient.get("test:lock")).toBeNull();
});
it("should not return a lock that expired before the reply", async () => {
@@ -134,15 +136,15 @@ describe("MutexLock", () => {
const lock = await MutexLock.acquire("test:lock", 1000);
expect(await MutexLock.release(lock)).toEqual(true);
expect(await Redis.defaultClient.get("test:lock")).toBeNull();
expect(await Redis.ephemeralClient.get("test:lock")).toBeNull();
});
it("should not delete a lock owned by another holder", async () => {
const lock = await MutexLock.acquire("test:lock", 1000);
await Redis.defaultClient.set("test:lock", "other");
await Redis.ephemeralClient.set("test:lock", "other");
expect(await MutexLock.release(lock)).toEqual(false);
expect(await Redis.defaultClient.get("test:lock")).toEqual("other");
expect(await Redis.ephemeralClient.get("test:lock")).toEqual("other");
});
});
@@ -156,7 +158,7 @@ describe("MutexLock", () => {
});
expect(result).toEqual("done");
expect(await Redis.defaultClient.get("test:lock")).toBeNull();
expect(await Redis.ephemeralClient.get("test:lock")).toBeNull();
});
it("should release the lock when the routine throws", async () => {
@@ -166,7 +168,7 @@ describe("MutexLock", () => {
})
).rejects.toThrow("failed");
expect(await Redis.defaultClient.get("test:lock")).toBeNull();
expect(await Redis.ephemeralClient.get("test:lock")).toBeNull();
});
it("should extend the lock past its initial duration", async () => {
@@ -174,7 +176,7 @@ describe("MutexLock", () => {
await new Promise((resolve) => setTimeout(resolve, 500));
expect(signal.aborted).toEqual(false);
expect(await Redis.defaultClient.get("test:lock")).not.toBeNull();
expect(await Redis.ephemeralClient.get("test:lock")).not.toBeNull();
});
});
@@ -192,7 +194,7 @@ describe("MutexLock", () => {
});
it("should abort the signal when an extension stays pending past expiry", async () => {
const client = Redis.defaultClient;
const client = Redis.ephemeralClient;
await MutexLock.using("test:lock", 200, async (signal) => {
const spy = vi.spyOn(client, "eval").mockImplementation(async () => {
@@ -209,14 +211,14 @@ describe("MutexLock", () => {
it("should abort the signal when the lock is lost", async () => {
await MutexLock.using("test:lock", 200, async (signal) => {
await Redis.defaultClient.set("test:lock", "other");
await Redis.ephemeralClient.set("test:lock", "other");
await new Promise((resolve) => setTimeout(resolve, 200));
expect(signal.aborted).toEqual(true);
expect(() => signal.throwIfAborted()).toThrow("was lost");
});
expect(await Redis.defaultClient.get("test:lock")).toEqual("other");
expect(await Redis.ephemeralClient.get("test:lock")).toEqual("other");
});
});
@@ -229,7 +231,7 @@ describe("MutexLock", () => {
);
expect(result).toEqual("done");
expect(await Redis.defaultClient.get("test:lock")).toBeNull();
expect(await Redis.ephemeralClient.get("test:lock")).toBeNull();
});
it("should skip the routine when the lock is held elsewhere", async () => {
@@ -263,7 +265,7 @@ describe("MutexLock", () => {
spy.mockRestore();
expect(routine).not.toHaveBeenCalled();
expect(await Redis.defaultClient.get("test:lock")).toBeNull();
expect(await Redis.ephemeralClient.get("test:lock")).toBeNull();
});
});
});
+3 -3
View File
@@ -297,7 +297,7 @@ export class MutexLock {
value,
expiration: Date.now() + timeout,
};
const result = await Redis.defaultClient.set(
const result = await Redis.ephemeralClient.set(
resource,
value,
"PX",
@@ -348,7 +348,7 @@ export class MutexLock {
*/
private static async pexpire(lock: Lock, timeout: number): Promise<boolean> {
const expiration = Date.now() + timeout;
const result = await Redis.defaultClient.eval(
const result = await Redis.ephemeralClient.eval(
this.extendScript,
1,
lock.resource,
@@ -369,7 +369,7 @@ export class MutexLock {
* @returns true if the lock was deleted.
*/
private static async del(lock: Lock): Promise<boolean> {
const result = await Redis.defaultClient.eval(
const result = await Redis.ephemeralClient.eval(
this.releaseScript,
1,
lock.resource,
+4 -4
View File
@@ -40,7 +40,7 @@ export default class RateLimiter {
static get defaultRateLimiter(): RateLimiterRedis {
if (!this._defaultRateLimiter) {
this._defaultRateLimiter = new RateLimiterRedis({
storeClient: Redis.defaultClient,
storeClient: Redis.ephemeralClient,
points: env.RATE_LIMITER_REQUESTS,
duration: env.RATE_LIMITER_DURATION_WINDOW,
keyPrefix: this.RATE_LIMITER_REDIS_KEY_PREFIX,
@@ -125,7 +125,7 @@ export default class RateLimiter {
*/
static async cacheUserForToken(token: string, userId: string): Promise<void> {
try {
await Redis.defaultClient.set(
await Redis.ephemeralClient.set(
this.tokenCacheKey(token),
userId,
"EX",
@@ -145,7 +145,7 @@ export default class RateLimiter {
*/
static async getCachedUserIdForToken(token: string): Promise<string | null> {
try {
return await Redis.defaultClient.get(this.tokenCacheKey(token));
return await Redis.ephemeralClient.get(this.tokenCacheKey(token));
} catch (err) {
Logger.warn(
"Failed to read cached user for rate limiter token",
@@ -163,7 +163,7 @@ export default class RateLimiter {
*/
static async clearCachedToken(token: string): Promise<void> {
try {
await Redis.defaultClient.del(this.tokenCacheKey(token));
await Redis.ephemeralClient.del(this.tokenCacheKey(token));
} catch (err) {
Logger.warn("Failed to clear cached rate limiter token", toError(err));
}