From eff1a43d9b18ff6571b8f444c0d77a7571cd7595 Mon Sep 17 00:00:00 2001 From: Tom Moor Date: Tue, 3 Nov 2026 08:45:56 -0500 Subject: [PATCH] fix: Keep serving requests when Redis is unavailable (#14457) * fix: Fail fast on Redis for cache and rate limits, lock migrations in Postgres * fix: Use the fail-fast Redis client for locks * fix: Require a database pool of at least two connections * chore: Rename cache Redis client to ephemeralClient --- .env.sample | 3 +- server/env.ts | 16 ++++++++++- server/middlewares/rateLimiter.ts | 2 +- server/scripts/checkMigrations.ts | 19 +++++++------ server/storage/redis.ts | 18 ++++++++++++ server/utils/CacheHelper.test.ts | 30 ++++++++++++++++++++ server/utils/CacheHelper.ts | 24 +++++++++------- server/utils/MutexLock.test.ts | 46 ++++++++++++++++--------------- server/utils/MutexLock.ts | 6 ++-- server/utils/RateLimiter.ts | 8 +++--- 10 files changed, 121 insertions(+), 51 deletions(-) diff --git a/.env.sample b/.env.sample index d56832fc34..58e8188cae 100644 --- a/.env.sample +++ b/.env.sample @@ -60,7 +60,8 @@ DEFAULT_LANGUAGE=en_US DATABASE_URL=postgres://user:pass@postgres:5432/outline # The in-memory database pool per-process settings. Ensure that the pool size that will not exceed -# the maximum number of connections allowed by your database. Defaults to 0 and 5. +# the maximum number of connections allowed by your database. Defaults to 0 and 5, the maximum +# must be at least 2. DATABASE_CONNECTION_POOL_MIN= DATABASE_CONNECTION_POOL_MAX= diff --git a/server/env.ts b/server/env.ts index 3b0028605c..59a398672d 100644 --- a/server/env.ts +++ b/server/env.ts @@ -167,10 +167,12 @@ export class Environment { ); /** - * Database connection pool configuration. + * Database connection pool configuration. At least two connections are + * needed, as some operations query outside of a transaction they hold open. */ @IsNumber() @IsOptional() + @Min(2) public DATABASE_CONNECTION_POOL_MAX = this.toOptionalNumber( environment.DATABASE_CONNECTION_POOL_MAX ); @@ -232,6 +234,18 @@ export class Environment { 10 ); + /** + * The timeout in milliseconds for a command on the redis connection used for + * ephemeral data such as caches, locks and rate limits, after which the + * command fails. + */ + @IsNumber() + @Min(100) + public REDIS_EPHEMERAL_COMMAND_TIMEOUT = parseInt( + environment.REDIS_EPHEMERAL_COMMAND_TIMEOUT || "500", + 10 + ); + /** * The fully qualified, external facing domain name of the server. * If not set, will be derived from HEROKU_APP_NAME for Heroku deployments. diff --git a/server/middlewares/rateLimiter.ts b/server/middlewares/rateLimiter.ts index 7c908470c6..6dd271b0e4 100644 --- a/server/middlewares/rateLimiter.ts +++ b/server/middlewares/rateLimiter.ts @@ -174,7 +174,7 @@ export function rateLimiter(config: RateLimiterConfig) { duration: 60, points: env.RATE_LIMITER_REQUESTS, keyPrefix: RateLimiter.RATE_LIMITER_REDIS_KEY_PREFIX, - storeClient: Redis.defaultClient, + storeClient: Redis.ephemeralClient, } ) ); diff --git a/server/scripts/checkMigrations.ts b/server/scripts/checkMigrations.ts index 8979c1c309..2625b5d297 100644 --- a/server/scripts/checkMigrations.ts +++ b/server/scripts/checkMigrations.ts @@ -1,8 +1,8 @@ import "./bootstrap"; import { styleText } from "node:util"; +import type { Transaction } from "sequelize"; import { isEmpty } from "es-toolkit/compat"; import { toError, errToString } from "@shared/utils/error"; -import { Minute } from "@shared/utils/time"; import env from "@server/env"; import Logger from "@server/logging/Logger"; import AuthenticationProvider from "@server/models/AuthenticationProvider"; @@ -12,8 +12,8 @@ import { migrations, sequelize, } from "@server/storage/database"; +import { LockHelper } from "@server/storage/LockHelper"; import { getArg } from "@server/utils/args"; -import { MutexLock } from "@server/utils/MutexLock"; import { ShutdownHelper } from "@server/utils/ShutdownHelper"; /** @@ -21,11 +21,12 @@ import { ShutdownHelper } from "@server/utils/ShutdownHelper"; * --no-migrate flag was passed in which case the process exits with an error. */ async function checkPendingMigrations() { - let lock; + let lock: Transaction | undefined; try { - lock = await MutexLock.acquire("migrations", 10 * Minute.ms, { - releaseOnShutdown: true, - }); + // The advisory lock is held by this transaction until it ends, while the + // migrations themselves run on other connections. + lock = await sequelize.transaction(); + await LockHelper.acquire(sequelize, "migrations", lock); const pending = await migrations.pending(); if (!isEmpty(pending)) { @@ -58,9 +59,9 @@ async function checkPendingMigrations() { Logger.fatal(styleText("red", message), error); } } finally { - if (lock) { - await MutexLock.release(lock); - } + await lock?.rollback().catch((err) => { + Logger.warn("Failed to release migrations lock", toError(err)); + }); } } diff --git a/server/storage/redis.ts b/server/storage/redis.ts index bbdbf3a215..56b1ca84ce 100644 --- a/server/storage/redis.ts +++ b/server/storage/redis.ts @@ -210,6 +210,7 @@ export default class RedisAdapter extends Redis { private static client: RedisAdapter; private static subscriber: RedisAdapter; + private static ephemeral: RedisAdapter; private static collabClient: RedisAdapter; private static collabSubscriber: RedisAdapter; @@ -232,6 +233,23 @@ export default class RedisAdapter extends Redis { ); } + /** + * A Redis adapter for ephemeral data that can be lost, such as caches, locks + * and rate limits. Commands fail at once when there is no connection, and + * after a short timeout when Redis does not reply, so that callers can + * continue without Redis. + */ + public static get ephemeralClient(): RedisAdapter { + return ( + this.ephemeral || + (this.ephemeral = new this(env.REDIS_URL, { + commandTimeout: env.REDIS_EPHEMERAL_COMMAND_TIMEOUT, + enableOfflineQueue: false, + connectionNameSuffix: "ephemeral", + })) + ); + } + /** * A Redis adapter for collaboration-related operations. */ diff --git a/server/utils/CacheHelper.test.ts b/server/utils/CacheHelper.test.ts index 3b63da7959..25c98968b0 100644 --- a/server/utils/CacheHelper.test.ts +++ b/server/utils/CacheHelper.test.ts @@ -1,5 +1,6 @@ import Redis from "@server/storage/redis"; import { CacheHelper } from "./CacheHelper"; +import { MutexLock } from "./MutexLock"; describe("CacheHelper", () => { beforeEach(async () => { @@ -112,6 +113,35 @@ describe("CacheHelper", () => { }); }); + describe("getDataOrSet when Redis is unavailable", () => { + const { status } = Redis.ephemeralClient; + + afterEach(() => { + Redis.ephemeralClient.status = status; + vi.restoreAllMocks(); + }); + + it("should return the callback result without taking a lock", async () => { + Redis.ephemeralClient.status = "reconnecting"; + vi.spyOn(Redis.ephemeralClient, "get").mockRejectedValue( + new Error("Stream isn't writeable") + ); + vi.spyOn(Redis.ephemeralClient, "set").mockRejectedValue( + new Error("Stream isn't writeable") + ); + const acquire = vi.spyOn(MutexLock, "acquire"); + + const result = await CacheHelper.getDataOrSet( + "test:unavailable", + async () => "value", + 60 + ); + + expect(result).toEqual("value"); + expect(acquire).not.toHaveBeenCalled(); + }); + }); + describe("removeData", () => { it("should remove a single key", async () => { await CacheHelper.setData("test:key", "value", 60); diff --git a/server/utils/CacheHelper.ts b/server/utils/CacheHelper.ts index 3d0b012d0d..99362df470 100644 --- a/server/utils/CacheHelper.ts +++ b/server/utils/CacheHelper.ts @@ -91,12 +91,15 @@ export class CacheHelper { let lock; const lockKey = `lock:${key}`; try { - try { - lock = await MutexLock.acquire(lockKey, lockTimeout, { - retry: MutexLock.cacheRetrySettings, - }); - } catch (err) { - Logger.error(`Could not acquire lock for ${key}`, toError(err)); + // Skip the lock when Redis is unavailable, as it would only add latency. + if (Redis.ephemeralClient.status === "ready") { + try { + lock = await MutexLock.acquire(lockKey, lockTimeout, { + retry: MutexLock.cacheRetrySettings, + }); + } catch (err) { + Logger.error(`Could not acquire lock for ${key}`, toError(err)); + } } cache = await this.getData(key); if (cache !== undefined) { @@ -139,7 +142,7 @@ export class CacheHelper { */ public static async getData(key: string): Promise { try { - const data = await Redis.defaultClient.get(key); + const data = await Redis.ephemeralClient.get(key); if (data !== null) { return JSON.parse(data); } @@ -162,7 +165,7 @@ export class CacheHelper { */ public static async setData(key: string, data: T, expiry?: number) { try { - await Redis.defaultClient.set( + await Redis.ephemeralClient.set( key, JSON.stringify(data), "EX", @@ -181,7 +184,7 @@ export class CacheHelper { */ public static async removeData(key: string) { try { - await Redis.defaultClient.del(key); + await Redis.ephemeralClient.del(key); } catch (err) { Logger.error( `Could not remove cached entry against ${key}`, @@ -194,7 +197,8 @@ export class CacheHelper { * Clears all cache data with the given prefix. Keys are discovered with an * incremental SCAN rather than KEYS and each batch is removed with UNLINK * as the scan progresses, so neither discovery nor deletion blocks the - * Redis event loop and matched keys are never buffered in full. + * Redis event loop and matched keys are never buffered in full. This uses + * the default client, which waits for Redis, as it runs in the background. * * @param prefix Prefix to clear cache data */ diff --git a/server/utils/MutexLock.test.ts b/server/utils/MutexLock.test.ts index 76913122a8..a549a8dca0 100644 --- a/server/utils/MutexLock.test.ts +++ b/server/utils/MutexLock.test.ts @@ -11,7 +11,7 @@ const noRetry = { retry: { retryCount: 0 } }; * Make the next SET apply in Redis and then reject, as when the reply is lost. */ function rejectAfterSet() { - const client = Redis.defaultClient; + const client = Redis.ephemeralClient; const set = client.set.bind(client); return vi .spyOn(client, "set") @@ -27,7 +27,7 @@ function rejectAfterSet() { * @param ms The delay in milliseconds. */ function delaySetReply(ms: number) { - const client = Redis.defaultClient; + const client = Redis.ephemeralClient; const set = client.set.bind(client); return vi .spyOn(client, "set") @@ -40,15 +40,15 @@ function delaySetReply(ms: number) { describe("MutexLock", () => { beforeEach(async () => { - await Redis.defaultClient.flushdb(); + await Redis.ephemeralClient.flushdb(); }); describe("acquire", () => { it("should set the key with an expiry", async () => { const lock = await MutexLock.acquire("test:lock", 1000); - expect(await Redis.defaultClient.get("test:lock")).toEqual(lock.value); - expect(await Redis.defaultClient.pttl("test:lock")).toBeGreaterThan(0); + expect(await Redis.ephemeralClient.get("test:lock")).toEqual(lock.value); + expect(await Redis.ephemeralClient.pttl("test:lock")).toBeGreaterThan(0); expect(lock.expiration).toBeGreaterThan(Date.now()); }); @@ -68,12 +68,14 @@ describe("MutexLock", () => { retry: { retryCount: 10, retryDelay: 20, retryJitter: 0 }, }); - expect(await Redis.defaultClient.get("test:lock")).toEqual(second.value); + expect(await Redis.ephemeralClient.get("test:lock")).toEqual( + second.value + ); }); it("should retry after a Redis error", async () => { const spy = vi - .spyOn(Redis.defaultClient, "set") + .spyOn(Redis.ephemeralClient, "set") .mockRejectedValueOnce(new Error("READONLY")); const lock = await MutexLock.acquire("test:lock", 1000, { @@ -81,12 +83,12 @@ describe("MutexLock", () => { }); spy.mockRestore(); - expect(await Redis.defaultClient.get("test:lock")).toEqual(lock.value); + expect(await Redis.ephemeralClient.get("test:lock")).toEqual(lock.value); }); it("should throw the Redis error when all attempts fail", async () => { const spy = vi - .spyOn(Redis.defaultClient, "set") + .spyOn(Redis.ephemeralClient, "set") .mockRejectedValue(new Error("READONLY")); await expect( @@ -105,7 +107,7 @@ describe("MutexLock", () => { }); spy.mockRestore(); - expect(await Redis.defaultClient.get("test:lock")).toEqual(lock.value); + expect(await Redis.ephemeralClient.get("test:lock")).toEqual(lock.value); }); it("should discard its own lock when giving up after a lost reply", async () => { @@ -116,7 +118,7 @@ describe("MutexLock", () => { ).rejects.toThrow("Connection lost"); spy.mockRestore(); - expect(await Redis.defaultClient.get("test:lock")).toBeNull(); + expect(await Redis.ephemeralClient.get("test:lock")).toBeNull(); }); it("should not return a lock that expired before the reply", async () => { @@ -134,15 +136,15 @@ describe("MutexLock", () => { const lock = await MutexLock.acquire("test:lock", 1000); expect(await MutexLock.release(lock)).toEqual(true); - expect(await Redis.defaultClient.get("test:lock")).toBeNull(); + expect(await Redis.ephemeralClient.get("test:lock")).toBeNull(); }); it("should not delete a lock owned by another holder", async () => { const lock = await MutexLock.acquire("test:lock", 1000); - await Redis.defaultClient.set("test:lock", "other"); + await Redis.ephemeralClient.set("test:lock", "other"); expect(await MutexLock.release(lock)).toEqual(false); - expect(await Redis.defaultClient.get("test:lock")).toEqual("other"); + expect(await Redis.ephemeralClient.get("test:lock")).toEqual("other"); }); }); @@ -156,7 +158,7 @@ describe("MutexLock", () => { }); expect(result).toEqual("done"); - expect(await Redis.defaultClient.get("test:lock")).toBeNull(); + expect(await Redis.ephemeralClient.get("test:lock")).toBeNull(); }); it("should release the lock when the routine throws", async () => { @@ -166,7 +168,7 @@ describe("MutexLock", () => { }) ).rejects.toThrow("failed"); - expect(await Redis.defaultClient.get("test:lock")).toBeNull(); + expect(await Redis.ephemeralClient.get("test:lock")).toBeNull(); }); it("should extend the lock past its initial duration", async () => { @@ -174,7 +176,7 @@ describe("MutexLock", () => { await new Promise((resolve) => setTimeout(resolve, 500)); expect(signal.aborted).toEqual(false); - expect(await Redis.defaultClient.get("test:lock")).not.toBeNull(); + expect(await Redis.ephemeralClient.get("test:lock")).not.toBeNull(); }); }); @@ -192,7 +194,7 @@ describe("MutexLock", () => { }); it("should abort the signal when an extension stays pending past expiry", async () => { - const client = Redis.defaultClient; + const client = Redis.ephemeralClient; await MutexLock.using("test:lock", 200, async (signal) => { const spy = vi.spyOn(client, "eval").mockImplementation(async () => { @@ -209,14 +211,14 @@ describe("MutexLock", () => { it("should abort the signal when the lock is lost", async () => { await MutexLock.using("test:lock", 200, async (signal) => { - await Redis.defaultClient.set("test:lock", "other"); + await Redis.ephemeralClient.set("test:lock", "other"); await new Promise((resolve) => setTimeout(resolve, 200)); expect(signal.aborted).toEqual(true); expect(() => signal.throwIfAborted()).toThrow("was lost"); }); - expect(await Redis.defaultClient.get("test:lock")).toEqual("other"); + expect(await Redis.ephemeralClient.get("test:lock")).toEqual("other"); }); }); @@ -229,7 +231,7 @@ describe("MutexLock", () => { ); expect(result).toEqual("done"); - expect(await Redis.defaultClient.get("test:lock")).toBeNull(); + expect(await Redis.ephemeralClient.get("test:lock")).toBeNull(); }); it("should skip the routine when the lock is held elsewhere", async () => { @@ -263,7 +265,7 @@ describe("MutexLock", () => { spy.mockRestore(); expect(routine).not.toHaveBeenCalled(); - expect(await Redis.defaultClient.get("test:lock")).toBeNull(); + expect(await Redis.ephemeralClient.get("test:lock")).toBeNull(); }); }); }); diff --git a/server/utils/MutexLock.ts b/server/utils/MutexLock.ts index 597b7c9909..f054a5a231 100644 --- a/server/utils/MutexLock.ts +++ b/server/utils/MutexLock.ts @@ -297,7 +297,7 @@ export class MutexLock { value, expiration: Date.now() + timeout, }; - const result = await Redis.defaultClient.set( + const result = await Redis.ephemeralClient.set( resource, value, "PX", @@ -348,7 +348,7 @@ export class MutexLock { */ private static async pexpire(lock: Lock, timeout: number): Promise { const expiration = Date.now() + timeout; - const result = await Redis.defaultClient.eval( + const result = await Redis.ephemeralClient.eval( this.extendScript, 1, lock.resource, @@ -369,7 +369,7 @@ export class MutexLock { * @returns true if the lock was deleted. */ private static async del(lock: Lock): Promise { - const result = await Redis.defaultClient.eval( + const result = await Redis.ephemeralClient.eval( this.releaseScript, 1, lock.resource, diff --git a/server/utils/RateLimiter.ts b/server/utils/RateLimiter.ts index 48f1f59877..044c3c4e28 100644 --- a/server/utils/RateLimiter.ts +++ b/server/utils/RateLimiter.ts @@ -40,7 +40,7 @@ export default class RateLimiter { static get defaultRateLimiter(): RateLimiterRedis { if (!this._defaultRateLimiter) { this._defaultRateLimiter = new RateLimiterRedis({ - storeClient: Redis.defaultClient, + storeClient: Redis.ephemeralClient, points: env.RATE_LIMITER_REQUESTS, duration: env.RATE_LIMITER_DURATION_WINDOW, keyPrefix: this.RATE_LIMITER_REDIS_KEY_PREFIX, @@ -125,7 +125,7 @@ export default class RateLimiter { */ static async cacheUserForToken(token: string, userId: string): Promise { try { - await Redis.defaultClient.set( + await Redis.ephemeralClient.set( this.tokenCacheKey(token), userId, "EX", @@ -145,7 +145,7 @@ export default class RateLimiter { */ static async getCachedUserIdForToken(token: string): Promise { try { - return await Redis.defaultClient.get(this.tokenCacheKey(token)); + return await Redis.ephemeralClient.get(this.tokenCacheKey(token)); } catch (err) { Logger.warn( "Failed to read cached user for rate limiter token", @@ -163,7 +163,7 @@ export default class RateLimiter { */ static async clearCachedToken(token: string): Promise { try { - await Redis.defaultClient.del(this.tokenCacheKey(token)); + await Redis.ephemeralClient.del(this.tokenCacheKey(token)); } catch (err) { Logger.warn("Failed to clear cached rate limiter token", toError(err)); }