mirror of
https://github.com/outline/outline.git
synced 2026-11-20 21:23:06 +03:00
fix: Keep serving requests when Redis is unavailable (#14457)
* fix: Fail fast on Redis for cache and rate limits, lock migrations in Postgres * fix: Use the fail-fast Redis client for locks * fix: Require a database pool of at least two connections * chore: Rename cache Redis client to ephemeralClient
This commit is contained in:
1 parent
fb2f0e76d1
commit
eff1a43d9b
10 files changed
+121
-51
No files matched your search
+2
-1
@@ -60,7 +60,8 @@ DEFAULT_LANGUAGE=en_US
|
||||
DATABASE_URL=postgres://user:pass@postgres:5432/outline
|
||||
|
||||
# The in-memory database pool per-process settings. Ensure that the pool size that will not exceed
|
||||
# the maximum number of connections allowed by your database. Defaults to 0 and 5.
|
||||
# the maximum number of connections allowed by your database. Defaults to 0 and 5, the maximum
|
||||
# must be at least 2.
|
||||
DATABASE_CONNECTION_POOL_MIN=
|
||||
DATABASE_CONNECTION_POOL_MAX=
|
||||
|
||||
|
||||
+15
-1
@@ -167,10 +167,12 @@ export class Environment {
|
||||
);
|
||||
|
||||
/**
|
||||
* Database connection pool configuration.
|
||||
* Database connection pool configuration. At least two connections are
|
||||
* needed, as some operations query outside of a transaction they hold open.
|
||||
*/
|
||||
@IsNumber()
|
||||
@IsOptional()
|
||||
@Min(2)
|
||||
public DATABASE_CONNECTION_POOL_MAX = this.toOptionalNumber(
|
||||
environment.DATABASE_CONNECTION_POOL_MAX
|
||||
);
|
||||
@@ -232,6 +234,18 @@ export class Environment {
|
||||
10
|
||||
);
|
||||
|
||||
/**
|
||||
* The timeout in milliseconds for a command on the redis connection used for
|
||||
* ephemeral data such as caches, locks and rate limits, after which the
|
||||
* command fails.
|
||||
*/
|
||||
@IsNumber()
|
||||
@Min(100)
|
||||
public REDIS_EPHEMERAL_COMMAND_TIMEOUT = parseInt(
|
||||
environment.REDIS_EPHEMERAL_COMMAND_TIMEOUT || "500",
|
||||
10
|
||||
);
|
||||
|
||||
/**
|
||||
* The fully qualified, external facing domain name of the server.
|
||||
* If not set, will be derived from HEROKU_APP_NAME for Heroku deployments.
|
||||
|
||||
@@ -174,7 +174,7 @@ export function rateLimiter(config: RateLimiterConfig) {
|
||||
duration: 60,
|
||||
points: env.RATE_LIMITER_REQUESTS,
|
||||
keyPrefix: RateLimiter.RATE_LIMITER_REDIS_KEY_PREFIX,
|
||||
storeClient: Redis.defaultClient,
|
||||
storeClient: Redis.ephemeralClient,
|
||||
}
|
||||
)
|
||||
);
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
import "./bootstrap";
|
||||
import { styleText } from "node:util";
|
||||
import type { Transaction } from "sequelize";
|
||||
import { isEmpty } from "es-toolkit/compat";
|
||||
import { toError, errToString } from "@shared/utils/error";
|
||||
import { Minute } from "@shared/utils/time";
|
||||
import env from "@server/env";
|
||||
import Logger from "@server/logging/Logger";
|
||||
import AuthenticationProvider from "@server/models/AuthenticationProvider";
|
||||
@@ -12,8 +12,8 @@ import {
|
||||
migrations,
|
||||
sequelize,
|
||||
} from "@server/storage/database";
|
||||
import { LockHelper } from "@server/storage/LockHelper";
|
||||
import { getArg } from "@server/utils/args";
|
||||
import { MutexLock } from "@server/utils/MutexLock";
|
||||
import { ShutdownHelper } from "@server/utils/ShutdownHelper";
|
||||
|
||||
/**
|
||||
@@ -21,11 +21,12 @@ import { ShutdownHelper } from "@server/utils/ShutdownHelper";
|
||||
* --no-migrate flag was passed in which case the process exits with an error.
|
||||
*/
|
||||
async function checkPendingMigrations() {
|
||||
let lock;
|
||||
let lock: Transaction | undefined;
|
||||
try {
|
||||
lock = await MutexLock.acquire("migrations", 10 * Minute.ms, {
|
||||
releaseOnShutdown: true,
|
||||
});
|
||||
// The advisory lock is held by this transaction until it ends, while the
|
||||
// migrations themselves run on other connections.
|
||||
lock = await sequelize.transaction();
|
||||
await LockHelper.acquire(sequelize, "migrations", lock);
|
||||
|
||||
const pending = await migrations.pending();
|
||||
if (!isEmpty(pending)) {
|
||||
@@ -58,9 +59,9 @@ async function checkPendingMigrations() {
|
||||
Logger.fatal(styleText("red", message), error);
|
||||
}
|
||||
} finally {
|
||||
if (lock) {
|
||||
await MutexLock.release(lock);
|
||||
}
|
||||
await lock?.rollback().catch((err) => {
|
||||
Logger.warn("Failed to release migrations lock", toError(err));
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -210,6 +210,7 @@ export default class RedisAdapter extends Redis {
|
||||
|
||||
private static client: RedisAdapter;
|
||||
private static subscriber: RedisAdapter;
|
||||
private static ephemeral: RedisAdapter;
|
||||
private static collabClient: RedisAdapter;
|
||||
private static collabSubscriber: RedisAdapter;
|
||||
|
||||
@@ -232,6 +233,23 @@ export default class RedisAdapter extends Redis {
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* A Redis adapter for ephemeral data that can be lost, such as caches, locks
|
||||
* and rate limits. Commands fail at once when there is no connection, and
|
||||
* after a short timeout when Redis does not reply, so that callers can
|
||||
* continue without Redis.
|
||||
*/
|
||||
public static get ephemeralClient(): RedisAdapter {
|
||||
return (
|
||||
this.ephemeral ||
|
||||
(this.ephemeral = new this(env.REDIS_URL, {
|
||||
commandTimeout: env.REDIS_EPHEMERAL_COMMAND_TIMEOUT,
|
||||
enableOfflineQueue: false,
|
||||
connectionNameSuffix: "ephemeral",
|
||||
}))
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* A Redis adapter for collaboration-related operations.
|
||||
*/
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import Redis from "@server/storage/redis";
|
||||
import { CacheHelper } from "./CacheHelper";
|
||||
import { MutexLock } from "./MutexLock";
|
||||
|
||||
describe("CacheHelper", () => {
|
||||
beforeEach(async () => {
|
||||
@@ -112,6 +113,35 @@ describe("CacheHelper", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("getDataOrSet when Redis is unavailable", () => {
|
||||
const { status } = Redis.ephemeralClient;
|
||||
|
||||
afterEach(() => {
|
||||
Redis.ephemeralClient.status = status;
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it("should return the callback result without taking a lock", async () => {
|
||||
Redis.ephemeralClient.status = "reconnecting";
|
||||
vi.spyOn(Redis.ephemeralClient, "get").mockRejectedValue(
|
||||
new Error("Stream isn't writeable")
|
||||
);
|
||||
vi.spyOn(Redis.ephemeralClient, "set").mockRejectedValue(
|
||||
new Error("Stream isn't writeable")
|
||||
);
|
||||
const acquire = vi.spyOn(MutexLock, "acquire");
|
||||
|
||||
const result = await CacheHelper.getDataOrSet(
|
||||
"test:unavailable",
|
||||
async () => "value",
|
||||
60
|
||||
);
|
||||
|
||||
expect(result).toEqual("value");
|
||||
expect(acquire).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe("removeData", () => {
|
||||
it("should remove a single key", async () => {
|
||||
await CacheHelper.setData("test:key", "value", 60);
|
||||
|
||||
+14
-10
@@ -91,12 +91,15 @@ export class CacheHelper {
|
||||
let lock;
|
||||
const lockKey = `lock:${key}`;
|
||||
try {
|
||||
try {
|
||||
lock = await MutexLock.acquire(lockKey, lockTimeout, {
|
||||
retry: MutexLock.cacheRetrySettings,
|
||||
});
|
||||
} catch (err) {
|
||||
Logger.error(`Could not acquire lock for ${key}`, toError(err));
|
||||
// Skip the lock when Redis is unavailable, as it would only add latency.
|
||||
if (Redis.ephemeralClient.status === "ready") {
|
||||
try {
|
||||
lock = await MutexLock.acquire(lockKey, lockTimeout, {
|
||||
retry: MutexLock.cacheRetrySettings,
|
||||
});
|
||||
} catch (err) {
|
||||
Logger.error(`Could not acquire lock for ${key}`, toError(err));
|
||||
}
|
||||
}
|
||||
cache = await this.getData<T>(key);
|
||||
if (cache !== undefined) {
|
||||
@@ -139,7 +142,7 @@ export class CacheHelper {
|
||||
*/
|
||||
public static async getData<T>(key: string): Promise<T | undefined> {
|
||||
try {
|
||||
const data = await Redis.defaultClient.get(key);
|
||||
const data = await Redis.ephemeralClient.get(key);
|
||||
if (data !== null) {
|
||||
return JSON.parse(data);
|
||||
}
|
||||
@@ -162,7 +165,7 @@ export class CacheHelper {
|
||||
*/
|
||||
public static async setData<T>(key: string, data: T, expiry?: number) {
|
||||
try {
|
||||
await Redis.defaultClient.set(
|
||||
await Redis.ephemeralClient.set(
|
||||
key,
|
||||
JSON.stringify(data),
|
||||
"EX",
|
||||
@@ -181,7 +184,7 @@ export class CacheHelper {
|
||||
*/
|
||||
public static async removeData(key: string) {
|
||||
try {
|
||||
await Redis.defaultClient.del(key);
|
||||
await Redis.ephemeralClient.del(key);
|
||||
} catch (err) {
|
||||
Logger.error(
|
||||
`Could not remove cached entry against ${key}`,
|
||||
@@ -194,7 +197,8 @@ export class CacheHelper {
|
||||
* Clears all cache data with the given prefix. Keys are discovered with an
|
||||
* incremental SCAN rather than KEYS and each batch is removed with UNLINK
|
||||
* as the scan progresses, so neither discovery nor deletion blocks the
|
||||
* Redis event loop and matched keys are never buffered in full.
|
||||
* Redis event loop and matched keys are never buffered in full. This uses
|
||||
* the default client, which waits for Redis, as it runs in the background.
|
||||
*
|
||||
* @param prefix Prefix to clear cache data
|
||||
*/
|
||||
|
||||
@@ -11,7 +11,7 @@ const noRetry = { retry: { retryCount: 0 } };
|
||||
* Make the next SET apply in Redis and then reject, as when the reply is lost.
|
||||
*/
|
||||
function rejectAfterSet() {
|
||||
const client = Redis.defaultClient;
|
||||
const client = Redis.ephemeralClient;
|
||||
const set = client.set.bind(client);
|
||||
return vi
|
||||
.spyOn(client, "set")
|
||||
@@ -27,7 +27,7 @@ function rejectAfterSet() {
|
||||
* @param ms The delay in milliseconds.
|
||||
*/
|
||||
function delaySetReply(ms: number) {
|
||||
const client = Redis.defaultClient;
|
||||
const client = Redis.ephemeralClient;
|
||||
const set = client.set.bind(client);
|
||||
return vi
|
||||
.spyOn(client, "set")
|
||||
@@ -40,15 +40,15 @@ function delaySetReply(ms: number) {
|
||||
|
||||
describe("MutexLock", () => {
|
||||
beforeEach(async () => {
|
||||
await Redis.defaultClient.flushdb();
|
||||
await Redis.ephemeralClient.flushdb();
|
||||
});
|
||||
|
||||
describe("acquire", () => {
|
||||
it("should set the key with an expiry", async () => {
|
||||
const lock = await MutexLock.acquire("test:lock", 1000);
|
||||
|
||||
expect(await Redis.defaultClient.get("test:lock")).toEqual(lock.value);
|
||||
expect(await Redis.defaultClient.pttl("test:lock")).toBeGreaterThan(0);
|
||||
expect(await Redis.ephemeralClient.get("test:lock")).toEqual(lock.value);
|
||||
expect(await Redis.ephemeralClient.pttl("test:lock")).toBeGreaterThan(0);
|
||||
expect(lock.expiration).toBeGreaterThan(Date.now());
|
||||
});
|
||||
|
||||
@@ -68,12 +68,14 @@ describe("MutexLock", () => {
|
||||
retry: { retryCount: 10, retryDelay: 20, retryJitter: 0 },
|
||||
});
|
||||
|
||||
expect(await Redis.defaultClient.get("test:lock")).toEqual(second.value);
|
||||
expect(await Redis.ephemeralClient.get("test:lock")).toEqual(
|
||||
second.value
|
||||
);
|
||||
});
|
||||
|
||||
it("should retry after a Redis error", async () => {
|
||||
const spy = vi
|
||||
.spyOn(Redis.defaultClient, "set")
|
||||
.spyOn(Redis.ephemeralClient, "set")
|
||||
.mockRejectedValueOnce(new Error("READONLY"));
|
||||
|
||||
const lock = await MutexLock.acquire("test:lock", 1000, {
|
||||
@@ -81,12 +83,12 @@ describe("MutexLock", () => {
|
||||
});
|
||||
spy.mockRestore();
|
||||
|
||||
expect(await Redis.defaultClient.get("test:lock")).toEqual(lock.value);
|
||||
expect(await Redis.ephemeralClient.get("test:lock")).toEqual(lock.value);
|
||||
});
|
||||
|
||||
it("should throw the Redis error when all attempts fail", async () => {
|
||||
const spy = vi
|
||||
.spyOn(Redis.defaultClient, "set")
|
||||
.spyOn(Redis.ephemeralClient, "set")
|
||||
.mockRejectedValue(new Error("READONLY"));
|
||||
|
||||
await expect(
|
||||
@@ -105,7 +107,7 @@ describe("MutexLock", () => {
|
||||
});
|
||||
spy.mockRestore();
|
||||
|
||||
expect(await Redis.defaultClient.get("test:lock")).toEqual(lock.value);
|
||||
expect(await Redis.ephemeralClient.get("test:lock")).toEqual(lock.value);
|
||||
});
|
||||
|
||||
it("should discard its own lock when giving up after a lost reply", async () => {
|
||||
@@ -116,7 +118,7 @@ describe("MutexLock", () => {
|
||||
).rejects.toThrow("Connection lost");
|
||||
spy.mockRestore();
|
||||
|
||||
expect(await Redis.defaultClient.get("test:lock")).toBeNull();
|
||||
expect(await Redis.ephemeralClient.get("test:lock")).toBeNull();
|
||||
});
|
||||
|
||||
it("should not return a lock that expired before the reply", async () => {
|
||||
@@ -134,15 +136,15 @@ describe("MutexLock", () => {
|
||||
const lock = await MutexLock.acquire("test:lock", 1000);
|
||||
|
||||
expect(await MutexLock.release(lock)).toEqual(true);
|
||||
expect(await Redis.defaultClient.get("test:lock")).toBeNull();
|
||||
expect(await Redis.ephemeralClient.get("test:lock")).toBeNull();
|
||||
});
|
||||
|
||||
it("should not delete a lock owned by another holder", async () => {
|
||||
const lock = await MutexLock.acquire("test:lock", 1000);
|
||||
await Redis.defaultClient.set("test:lock", "other");
|
||||
await Redis.ephemeralClient.set("test:lock", "other");
|
||||
|
||||
expect(await MutexLock.release(lock)).toEqual(false);
|
||||
expect(await Redis.defaultClient.get("test:lock")).toEqual("other");
|
||||
expect(await Redis.ephemeralClient.get("test:lock")).toEqual("other");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -156,7 +158,7 @@ describe("MutexLock", () => {
|
||||
});
|
||||
|
||||
expect(result).toEqual("done");
|
||||
expect(await Redis.defaultClient.get("test:lock")).toBeNull();
|
||||
expect(await Redis.ephemeralClient.get("test:lock")).toBeNull();
|
||||
});
|
||||
|
||||
it("should release the lock when the routine throws", async () => {
|
||||
@@ -166,7 +168,7 @@ describe("MutexLock", () => {
|
||||
})
|
||||
).rejects.toThrow("failed");
|
||||
|
||||
expect(await Redis.defaultClient.get("test:lock")).toBeNull();
|
||||
expect(await Redis.ephemeralClient.get("test:lock")).toBeNull();
|
||||
});
|
||||
|
||||
it("should extend the lock past its initial duration", async () => {
|
||||
@@ -174,7 +176,7 @@ describe("MutexLock", () => {
|
||||
await new Promise((resolve) => setTimeout(resolve, 500));
|
||||
|
||||
expect(signal.aborted).toEqual(false);
|
||||
expect(await Redis.defaultClient.get("test:lock")).not.toBeNull();
|
||||
expect(await Redis.ephemeralClient.get("test:lock")).not.toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -192,7 +194,7 @@ describe("MutexLock", () => {
|
||||
});
|
||||
|
||||
it("should abort the signal when an extension stays pending past expiry", async () => {
|
||||
const client = Redis.defaultClient;
|
||||
const client = Redis.ephemeralClient;
|
||||
|
||||
await MutexLock.using("test:lock", 200, async (signal) => {
|
||||
const spy = vi.spyOn(client, "eval").mockImplementation(async () => {
|
||||
@@ -209,14 +211,14 @@ describe("MutexLock", () => {
|
||||
|
||||
it("should abort the signal when the lock is lost", async () => {
|
||||
await MutexLock.using("test:lock", 200, async (signal) => {
|
||||
await Redis.defaultClient.set("test:lock", "other");
|
||||
await Redis.ephemeralClient.set("test:lock", "other");
|
||||
await new Promise((resolve) => setTimeout(resolve, 200));
|
||||
|
||||
expect(signal.aborted).toEqual(true);
|
||||
expect(() => signal.throwIfAborted()).toThrow("was lost");
|
||||
});
|
||||
|
||||
expect(await Redis.defaultClient.get("test:lock")).toEqual("other");
|
||||
expect(await Redis.ephemeralClient.get("test:lock")).toEqual("other");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -229,7 +231,7 @@ describe("MutexLock", () => {
|
||||
);
|
||||
|
||||
expect(result).toEqual("done");
|
||||
expect(await Redis.defaultClient.get("test:lock")).toBeNull();
|
||||
expect(await Redis.ephemeralClient.get("test:lock")).toBeNull();
|
||||
});
|
||||
|
||||
it("should skip the routine when the lock is held elsewhere", async () => {
|
||||
@@ -263,7 +265,7 @@ describe("MutexLock", () => {
|
||||
spy.mockRestore();
|
||||
|
||||
expect(routine).not.toHaveBeenCalled();
|
||||
expect(await Redis.defaultClient.get("test:lock")).toBeNull();
|
||||
expect(await Redis.ephemeralClient.get("test:lock")).toBeNull();
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -297,7 +297,7 @@ export class MutexLock {
|
||||
value,
|
||||
expiration: Date.now() + timeout,
|
||||
};
|
||||
const result = await Redis.defaultClient.set(
|
||||
const result = await Redis.ephemeralClient.set(
|
||||
resource,
|
||||
value,
|
||||
"PX",
|
||||
@@ -348,7 +348,7 @@ export class MutexLock {
|
||||
*/
|
||||
private static async pexpire(lock: Lock, timeout: number): Promise<boolean> {
|
||||
const expiration = Date.now() + timeout;
|
||||
const result = await Redis.defaultClient.eval(
|
||||
const result = await Redis.ephemeralClient.eval(
|
||||
this.extendScript,
|
||||
1,
|
||||
lock.resource,
|
||||
@@ -369,7 +369,7 @@ export class MutexLock {
|
||||
* @returns true if the lock was deleted.
|
||||
*/
|
||||
private static async del(lock: Lock): Promise<boolean> {
|
||||
const result = await Redis.defaultClient.eval(
|
||||
const result = await Redis.ephemeralClient.eval(
|
||||
this.releaseScript,
|
||||
1,
|
||||
lock.resource,
|
||||
|
||||
@@ -40,7 +40,7 @@ export default class RateLimiter {
|
||||
static get defaultRateLimiter(): RateLimiterRedis {
|
||||
if (!this._defaultRateLimiter) {
|
||||
this._defaultRateLimiter = new RateLimiterRedis({
|
||||
storeClient: Redis.defaultClient,
|
||||
storeClient: Redis.ephemeralClient,
|
||||
points: env.RATE_LIMITER_REQUESTS,
|
||||
duration: env.RATE_LIMITER_DURATION_WINDOW,
|
||||
keyPrefix: this.RATE_LIMITER_REDIS_KEY_PREFIX,
|
||||
@@ -125,7 +125,7 @@ export default class RateLimiter {
|
||||
*/
|
||||
static async cacheUserForToken(token: string, userId: string): Promise<void> {
|
||||
try {
|
||||
await Redis.defaultClient.set(
|
||||
await Redis.ephemeralClient.set(
|
||||
this.tokenCacheKey(token),
|
||||
userId,
|
||||
"EX",
|
||||
@@ -145,7 +145,7 @@ export default class RateLimiter {
|
||||
*/
|
||||
static async getCachedUserIdForToken(token: string): Promise<string | null> {
|
||||
try {
|
||||
return await Redis.defaultClient.get(this.tokenCacheKey(token));
|
||||
return await Redis.ephemeralClient.get(this.tokenCacheKey(token));
|
||||
} catch (err) {
|
||||
Logger.warn(
|
||||
"Failed to read cached user for rate limiter token",
|
||||
@@ -163,7 +163,7 @@ export default class RateLimiter {
|
||||
*/
|
||||
static async clearCachedToken(token: string): Promise<void> {
|
||||
try {
|
||||
await Redis.defaultClient.del(this.tokenCacheKey(token));
|
||||
await Redis.ephemeralClient.del(this.tokenCacheKey(token));
|
||||
} catch (err) {
|
||||
Logger.warn("Failed to clear cached rate limiter token", toError(err));
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user