mirror of
https://github.com/outline/outline.git
synced 2026-11-22 05:51:43 +03:00
perf: Use SCAN and batched UNLINK instead of KEYS in CacheHelper.clearData (#12934)
* perf: Use SCAN and batched UNLINK instead of KEYS in CacheHelper.clearData The KEYS command blocks the Redis event loop while it walks the entire keyspace, which stalls all other clients on large installations — notably on startup where the whole unfurl cache prefix is cleared. Replace it with incremental SCAN iteration and delete the matched keys in batches with UNLINK, which reclaims memory asynchronously, so neither discovery nor deletion blocks the server. Adds test coverage for CacheHelper including multi-page prefix clears. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_015K9AAwaGTLVQZnp7uv5Ebj * fix: Address review comments on CacheHelper.clearData Unlink each SCAN batch as the scan progresses instead of buffering all matching keys in memory, repeating until a full pass matches nothing so implementations with unstable scan cursors still clear every key. Wrap the operation in try/catch with logging for consistency with the other CacheHelper methods, and mark constants as readonly. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_015K9AAwaGTLVQZnp7uv5Ebj --------- Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
1 parent
8ea04c3bd4
commit
db1d93f9b5
2 files changed
+117
-7
No files matched your search
@@ -0,0 +1,70 @@
|
||||
import Redis from "@server/storage/redis";
|
||||
import { CacheHelper } from "./CacheHelper";
|
||||
|
||||
describe("CacheHelper", () => {
|
||||
beforeEach(async () => {
|
||||
await Redis.defaultClient.flushdb();
|
||||
});
|
||||
|
||||
describe("setData and getData", () => {
|
||||
it("should round-trip a value", async () => {
|
||||
await CacheHelper.setData("test:key", { foo: "bar" }, 60);
|
||||
const result = await CacheHelper.getData<{ foo: string }>("test:key");
|
||||
expect(result).toEqual({ foo: "bar" });
|
||||
});
|
||||
|
||||
it("should return undefined for a missing key", async () => {
|
||||
const result = await CacheHelper.getData("test:missing");
|
||||
expect(result).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe("removeData", () => {
|
||||
it("should remove a single key", async () => {
|
||||
await CacheHelper.setData("test:key", "value", 60);
|
||||
await CacheHelper.removeData("test:key");
|
||||
const result = await CacheHelper.getData("test:key");
|
||||
expect(result).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe("clearData", () => {
|
||||
it("should remove all keys matching the prefix", async () => {
|
||||
await CacheHelper.setData("unfurl:team-1:https://a.com", "a", 60);
|
||||
await CacheHelper.setData("unfurl:team-1:https://b.com", "b", 60);
|
||||
await CacheHelper.setData("unfurl:team-2:https://c.com", "c", 60);
|
||||
|
||||
await CacheHelper.clearData("unfurl:team-1");
|
||||
|
||||
expect(
|
||||
await CacheHelper.getData("unfurl:team-1:https://a.com")
|
||||
).toBeUndefined();
|
||||
expect(
|
||||
await CacheHelper.getData("unfurl:team-1:https://b.com")
|
||||
).toBeUndefined();
|
||||
expect(await CacheHelper.getData("unfurl:team-2:https://c.com")).toEqual(
|
||||
"c"
|
||||
);
|
||||
});
|
||||
|
||||
it("should resolve when no keys match the prefix", async () => {
|
||||
await expect(CacheHelper.clearData("test:nothing")).resolves.toBe(
|
||||
undefined
|
||||
);
|
||||
});
|
||||
|
||||
it("should remove more keys than a single scan page", async () => {
|
||||
const count = 2500;
|
||||
const pipeline = Redis.defaultClient.pipeline();
|
||||
for (let i = 0; i < count; i++) {
|
||||
pipeline.set(`test:bulk:${i}`, "value");
|
||||
}
|
||||
await pipeline.exec();
|
||||
|
||||
await CacheHelper.clearData("test:bulk:");
|
||||
|
||||
const remaining = await Redis.defaultClient.keys("test:bulk:*");
|
||||
expect(remaining.length).toEqual(0);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -147,17 +147,57 @@ export class CacheHelper {
|
||||
}
|
||||
|
||||
/**
|
||||
* Clears all cache data with the given prefix
|
||||
* Clears all cache data with the given prefix. Keys are discovered with an
|
||||
* incremental SCAN rather than KEYS and each batch is removed with UNLINK
|
||||
* as the scan progresses, so neither discovery nor deletion blocks the
|
||||
* Redis event loop and matched keys are never buffered in full.
|
||||
*
|
||||
* @param prefix Prefix to clear cache data
|
||||
*/
|
||||
public static async clearData(prefix: string) {
|
||||
const keys = await Redis.defaultClient.keys(`${prefix}*`);
|
||||
const match = `${prefix}*`;
|
||||
|
||||
await Promise.all(
|
||||
keys.map(async (key) => {
|
||||
await Redis.defaultClient.del(key);
|
||||
})
|
||||
);
|
||||
try {
|
||||
// Deleting keys while a scan is in progress can skip keys in some
|
||||
// Redis-compatible implementations, so repeat until a full pass finds
|
||||
// nothing to delete. With real Redis the final pass is a single
|
||||
// confirming sweep.
|
||||
for (let pass = 0; pass < CacheHelper.maxClearPasses; pass++) {
|
||||
let deleted = 0;
|
||||
let cursor = "0";
|
||||
|
||||
do {
|
||||
const [nextCursor, keys] = await Redis.defaultClient.scan(
|
||||
cursor,
|
||||
"MATCH",
|
||||
match,
|
||||
"COUNT",
|
||||
CacheHelper.scanPageSize
|
||||
);
|
||||
cursor = nextCursor;
|
||||
|
||||
if (keys.length > 0) {
|
||||
await Redis.defaultClient.unlink(...keys);
|
||||
deleted += keys.length;
|
||||
}
|
||||
} while (cursor !== "0");
|
||||
|
||||
if (deleted === 0) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
Logger.error(
|
||||
`Could not clear cached data for prefix ${prefix}`,
|
||||
toError(err)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Number of keys to request per SCAN iteration when clearing by prefix
|
||||
private static readonly scanPageSize = 1000;
|
||||
|
||||
// Upper bound on full scan+delete passes in clearData, guarding against a
|
||||
// pathological loop if matching keys are written as fast as they are cleared
|
||||
private static readonly maxClearPasses = 10;
|
||||
}
|
||||
Reference in new issue
Block a user