mirror of
https://github.com/outline/outline.git
synced 2026-11-22 05:51:43 +03:00
* fix: Respect per-task cron limit across partitions Adds an optional limit to TaskSchedule that caps the number of items a cron task processes per run. Partitioned runs split the limit across partitions instead of giving each partition the full request limit. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> * fix: Cap cron partitions at the task limit so every UUID range is processed --------- Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
421 lines
13 KiB
TypeScript
421 lines
13 KiB
TypeScript
import { Op } from "sequelize";
|
|
import { Minute } from "@shared/utils/time";
|
|
import type { PartitionInfo } from "./CronTask";
|
|
import { CronTask, TaskInterval } from "./CronTask";
|
|
|
|
type RangeWhere = Record<string, { [Op.gte]: string; [Op.lte]: string }>;
|
|
|
|
// Create a concrete implementation of CronTask for testing
|
|
class TestTask extends CronTask {
|
|
public async perform() {
|
|
// Not used in these tests
|
|
}
|
|
|
|
public get cron() {
|
|
return {
|
|
interval: TaskInterval.Day,
|
|
};
|
|
}
|
|
|
|
public testPartitionWhereClause(
|
|
idField: string,
|
|
partition: PartitionInfo | undefined
|
|
): RangeWhere {
|
|
return this.getPartitionWhereClause(idField, partition) as RangeWhere;
|
|
}
|
|
}
|
|
|
|
describe("CronTask", () => {
|
|
let task: TestTask;
|
|
|
|
beforeEach(() => {
|
|
task = new TestTask();
|
|
});
|
|
|
|
describe("getStaggerDelay", () => {
|
|
it("should return a deterministic delay for the same task name", () => {
|
|
const delay1 = CronTask.getStaggerDelay("TaskA", TaskInterval.Hour);
|
|
const delay2 = CronTask.getStaggerDelay("TaskA", TaskInterval.Hour);
|
|
expect(delay1).toBe(delay2);
|
|
});
|
|
|
|
it("should return different delays for different task names", () => {
|
|
const delayA = CronTask.getStaggerDelay(
|
|
"CleanupDeletedDocumentsTask",
|
|
TaskInterval.Hour
|
|
);
|
|
const delayB = CronTask.getStaggerDelay(
|
|
"CleanupOldEventsTask",
|
|
TaskInterval.Hour
|
|
);
|
|
expect(delayA).not.toBe(delayB);
|
|
});
|
|
|
|
it("should stay within the hourly stagger window (10 minutes)", () => {
|
|
const names = [
|
|
"CleanupDeletedDocumentsTask",
|
|
"CleanupOldEventsTask",
|
|
"CleanupOldNotificationsTask",
|
|
"CleanupDeletedTeamsTask",
|
|
"CleanupExpiredAttachmentsTask",
|
|
"CleanupExpiredFileOperationsTask",
|
|
];
|
|
for (const name of names) {
|
|
const delay = CronTask.getStaggerDelay(name, TaskInterval.Hour);
|
|
expect(delay).toBeGreaterThanOrEqual(0);
|
|
expect(delay).toBeLessThan(10 * Minute.ms);
|
|
}
|
|
});
|
|
|
|
it("should stay within the daily stagger window (30 minutes)", () => {
|
|
const names = [
|
|
"CleanupOAuthAuthorizationCodeTask",
|
|
"CleanupDynamicOAuthClientsTask",
|
|
"CleanupOldImportsTask",
|
|
];
|
|
for (const name of names) {
|
|
const delay = CronTask.getStaggerDelay(name, TaskInterval.Day);
|
|
expect(delay).toBeGreaterThanOrEqual(0);
|
|
expect(delay).toBeLessThan(30 * Minute.ms);
|
|
}
|
|
});
|
|
|
|
it("should stay within an explicit window", () => {
|
|
const names = [
|
|
"CleanupDeletedDocumentsTask",
|
|
"CleanupOldEventsTask",
|
|
"CleanupOldSearchQueriesTask",
|
|
];
|
|
for (const name of names) {
|
|
const delay = CronTask.getStaggerDelay(
|
|
name,
|
|
TaskInterval.Hour,
|
|
Minute.ms
|
|
);
|
|
expect(delay).toBeGreaterThanOrEqual(0);
|
|
expect(delay).toBeLessThan(Minute.ms);
|
|
}
|
|
});
|
|
|
|
it("should distribute delays across the window for real task names", () => {
|
|
const names = [
|
|
"CleanupDeletedDocumentsTask",
|
|
"CleanupOldEventsTask",
|
|
"CleanupOldNotificationsTask",
|
|
"CleanupDeletedTeamsTask",
|
|
"CleanupExpiredAttachmentsTask",
|
|
"CleanupExpiredFileOperationsTask",
|
|
];
|
|
const delays = names.map((name) =>
|
|
CronTask.getStaggerDelay(name, TaskInterval.Hour)
|
|
);
|
|
const unique = new Set(delays);
|
|
expect(unique.size).toBe(delays.length);
|
|
});
|
|
});
|
|
|
|
describe("getPartitionLimit", () => {
|
|
const sumLimits = (limit: number, partitionCount: number) =>
|
|
Array.from({ length: partitionCount }, (_, partitionIndex) =>
|
|
CronTask.getPartitionLimit(limit, { partitionIndex, partitionCount })
|
|
).reduce((sum, value) => sum + value, 0);
|
|
|
|
it("should return the full limit for a single partition", () => {
|
|
expect(
|
|
CronTask.getPartitionLimit(500, {
|
|
partitionIndex: 0,
|
|
partitionCount: 1,
|
|
})
|
|
).toBe(500);
|
|
});
|
|
|
|
it("should never exceed the limit across all partitions", () => {
|
|
expect(sumLimits(500, 30)).toBe(500);
|
|
expect(sumLimits(10, 30)).toBe(10);
|
|
expect(sumLimits(10000, 7)).toBe(10000);
|
|
});
|
|
|
|
it("should give the remainder to the first partitions", () => {
|
|
expect(
|
|
CronTask.getPartitionLimit(500, {
|
|
partitionIndex: 0,
|
|
partitionCount: 30,
|
|
})
|
|
).toBe(17);
|
|
expect(
|
|
CronTask.getPartitionLimit(500, {
|
|
partitionIndex: 29,
|
|
partitionCount: 30,
|
|
})
|
|
).toBe(16);
|
|
});
|
|
});
|
|
|
|
describe("getPartitionWhereClause", () => {
|
|
it("should return empty object when partition is undefined", () => {
|
|
const where = task.testPartitionWhereClause("id", undefined);
|
|
expect(where).toEqual({});
|
|
});
|
|
|
|
it("should generate range WHERE clause for valid partition", () => {
|
|
const where = task.testPartitionWhereClause("id", {
|
|
partitionIndex: 0,
|
|
partitionCount: 3,
|
|
});
|
|
|
|
expect(where).toBeDefined();
|
|
expect(where.id).toBeDefined();
|
|
expect(where.id[Op.gte]).toBeDefined();
|
|
expect(where.id[Op.lte]).toBeDefined();
|
|
});
|
|
|
|
it("should generate correct UUID ranges for 3 partitions", () => {
|
|
const where0 = task.testPartitionWhereClause("id", {
|
|
partitionIndex: 0,
|
|
partitionCount: 3,
|
|
});
|
|
|
|
const where1 = task.testPartitionWhereClause("id", {
|
|
partitionIndex: 1,
|
|
partitionCount: 3,
|
|
});
|
|
|
|
const where2 = task.testPartitionWhereClause("id", {
|
|
partitionIndex: 2,
|
|
partitionCount: 3,
|
|
});
|
|
|
|
// Partition 0: Should start from 00000000
|
|
expect(where0.id[Op.gte]).toBe("00000000-0000-4000-8000-000000000000");
|
|
expect(where0.id[Op.lte]).toBe("55555554-ffff-4fff-bfff-ffffffffffff");
|
|
|
|
// Partition 1: Should start from 55555555
|
|
expect(where1.id[Op.gte]).toBe("55555555-0000-4000-8000-000000000000");
|
|
expect(where1.id[Op.lte]).toBe("aaaaaaa9-ffff-4fff-bfff-ffffffffffff");
|
|
|
|
// Partition 2: Should end at ffffffff
|
|
expect(where2.id[Op.gte]).toBe("aaaaaaaa-0000-4000-8000-000000000000");
|
|
expect(where2.id[Op.lte]).toBe("ffffffff-ffff-4fff-bfff-ffffffffffff");
|
|
});
|
|
|
|
it("should generate correct UUID ranges for 2 partitions", () => {
|
|
const where0 = task.testPartitionWhereClause("id", {
|
|
partitionIndex: 0,
|
|
partitionCount: 2,
|
|
});
|
|
|
|
const where1 = task.testPartitionWhereClause("id", {
|
|
partitionIndex: 1,
|
|
partitionCount: 2,
|
|
});
|
|
|
|
// Partition 0: 0x00000000 to 0x7fffffff
|
|
expect(where0.id[Op.gte]).toBe("00000000-0000-4000-8000-000000000000");
|
|
expect(where0.id[Op.lte]).toBe("7fffffff-ffff-4fff-bfff-ffffffffffff");
|
|
|
|
// Partition 1: 0x80000000 to 0xffffffff
|
|
expect(where1.id[Op.gte]).toBe("80000000-0000-4000-8000-000000000000");
|
|
expect(where1.id[Op.lte]).toBe("ffffffff-ffff-4fff-bfff-ffffffffffff");
|
|
});
|
|
|
|
it("should distribute UUID space evenly", () => {
|
|
const partitionCount = 4;
|
|
const ranges: Array<{ start: string; end: string }> = [];
|
|
|
|
for (let i = 0; i < partitionCount; i++) {
|
|
const where = task.testPartitionWhereClause("id", {
|
|
partitionIndex: i,
|
|
partitionCount,
|
|
});
|
|
ranges.push({
|
|
start: where.id[Op.gte],
|
|
end: where.id[Op.lte],
|
|
});
|
|
}
|
|
|
|
// Check that ranges don't overlap and cover the entire space
|
|
expect(ranges[0].start).toBe("00000000-0000-4000-8000-000000000000");
|
|
expect(ranges[3].end).toBe("ffffffff-ffff-4fff-bfff-ffffffffffff");
|
|
|
|
// Check that each range ends where the next begins (approximately)
|
|
for (let i = 0; i < partitionCount - 1; i++) {
|
|
const currentEnd = ranges[i].end.substring(0, 8);
|
|
const nextStart = ranges[i + 1].start.substring(0, 8);
|
|
|
|
// Convert to numbers to check they're consecutive
|
|
const currentEndNum = parseInt(currentEnd, 16);
|
|
const nextStartNum = parseInt(nextStart, 16);
|
|
|
|
// Should be consecutive or very close
|
|
expect(nextStartNum - currentEndNum).toBeLessThanOrEqual(1);
|
|
}
|
|
});
|
|
|
|
it("should handle single partition (no partitioning)", () => {
|
|
const where = task.testPartitionWhereClause("id", {
|
|
partitionIndex: 0,
|
|
partitionCount: 1,
|
|
});
|
|
|
|
// Should cover entire UUID space
|
|
expect(where.id[Op.gte]).toBe("00000000-0000-4000-8000-000000000000");
|
|
expect(where.id[Op.lte]).toBe("ffffffff-ffff-4fff-bfff-ffffffffffff");
|
|
});
|
|
|
|
it("should throw error for invalid partition info", () => {
|
|
expect(() => {
|
|
task.testPartitionWhereClause("id", {
|
|
partitionIndex: -1,
|
|
partitionCount: 3,
|
|
});
|
|
}).toThrow("Invalid partition info: index -1, count 3");
|
|
|
|
expect(() => {
|
|
task.testPartitionWhereClause("id", {
|
|
partitionIndex: 3,
|
|
partitionCount: 3,
|
|
});
|
|
}).toThrow("Invalid partition info: index 3, count 3");
|
|
|
|
expect(() => {
|
|
task.testPartitionWhereClause("id", {
|
|
partitionIndex: 0,
|
|
partitionCount: 0,
|
|
});
|
|
}).toThrow("Invalid partition info: index 0, count 0");
|
|
});
|
|
|
|
it("should work with different field names", () => {
|
|
const where1 = task.testPartitionWhereClause("id", {
|
|
partitionIndex: 0,
|
|
partitionCount: 2,
|
|
});
|
|
|
|
const where2 = task.testPartitionWhereClause("documentId", {
|
|
partitionIndex: 0,
|
|
partitionCount: 2,
|
|
});
|
|
|
|
expect(where1.id).toBeDefined();
|
|
expect(where1.documentId).toBeUndefined();
|
|
expect(where2.documentId).toBeDefined();
|
|
expect(where2.id).toBeUndefined();
|
|
});
|
|
|
|
it("should handle large partition counts efficiently", () => {
|
|
const partitionCount = 100;
|
|
const ranges: Array<{ start: string; end: string }> = [];
|
|
|
|
for (let i = 0; i < partitionCount; i++) {
|
|
const where = task.testPartitionWhereClause("id", {
|
|
partitionIndex: i,
|
|
partitionCount,
|
|
});
|
|
ranges.push({
|
|
start: where.id[Op.gte],
|
|
end: where.id[Op.lte],
|
|
});
|
|
}
|
|
|
|
// First partition should start at 00000000
|
|
expect(ranges[0].start).toBe("00000000-0000-4000-8000-000000000000");
|
|
// Last partition should end at ffffffff
|
|
expect(ranges[99].end).toBe("ffffffff-ffff-4fff-bfff-ffffffffffff");
|
|
|
|
// Each partition should have a unique range
|
|
const startValues = new Set(ranges.map((r) => r.start));
|
|
expect(startValues.size).toBe(100);
|
|
});
|
|
|
|
it("should calculate correct hex values for partition boundaries", () => {
|
|
// Test specific calculations
|
|
const where = task.testPartitionWhereClause("id", {
|
|
partitionIndex: 1,
|
|
partitionCount: 16, // 16 partitions = 0x10000000 per partition
|
|
});
|
|
|
|
// Partition 1 should be from 0x10000000 to 0x1fffffff
|
|
expect(where.id[Op.gte]).toBe("10000000-0000-4000-8000-000000000000");
|
|
expect(where.id[Op.lte]).toBe("1fffffff-ffff-4fff-bfff-ffffffffffff");
|
|
});
|
|
|
|
it("should ensure all UUIDs map to exactly one partition", () => {
|
|
const testUuids = [
|
|
"00000000-0000-4000-8000-000000000000", // Min UUID
|
|
"12345678-9abc-4ef0-9234-567890abcdef",
|
|
"55555555-5555-4555-9555-555555555555",
|
|
"87654321-fedc-4a98-b654-321098765432",
|
|
"aaaaaaaa-aaaa-4aaa-aaaa-aaaaaaaaaaaa",
|
|
"deadbeef-cafe-4abe-aeed-dec0debac1e5",
|
|
"ffffffff-ffff-4fff-bfff-ffffffffffff", // Max UUID
|
|
];
|
|
|
|
const partitionCount = 3;
|
|
|
|
for (const uuid of testUuids) {
|
|
let matchCount = 0;
|
|
let matchedPartition = -1;
|
|
|
|
for (let i = 0; i < partitionCount; i++) {
|
|
const where = task.testPartitionWhereClause("id", {
|
|
partitionIndex: i,
|
|
partitionCount,
|
|
});
|
|
|
|
const startUuid = where.id[Op.gte];
|
|
const endUuid = where.id[Op.lte];
|
|
|
|
// Check if UUID falls within this partition's range
|
|
if (uuid >= startUuid && uuid <= endUuid) {
|
|
matchCount++;
|
|
matchedPartition = i;
|
|
}
|
|
}
|
|
|
|
// Each UUID should match exactly one partition
|
|
expect(matchCount).toBe(1);
|
|
expect(matchedPartition).toBeGreaterThanOrEqual(0);
|
|
expect(matchedPartition).toBeLessThan(partitionCount);
|
|
}
|
|
});
|
|
|
|
it("should generate non-overlapping ranges for any partition count", () => {
|
|
const testCounts = [2, 3, 5, 7, 10, 16, 32];
|
|
|
|
for (const partitionCount of testCounts) {
|
|
const ranges: Array<{ start: string; end: string }> = [];
|
|
|
|
// Get all partition ranges
|
|
for (let i = 0; i < partitionCount; i++) {
|
|
const where = task.testPartitionWhereClause("id", {
|
|
partitionIndex: i,
|
|
partitionCount,
|
|
});
|
|
ranges.push({
|
|
start: where.id[Op.gte],
|
|
end: where.id[Op.lte],
|
|
});
|
|
}
|
|
|
|
// Verify no gaps between consecutive partitions
|
|
for (let i = 0; i < partitionCount - 1; i++) {
|
|
const currentEnd = ranges[i].end.substring(0, 8);
|
|
const nextStart = ranges[i + 1].start.substring(0, 8);
|
|
|
|
const currentEndNum = parseInt(currentEnd, 16);
|
|
const nextStartNum = parseInt(nextStart, 16);
|
|
|
|
// Next partition should start exactly where current ends + 1
|
|
expect(nextStartNum).toBe(currentEndNum + 1);
|
|
}
|
|
|
|
// Verify coverage of entire UUID space
|
|
expect(ranges[0].start).toBe("00000000-0000-4000-8000-000000000000");
|
|
expect(ranges[partitionCount - 1].end).toBe(
|
|
"ffffffff-ffff-4fff-bfff-ffffffffffff"
|
|
);
|
|
}
|
|
});
|
|
});
|
|
});
|