mirror of
https://github.com/outline/outline.git
synced 2026-11-22 05:51:43 +03:00
* perf: Index-friendly candidate scan in popularity score task The chunked scan that populates the working table combined "has a non-zero score" and "has recent insights" into a single OR, which forced a scan of every document in the partition with a correlated subquery per row. In production this reached a p95 of 16.9s per chunk on the read replica, which also serves user reads. Query each source separately so both can use an index, then union. Limiting each branch and then the union still yields the globally lowest ids, so keyset pagination is unchanged. Validity is now a join rather than a predicate, with the cursor tracked over candidates so filtered rows can't end the walk early. Adds a partial index on documents(id) where the score is non-zero; the full index on popularityScore was dropped as unused in 20251217161051 and would not help this access pattern. * perf: Drop per-batch COUNT(*) from popularity score task Every batch issued a COUNT(*) over the working table purely to decide whether to continue, doubling the round-trips for a loop that already runs thousands of iterations. processBatch already reports how many rows it handled, so a short batch is sufficient to detect a drained table. setupWorkingTable now returns the row count it inserted, which removes the remaining call site. * perf: Spread popularity scoring across teams over six hours The task partitioned by document id over a 30 minute window, which the cron route expands to one partition per minute. Thirty partitions each took minutes to run, so they piled up and saturated the database for the whole window. Partition by team over six hours instead: 360 slices, one starting each minute, each covering a fraction of the team space. Peak concurrency is bounded by how long a slice takes rather than by the number of partitions. Spreading past the hour breaks the interval gate, which read the wall clock at execution time and so would skip every partition that ran in a later hour -- and worse, would let the next hour's cron schedule a second full run. Cron now records when a run was scheduled and the gate reads that instead. The partial index added for the score walk leads with teamId to match the new access pattern; it is not deployed yet, so it is amended in place. * perf: Re-key popularity partitions to document id, reduce spread to one hour Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * fix: Compute scheduledAt once per cron run Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --------- Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
351 lines
9.8 KiB
TypeScript
351 lines
9.8 KiB
TypeScript
import { subDays } from "date-fns";
|
|
import { Document, DocumentInsight } from "@server/models";
|
|
import { sequelize, sequelizeReadOnly } from "@server/storage/database";
|
|
import { buildDocument, buildTeam } from "@server/test/factories";
|
|
import UpdateDocumentsPopularityScoreTask from "./UpdateDocumentsPopularityScoreTask";
|
|
|
|
const props = {
|
|
limit: 10000,
|
|
partition: {
|
|
partitionIndex: 0,
|
|
partitionCount: 1,
|
|
},
|
|
};
|
|
|
|
vi.setConfig({ testTimeout: 30000 });
|
|
|
|
const dayStr = (d: Date) => d.toISOString().slice(0, 10);
|
|
|
|
describe("UpdateDocumentsPopularityScoreTask", () => {
|
|
let task: UpdateDocumentsPopularityScoreTask;
|
|
|
|
beforeEach(() => {
|
|
task = new UpdateDocumentsPopularityScoreTask();
|
|
vi.spyOn(Date.prototype, "getHours").mockReturnValue(0);
|
|
|
|
// Ensure calculation query sees data created in tests by redirecting to main sequelize instance.
|
|
// We only mock if the instances are different to avoid infinite recursion.
|
|
if (sequelizeReadOnly !== sequelize) {
|
|
vi.spyOn(sequelizeReadOnly, "query").mockImplementation(
|
|
sequelize.query.bind(sequelize)
|
|
);
|
|
}
|
|
});
|
|
|
|
afterEach(() => {
|
|
vi.restoreAllMocks();
|
|
});
|
|
|
|
it("should skip execution if not at a 12-hour interval", async () => {
|
|
vi.spyOn(Date.prototype, "getHours").mockReturnValue(1);
|
|
const team = await buildTeam();
|
|
const document = await buildDocument({
|
|
teamId: team.id,
|
|
publishedAt: new Date(),
|
|
});
|
|
await DocumentInsight.create({
|
|
documentId: document.id,
|
|
teamId: team.id,
|
|
date: dayStr(subDays(new Date(), 1)),
|
|
revisionCount: 1,
|
|
});
|
|
|
|
await task.perform(props);
|
|
|
|
const updatedDocument = await Document.findByPk(document.id);
|
|
expect(updatedDocument?.popularityScore).toBe(0);
|
|
});
|
|
|
|
it("should update popularity score based on revision insights", async () => {
|
|
const team = await buildTeam();
|
|
const document = await buildDocument({
|
|
teamId: team.id,
|
|
publishedAt: new Date(),
|
|
});
|
|
|
|
await DocumentInsight.create({
|
|
documentId: document.id,
|
|
teamId: team.id,
|
|
date: dayStr(subDays(new Date(), 1)),
|
|
revisionCount: 3,
|
|
});
|
|
|
|
await task.perform(props);
|
|
|
|
const updatedDocument = await Document.findByPk(document.id);
|
|
expect(updatedDocument?.popularityScore).toBeGreaterThan(0);
|
|
});
|
|
|
|
it("should update popularity score based on view insights", async () => {
|
|
const team = await buildTeam();
|
|
const document = await buildDocument({
|
|
teamId: team.id,
|
|
publishedAt: new Date(),
|
|
});
|
|
|
|
await DocumentInsight.create({
|
|
documentId: document.id,
|
|
teamId: team.id,
|
|
date: dayStr(new Date()),
|
|
viewCount: 5,
|
|
});
|
|
|
|
await task.perform(props);
|
|
|
|
const updatedDocument = await Document.findByPk(document.id);
|
|
expect(updatedDocument?.popularityScore).toBeGreaterThan(0);
|
|
});
|
|
|
|
it("should update popularity score based on comment insights", async () => {
|
|
const team = await buildTeam();
|
|
const document = await buildDocument({
|
|
teamId: team.id,
|
|
publishedAt: new Date(),
|
|
});
|
|
|
|
await DocumentInsight.create({
|
|
documentId: document.id,
|
|
teamId: team.id,
|
|
date: dayStr(new Date()),
|
|
commentCount: 2,
|
|
});
|
|
|
|
await task.perform(props);
|
|
|
|
const updatedDocument = await Document.findByPk(document.id);
|
|
expect(updatedDocument?.popularityScore).toBeGreaterThan(0);
|
|
});
|
|
|
|
it("should update popularity score based on reaction insights", async () => {
|
|
const team = await buildTeam();
|
|
const document = await buildDocument({
|
|
teamId: team.id,
|
|
publishedAt: new Date(),
|
|
});
|
|
|
|
await DocumentInsight.create({
|
|
documentId: document.id,
|
|
teamId: team.id,
|
|
date: dayStr(new Date()),
|
|
reactionCount: 4,
|
|
});
|
|
|
|
await task.perform(props);
|
|
|
|
const updatedDocument = await Document.findByPk(document.id);
|
|
expect(updatedDocument?.popularityScore).toBeGreaterThan(0);
|
|
});
|
|
|
|
it("should give higher score to more recent activity", async () => {
|
|
const team = await buildTeam();
|
|
|
|
const doc1 = await buildDocument({
|
|
teamId: team.id,
|
|
publishedAt: new Date(),
|
|
});
|
|
const doc2 = await buildDocument({
|
|
teamId: team.id,
|
|
publishedAt: new Date(),
|
|
});
|
|
|
|
await DocumentInsight.create({
|
|
documentId: doc1.id,
|
|
teamId: team.id,
|
|
date: dayStr(subDays(new Date(), 1)),
|
|
revisionCount: 1,
|
|
});
|
|
|
|
await DocumentInsight.create({
|
|
documentId: doc2.id,
|
|
teamId: team.id,
|
|
date: dayStr(subDays(new Date(), 5)),
|
|
revisionCount: 1,
|
|
});
|
|
|
|
await task.perform(props);
|
|
|
|
const updatedDoc1 = await Document.findByPk(doc1.id);
|
|
const updatedDoc2 = await Document.findByPk(doc2.id);
|
|
|
|
expect(updatedDoc1?.popularityScore).toBeGreaterThan(
|
|
updatedDoc2?.popularityScore || 0
|
|
);
|
|
});
|
|
|
|
it("should sum activity across multiple days within the threshold window", async () => {
|
|
const team = await buildTeam();
|
|
const document = await buildDocument({
|
|
teamId: team.id,
|
|
publishedAt: new Date(),
|
|
});
|
|
|
|
await DocumentInsight.create({
|
|
documentId: document.id,
|
|
teamId: team.id,
|
|
date: dayStr(subDays(new Date(), 1)),
|
|
viewCount: 2,
|
|
commentCount: 1,
|
|
});
|
|
await DocumentInsight.create({
|
|
documentId: document.id,
|
|
teamId: team.id,
|
|
date: dayStr(subDays(new Date(), 3)),
|
|
viewCount: 3,
|
|
revisionCount: 1,
|
|
});
|
|
|
|
await task.perform(props);
|
|
|
|
const updatedDocument = await Document.findByPk(document.id);
|
|
expect(updatedDocument?.popularityScore).toBeGreaterThan(0);
|
|
});
|
|
|
|
it("should ignore insights outside the threshold window", async () => {
|
|
const team = await buildTeam();
|
|
const document = await buildDocument({
|
|
teamId: team.id,
|
|
publishedAt: new Date(),
|
|
});
|
|
|
|
await DocumentInsight.create({
|
|
documentId: document.id,
|
|
teamId: team.id,
|
|
date: dayStr(subDays(new Date(), 60)),
|
|
viewCount: 100,
|
|
});
|
|
|
|
await task.perform(props);
|
|
|
|
const updatedDocument = await Document.findByPk(document.id);
|
|
expect(Number(updatedDocument?.popularityScore)).toBe(0);
|
|
});
|
|
|
|
it("should reset score to zero when activity falls outside the window", async () => {
|
|
const team = await buildTeam();
|
|
const document = await buildDocument({
|
|
teamId: team.id,
|
|
publishedAt: new Date(),
|
|
});
|
|
|
|
// Simulate a previously-computed non-zero score with no current activity
|
|
await document.update({ popularityScore: 5.2 });
|
|
|
|
await DocumentInsight.create({
|
|
documentId: document.id,
|
|
teamId: team.id,
|
|
date: dayStr(subDays(new Date(), 60)),
|
|
viewCount: 100,
|
|
});
|
|
|
|
await task.perform(props);
|
|
|
|
const updatedDocument = await Document.findByPk(document.id);
|
|
expect(Number(updatedDocument?.popularityScore)).toBe(0);
|
|
});
|
|
|
|
it("should only process documents within its partition", async () => {
|
|
const team = await buildTeam();
|
|
const document = await buildDocument({
|
|
teamId: team.id,
|
|
publishedAt: new Date(),
|
|
});
|
|
await DocumentInsight.create({
|
|
documentId: document.id,
|
|
teamId: team.id,
|
|
date: dayStr(new Date()),
|
|
viewCount: 5,
|
|
});
|
|
|
|
// Partition 0 of 2 covers the lower half of the UUID space
|
|
const partitionIndex =
|
|
document.id < "80000000-0000-0000-0000-000000000000" ? 0 : 1;
|
|
|
|
await task.perform({
|
|
limit: 10000,
|
|
partition: { partitionIndex: 1 - partitionIndex, partitionCount: 2 },
|
|
});
|
|
|
|
const untouched = await Document.findByPk(document.id);
|
|
expect(Number(untouched?.popularityScore)).toBe(0);
|
|
|
|
await task.perform({
|
|
limit: 10000,
|
|
partition: { partitionIndex, partitionCount: 2 },
|
|
});
|
|
|
|
const updated = await Document.findByPk(document.id);
|
|
expect(Number(updated?.popularityScore)).toBeGreaterThan(0);
|
|
});
|
|
|
|
it("should skip execution based on when the run was scheduled", async () => {
|
|
// Partitions execute hours after they are scheduled, so the interval check
|
|
// must use the scheduled time rather than the current hour.
|
|
vi.spyOn(Date.prototype, "getHours").mockRestore();
|
|
|
|
const team = await buildTeam();
|
|
const document = await buildDocument({
|
|
teamId: team.id,
|
|
publishedAt: new Date(),
|
|
});
|
|
await DocumentInsight.create({
|
|
documentId: document.id,
|
|
teamId: team.id,
|
|
date: dayStr(new Date()),
|
|
viewCount: 5,
|
|
});
|
|
|
|
const offInterval = new Date();
|
|
offInterval.setHours(1, 0, 0, 0);
|
|
|
|
await task.perform({ ...props, scheduledAt: offInterval.getTime() });
|
|
|
|
const untouched = await Document.findByPk(document.id);
|
|
expect(Number(untouched?.popularityScore)).toBe(0);
|
|
|
|
const onInterval = new Date();
|
|
onInterval.setHours(0, 0, 0, 0);
|
|
|
|
await task.perform({ ...props, scheduledAt: onInterval.getTime() });
|
|
|
|
const updated = await Document.findByPk(document.id);
|
|
expect(Number(updated?.popularityScore)).toBeGreaterThan(0);
|
|
});
|
|
|
|
it("should only process published and non-deleted documents", async () => {
|
|
const team = await buildTeam();
|
|
|
|
const draft = await buildDocument({
|
|
teamId: team.id,
|
|
publishedAt: undefined,
|
|
});
|
|
await DocumentInsight.create({
|
|
documentId: draft.id,
|
|
teamId: team.id,
|
|
date: dayStr(new Date()),
|
|
revisionCount: 1,
|
|
});
|
|
|
|
const deleted = await buildDocument({
|
|
teamId: team.id,
|
|
publishedAt: new Date(),
|
|
deletedAt: new Date(),
|
|
});
|
|
await DocumentInsight.create({
|
|
documentId: deleted.id,
|
|
teamId: team.id,
|
|
date: dayStr(new Date()),
|
|
revisionCount: 1,
|
|
});
|
|
|
|
await task.perform(props);
|
|
|
|
const updatedDraft = await Document.unscoped().findByPk(draft.id);
|
|
const updatedDeleted = await Document.unscoped().findByPk(deleted.id, {
|
|
paranoid: false,
|
|
});
|
|
|
|
expect(Number(updatedDraft?.popularityScore)).toEqual(0);
|
|
expect(Number(updatedDeleted?.popularityScore)).toEqual(0);
|
|
});
|
|
});
|