mirror of
https://github.com/outline/outline.git
synced 2026-11-21 13:34:02 +03:00
perf: reduce per-item queries on hot paths and add missing indexes
- presentDocuments: batch-load collection deletedAt for archived/deleted documents so trash and archive listings no longer issue one Collection query per document via isCollectionDeleted - documents.search: present results through the batched presentDocuments helper instead of one presentDocument call per result - revisions.list: load collaborators for the whole page in a single User query instead of one per revision, and stop fetching the unused content and text columns when listing without content - TextHelper.attachmentsToSignedUrls: fetch all attachments in one query instead of one findOne per attachment id - BacklinksProcessor: resolve linked documents and existing backlinks in bulk queries and create missing backlinks with a single bulkCreate - NotificationHelper: batch the per-recipient View lookup when filtering comment notification recipients - Add indexes covering hot queries that previously scanned: events (teamId, createdAt) for the audit log, and shares (documentId) / (collectionId) for shareLoader lookups Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01TTaaEHNxBrmBnoVa5iSJV6
This commit is contained in:
8 files changed
+265
-118
No files matched your search
@@ -0,0 +1,35 @@
|
||||
"use strict";
|
||||
|
||||
/** @type {import('sequelize-cli').Migration} */
|
||||
module.exports = {
|
||||
async up(queryInterface) {
|
||||
// Covers the events.list audit log query (teamId filter + createdAt
|
||||
// ordering with pagination).
|
||||
await queryInterface.sequelize.query(
|
||||
'CREATE INDEX CONCURRENTLY IF NOT EXISTS "events_team_id_created_at" ON "events" ("teamId", "createdAt");'
|
||||
);
|
||||
|
||||
// Covers share lookups by document in shareLoader (shares.info flow);
|
||||
// the foreign key column has no index by default.
|
||||
await queryInterface.sequelize.query(
|
||||
'CREATE INDEX CONCURRENTLY IF NOT EXISTS "shares_document_id" ON "shares" ("documentId");'
|
||||
);
|
||||
|
||||
// Covers share lookups by collection in shareLoader (shares.info flow).
|
||||
await queryInterface.sequelize.query(
|
||||
'CREATE INDEX CONCURRENTLY IF NOT EXISTS "shares_collection_id" ON "shares" ("collectionId");'
|
||||
);
|
||||
},
|
||||
|
||||
async down(queryInterface) {
|
||||
await queryInterface.sequelize.query(
|
||||
'DROP INDEX CONCURRENTLY IF EXISTS "shares_collection_id";'
|
||||
);
|
||||
await queryInterface.sequelize.query(
|
||||
'DROP INDEX CONCURRENTLY IF EXISTS "shares_document_id";'
|
||||
);
|
||||
await queryInterface.sequelize.query(
|
||||
'DROP INDEX CONCURRENTLY IF EXISTS "events_team_id_created_at";'
|
||||
);
|
||||
},
|
||||
};
|
||||
@@ -107,26 +107,33 @@ export default class NotificationHelper {
|
||||
});
|
||||
}
|
||||
|
||||
const activeRecipients = recipients.filter(
|
||||
(recipient) => !recipient.isSuspended
|
||||
);
|
||||
|
||||
// If a recipient has viewed the document since the comment was made
|
||||
// then we can avoid sending them a useless notification, yay. Load the
|
||||
// views for all recipients in a single query.
|
||||
const views = activeRecipients.length
|
||||
? await View.findAll({
|
||||
attributes: ["userId"],
|
||||
where: {
|
||||
userId: {
|
||||
[Op.in]: activeRecipients.map((recipient) => recipient.id),
|
||||
},
|
||||
documentId: document.id,
|
||||
updatedAt: {
|
||||
[Op.gt]: comment.createdAt,
|
||||
},
|
||||
},
|
||||
})
|
||||
: [];
|
||||
const viewedUserIds = new Set(views.map((view) => view.userId));
|
||||
|
||||
const filtered: User[] = [];
|
||||
|
||||
for (const recipient of recipients) {
|
||||
if (recipient.isSuspended) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// If this recipient has viewed the document since the comment was made
|
||||
// then we can avoid sending them a useless notification, yay.
|
||||
const view = await View.findOne({
|
||||
where: {
|
||||
userId: recipient.id,
|
||||
documentId: document.id,
|
||||
updatedAt: {
|
||||
[Op.gt]: comment.createdAt,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
if (view) {
|
||||
for (const recipient of activeRecipients) {
|
||||
if (viewedUserIds.has(recipient.id)) {
|
||||
Logger.info(
|
||||
"processor",
|
||||
`suppressing notification to ${recipient.id} because doc viewed`
|
||||
|
||||
@@ -30,27 +30,28 @@ export class TextHelper {
|
||||
expiresIn = 3000
|
||||
) {
|
||||
const attachmentIds = parseAttachmentIds(text);
|
||||
if (attachmentIds.length === 0) {
|
||||
return text;
|
||||
}
|
||||
|
||||
const attachments = await Attachment.findAll({
|
||||
where: {
|
||||
id: attachmentIds,
|
||||
teamId,
|
||||
},
|
||||
});
|
||||
|
||||
await Promise.all(
|
||||
attachmentIds.map(async (id) => {
|
||||
const attachment = await Attachment.findOne({
|
||||
where: {
|
||||
id,
|
||||
teamId,
|
||||
},
|
||||
});
|
||||
attachments.map(async (attachment) => {
|
||||
const signedUrl = await FileStorage.getSignedUrl(
|
||||
attachment.key,
|
||||
expiresIn
|
||||
);
|
||||
|
||||
if (attachment) {
|
||||
const signedUrl = await FileStorage.getSignedUrl(
|
||||
attachment.key,
|
||||
expiresIn
|
||||
);
|
||||
|
||||
text = text.replace(
|
||||
new RegExp(escapeRegExp(attachment.redirectUrl), "g"),
|
||||
signedUrl
|
||||
);
|
||||
}
|
||||
text = text.replace(
|
||||
new RegExp(escapeRegExp(attachment.redirectUrl), "g"),
|
||||
signedUrl
|
||||
);
|
||||
})
|
||||
);
|
||||
return text;
|
||||
|
||||
@@ -2,6 +2,7 @@ import { Op } from "sequelize";
|
||||
import { Hour } from "@shared/utils/time";
|
||||
import { traceFunction } from "@server/logging/tracing";
|
||||
import type { Document } from "@server/models";
|
||||
import Collection from "@server/models/Collection";
|
||||
import FileOperation from "@server/models/FileOperation";
|
||||
import { DocumentHelper } from "@server/models/helpers/DocumentHelper";
|
||||
import type { APIContext } from "@server/types";
|
||||
@@ -136,8 +137,8 @@ export default traceFunction({
|
||||
})(presentDocument);
|
||||
|
||||
/**
|
||||
* Batch-present multiple documents, fetching all related FileOperation records
|
||||
* in a single query instead of one per document.
|
||||
* Batch-present multiple documents, fetching all related FileOperation and
|
||||
* Collection records in a single query each instead of one per document.
|
||||
*
|
||||
* @param ctx the API context.
|
||||
* @param documents the documents to present.
|
||||
@@ -168,6 +169,40 @@ export async function presentDocuments(
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Preload the collections needed by isCollectionDeleted so that deleted
|
||||
// and archived listings don't trigger one Collection query per document.
|
||||
const collectionIds = [
|
||||
...new Set(
|
||||
documents
|
||||
.filter(
|
||||
(doc) =>
|
||||
(doc.deletedAt || doc.archivedAt) &&
|
||||
doc.collectionId &&
|
||||
!doc.collection
|
||||
)
|
||||
.map((doc) => doc.collectionId!)
|
||||
),
|
||||
];
|
||||
|
||||
if (collectionIds.length > 0) {
|
||||
const collections = await Collection.unscoped().findAll({
|
||||
attributes: ["id", "deletedAt"],
|
||||
where: { id: { [Op.in]: collectionIds } },
|
||||
paranoid: false,
|
||||
});
|
||||
const collectionMap = new Map(collections.map((c) => [c.id, c]));
|
||||
|
||||
for (const doc of documents) {
|
||||
if (
|
||||
(doc.deletedAt || doc.archivedAt) &&
|
||||
doc.collectionId &&
|
||||
!doc.collection
|
||||
) {
|
||||
doc.collection = collectionMap.get(doc.collectionId) ?? null;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return Promise.all(
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import parseTitle from "@shared/utils/parseTitle";
|
||||
import { traceFunction } from "@server/logging/tracing";
|
||||
import type { Revision } from "@server/models";
|
||||
import type { Revision, User } from "@server/models";
|
||||
import { DocumentHelper } from "@server/models/helpers/DocumentHelper";
|
||||
import presentUser from "./user";
|
||||
|
||||
@@ -9,6 +9,11 @@ type PresentRevisionOptions = {
|
||||
* Whether to include the document content. Defaults to true.
|
||||
*/
|
||||
includeContent?: boolean;
|
||||
/**
|
||||
* Preloaded collaborators for the revision. When presenting many revisions
|
||||
* pass these to avoid one User query per revision.
|
||||
*/
|
||||
collaborators?: User[];
|
||||
};
|
||||
|
||||
async function presentRevision(
|
||||
@@ -23,7 +28,7 @@ async function presentRevision(
|
||||
const [data, text, collaborators] = await Promise.all([
|
||||
includeContent ? DocumentHelper.toJSON(revision) : undefined,
|
||||
includeContent ? DocumentHelper.toMarkdown(revision) : undefined,
|
||||
revision.collaborators,
|
||||
options.collaborators ?? revision.collaborators,
|
||||
]);
|
||||
|
||||
return {
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import { Op } from "sequelize";
|
||||
import isUUID from "validator/lib/isUUID";
|
||||
import { UrlHelper } from "@shared/utils/UrlHelper";
|
||||
import { Document, Relationship } from "@server/models";
|
||||
import { RelationshipType } from "@server/models/Relationship";
|
||||
import { DocumentHelper } from "@server/models/helpers/DocumentHelper";
|
||||
@@ -22,39 +24,12 @@ export default class BacklinksProcessor extends BaseProcessor {
|
||||
|
||||
// Note: These can be UUID or slugs
|
||||
const linkIds = DocumentHelper.parseDocumentIds(document);
|
||||
|
||||
await Promise.all(
|
||||
linkIds.map(async (linkId) => {
|
||||
const linkedDocument = await Document.findByPk(linkId, {
|
||||
attributes: ["id", "teamId"],
|
||||
});
|
||||
|
||||
if (
|
||||
!linkedDocument ||
|
||||
linkedDocument.id === event.documentId ||
|
||||
linkedDocument.teamId !== document.teamId
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
const existing = await Relationship.findOne({
|
||||
where: {
|
||||
documentId: linkedDocument.id,
|
||||
reverseDocumentId: event.documentId,
|
||||
type: RelationshipType.Backlink,
|
||||
},
|
||||
});
|
||||
|
||||
if (!existing) {
|
||||
await Relationship.create({
|
||||
documentId: linkedDocument.id,
|
||||
reverseDocumentId: event.documentId,
|
||||
userId: document.lastModifiedById,
|
||||
type: RelationshipType.Backlink,
|
||||
});
|
||||
}
|
||||
})
|
||||
const linkedDocuments = await this.findLinkedDocuments(
|
||||
linkIds,
|
||||
document
|
||||
);
|
||||
|
||||
await this.createMissingBacklinks(linkedDocuments, document);
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -70,48 +45,19 @@ export default class BacklinksProcessor extends BaseProcessor {
|
||||
}
|
||||
|
||||
const linkIds = DocumentHelper.parseDocumentIds(document);
|
||||
const linkedDocumentIds: string[] = [];
|
||||
const linkedDocuments = await this.findLinkedDocuments(
|
||||
linkIds,
|
||||
document
|
||||
);
|
||||
|
||||
// create or find existing backlink records for referenced docs
|
||||
await Promise.all(
|
||||
linkIds.map(async (linkId) => {
|
||||
const linkedDocument = await Document.findByPk(linkId, {
|
||||
attributes: ["id", "teamId"],
|
||||
});
|
||||
|
||||
if (
|
||||
!linkedDocument ||
|
||||
linkedDocument.id === event.documentId ||
|
||||
linkedDocument.teamId !== document.teamId
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
const existing = await Relationship.findOne({
|
||||
where: {
|
||||
documentId: linkedDocument.id,
|
||||
reverseDocumentId: event.documentId,
|
||||
type: RelationshipType.Backlink,
|
||||
},
|
||||
});
|
||||
|
||||
if (!existing) {
|
||||
await Relationship.create({
|
||||
documentId: linkedDocument.id,
|
||||
reverseDocumentId: event.documentId,
|
||||
userId: document.lastModifiedById,
|
||||
type: RelationshipType.Backlink,
|
||||
});
|
||||
}
|
||||
linkedDocumentIds.push(linkedDocument.id);
|
||||
})
|
||||
);
|
||||
await this.createMissingBacklinks(linkedDocuments, document);
|
||||
|
||||
// delete any backlinks that no longer exist
|
||||
await Relationship.destroy({
|
||||
where: {
|
||||
documentId: {
|
||||
[Op.notIn]: linkedDocumentIds,
|
||||
[Op.notIn]: linkedDocuments.map((doc) => doc.id),
|
||||
},
|
||||
reverseDocumentId: event.documentId,
|
||||
type: RelationshipType.Backlink,
|
||||
@@ -140,4 +86,88 @@ export default class BacklinksProcessor extends BaseProcessor {
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Load all documents referenced by the given link identifiers in a single
|
||||
* query, restricted to the same team and excluding the source document.
|
||||
*
|
||||
* @param linkIds document identifiers parsed from the document body, either
|
||||
* UUIDs or url slugs.
|
||||
* @param document the source document containing the links.
|
||||
* @returns array of linked documents with minimal attributes.
|
||||
*/
|
||||
private async findLinkedDocuments(linkIds: string[], document: Document) {
|
||||
const ids: string[] = [];
|
||||
const urlIds: string[] = [];
|
||||
|
||||
for (const linkId of linkIds) {
|
||||
if (isUUID(linkId)) {
|
||||
ids.push(linkId);
|
||||
} else {
|
||||
const match = linkId.match(UrlHelper.SLUG_URL_REGEX);
|
||||
if (match) {
|
||||
urlIds.push(match[1]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (ids.length === 0 && urlIds.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return Document.unscoped().findAll({
|
||||
attributes: ["id", "teamId"],
|
||||
where: {
|
||||
teamId: document.teamId,
|
||||
id: {
|
||||
[Op.ne]: document.id,
|
||||
},
|
||||
[Op.or]: [
|
||||
...(ids.length > 0 ? [{ id: { [Op.in]: ids } }] : []),
|
||||
...(urlIds.length > 0 ? [{ urlId: { [Op.in]: urlIds } }] : []),
|
||||
],
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Create backlink relationships from the source document to each linked
|
||||
* document that does not already have one, using bulk queries.
|
||||
*
|
||||
* @param linkedDocuments the documents referenced by the source document.
|
||||
* @param document the source document containing the links.
|
||||
*/
|
||||
private async createMissingBacklinks(
|
||||
linkedDocuments: Document[],
|
||||
document: Document
|
||||
) {
|
||||
if (linkedDocuments.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const existing = await Relationship.findAll({
|
||||
attributes: ["documentId"],
|
||||
where: {
|
||||
documentId: {
|
||||
[Op.in]: linkedDocuments.map((doc) => doc.id),
|
||||
},
|
||||
reverseDocumentId: document.id,
|
||||
type: RelationshipType.Backlink,
|
||||
},
|
||||
});
|
||||
const existingIds = new Set(existing.map((rel) => rel.documentId));
|
||||
|
||||
const missing = linkedDocuments
|
||||
.filter((doc) => !existingIds.has(doc.id))
|
||||
.map((doc) => ({
|
||||
documentId: doc.id,
|
||||
reverseDocumentId: document.id,
|
||||
userId: document.lastModifiedById,
|
||||
type: RelationshipType.Backlink,
|
||||
}));
|
||||
|
||||
if (missing.length > 0) {
|
||||
await Relationship.bulkCreate(missing);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1159,15 +1159,14 @@ router.post(
|
||||
const { results, total } = response;
|
||||
const documents = results.map((result) => result.document);
|
||||
|
||||
const data = await Promise.all(
|
||||
results.map(async (result) => {
|
||||
const document = await presentDocument(ctx, result.document, {
|
||||
isPublic,
|
||||
shareId,
|
||||
});
|
||||
return { ...result, document };
|
||||
})
|
||||
);
|
||||
const presented = await presentDocuments(ctx, documents, {
|
||||
isPublic,
|
||||
shareId,
|
||||
});
|
||||
const data = results.map((result, index) => ({
|
||||
...result,
|
||||
document: presented[index],
|
||||
}));
|
||||
|
||||
// When requesting subsequent pages of search results we don't want to record
|
||||
// duplicate search query records
|
||||
|
||||
@@ -3,6 +3,7 @@ import Router from "koa-router";
|
||||
import contentDisposition from "content-disposition";
|
||||
import { escapeRegExp } from "es-toolkit/compat";
|
||||
import mime from "mime-types";
|
||||
import { Op } from "sequelize";
|
||||
import { errToString } from "@shared/utils/error";
|
||||
import { UserRole } from "@shared/types";
|
||||
import { RevisionHelper } from "@shared/utils/RevisionHelper";
|
||||
@@ -13,7 +14,7 @@ import auth from "@server/middlewares/authentication";
|
||||
import { rateLimiter } from "@server/middlewares/rateLimiter";
|
||||
import { transaction } from "@server/middlewares/transaction";
|
||||
import validate from "@server/middlewares/validate";
|
||||
import { Attachment, Document, Revision } from "@server/models";
|
||||
import { Attachment, Document, Revision, User } from "@server/models";
|
||||
import { DocumentHelper } from "@server/models/helpers/DocumentHelper";
|
||||
import { ProsemirrorHelper } from "@server/models/helpers/ProsemirrorHelper";
|
||||
import { authorize } from "@server/policies";
|
||||
@@ -246,15 +247,49 @@ router.post(
|
||||
where: {
|
||||
documentId: document.id,
|
||||
},
|
||||
attributes: {
|
||||
exclude: ["content", "text"],
|
||||
},
|
||||
order: [[sort, direction]],
|
||||
offset: ctx.state.pagination.offset,
|
||||
limit: ctx.state.pagination.limit,
|
||||
paranoid: false,
|
||||
});
|
||||
|
||||
// Load the collaborators for the whole page of revisions in a single
|
||||
// query rather than one query per revision.
|
||||
const collaboratorIds = new Set<string>();
|
||||
for (const revision of revisions) {
|
||||
for (const collaboratorId of revision.collaboratorIds ?? []) {
|
||||
if (collaboratorId !== revision.userId) {
|
||||
collaboratorIds.add(collaboratorId);
|
||||
}
|
||||
}
|
||||
}
|
||||
const collaborators = collaboratorIds.size
|
||||
? await User.findAll({
|
||||
where: {
|
||||
id: {
|
||||
[Op.in]: [...collaboratorIds],
|
||||
},
|
||||
},
|
||||
paranoid: false,
|
||||
})
|
||||
: [];
|
||||
const collaboratorMap = new Map(collaborators.map((u) => [u.id, u]));
|
||||
|
||||
const data = await Promise.all(
|
||||
revisions.map((revision) =>
|
||||
presentRevision(revision, { includeContent: false })
|
||||
presentRevision(revision, {
|
||||
includeContent: false,
|
||||
collaborators: [
|
||||
revision.user,
|
||||
...(revision.collaboratorIds ?? [])
|
||||
.filter((collaboratorId) => collaboratorId !== revision.userId)
|
||||
.map((collaboratorId) => collaboratorMap.get(collaboratorId))
|
||||
.filter((u): u is User => !!u),
|
||||
],
|
||||
})
|
||||
)
|
||||
);
|
||||
|
||||
|
||||
Reference in new issue
Block a user