diff --git a/server/migrations/20260705000000-add-query-performance-indexes.js b/server/migrations/20260705000000-add-query-performance-indexes.js new file mode 100644 index 0000000000..62f6085dd2 --- /dev/null +++ b/server/migrations/20260705000000-add-query-performance-indexes.js @@ -0,0 +1,35 @@ +"use strict"; + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + async up(queryInterface) { + // Covers the events.list audit log query (teamId filter + createdAt + // ordering with pagination). + await queryInterface.sequelize.query( + 'CREATE INDEX CONCURRENTLY IF NOT EXISTS "events_team_id_created_at" ON "events" ("teamId", "createdAt");' + ); + + // Covers share lookups by document in shareLoader (shares.info flow); + // the foreign key column has no index by default. + await queryInterface.sequelize.query( + 'CREATE INDEX CONCURRENTLY IF NOT EXISTS "shares_document_id" ON "shares" ("documentId");' + ); + + // Covers share lookups by collection in shareLoader (shares.info flow). + await queryInterface.sequelize.query( + 'CREATE INDEX CONCURRENTLY IF NOT EXISTS "shares_collection_id" ON "shares" ("collectionId");' + ); + }, + + async down(queryInterface) { + await queryInterface.sequelize.query( + 'DROP INDEX CONCURRENTLY IF EXISTS "shares_collection_id";' + ); + await queryInterface.sequelize.query( + 'DROP INDEX CONCURRENTLY IF EXISTS "shares_document_id";' + ); + await queryInterface.sequelize.query( + 'DROP INDEX CONCURRENTLY IF EXISTS "events_team_id_created_at";' + ); + }, +}; diff --git a/server/models/helpers/NotificationHelper.ts b/server/models/helpers/NotificationHelper.ts index 951732ad0d..555bf87a3b 100644 --- a/server/models/helpers/NotificationHelper.ts +++ b/server/models/helpers/NotificationHelper.ts @@ -107,26 +107,33 @@ export default class NotificationHelper { }); } + const activeRecipients = recipients.filter( + (recipient) => !recipient.isSuspended + ); + + // If a recipient has viewed the document since the comment was made + // then we can avoid sending them a useless notification, yay. Load the + // views for all recipients in a single query. + const views = activeRecipients.length + ? await View.findAll({ + attributes: ["userId"], + where: { + userId: { + [Op.in]: activeRecipients.map((recipient) => recipient.id), + }, + documentId: document.id, + updatedAt: { + [Op.gt]: comment.createdAt, + }, + }, + }) + : []; + const viewedUserIds = new Set(views.map((view) => view.userId)); + const filtered: User[] = []; - for (const recipient of recipients) { - if (recipient.isSuspended) { - continue; - } - - // If this recipient has viewed the document since the comment was made - // then we can avoid sending them a useless notification, yay. - const view = await View.findOne({ - where: { - userId: recipient.id, - documentId: document.id, - updatedAt: { - [Op.gt]: comment.createdAt, - }, - }, - }); - - if (view) { + for (const recipient of activeRecipients) { + if (viewedUserIds.has(recipient.id)) { Logger.info( "processor", `suppressing notification to ${recipient.id} because doc viewed` diff --git a/server/models/helpers/TextHelper.ts b/server/models/helpers/TextHelper.ts index ca21907758..3f4620ebf8 100644 --- a/server/models/helpers/TextHelper.ts +++ b/server/models/helpers/TextHelper.ts @@ -30,27 +30,28 @@ export class TextHelper { expiresIn = 3000 ) { const attachmentIds = parseAttachmentIds(text); + if (attachmentIds.length === 0) { + return text; + } + + const attachments = await Attachment.findAll({ + where: { + id: attachmentIds, + teamId, + }, + }); await Promise.all( - attachmentIds.map(async (id) => { - const attachment = await Attachment.findOne({ - where: { - id, - teamId, - }, - }); + attachments.map(async (attachment) => { + const signedUrl = await FileStorage.getSignedUrl( + attachment.key, + expiresIn + ); - if (attachment) { - const signedUrl = await FileStorage.getSignedUrl( - attachment.key, - expiresIn - ); - - text = text.replace( - new RegExp(escapeRegExp(attachment.redirectUrl), "g"), - signedUrl - ); - } + text = text.replace( + new RegExp(escapeRegExp(attachment.redirectUrl), "g"), + signedUrl + ); }) ); return text; diff --git a/server/presenters/document.ts b/server/presenters/document.ts index 6f93f7fa42..e8f531efac 100644 --- a/server/presenters/document.ts +++ b/server/presenters/document.ts @@ -2,6 +2,7 @@ import { Op } from "sequelize"; import { Hour } from "@shared/utils/time"; import { traceFunction } from "@server/logging/tracing"; import type { Document } from "@server/models"; +import Collection from "@server/models/Collection"; import FileOperation from "@server/models/FileOperation"; import { DocumentHelper } from "@server/models/helpers/DocumentHelper"; import type { APIContext } from "@server/types"; @@ -136,8 +137,8 @@ export default traceFunction({ })(presentDocument); /** - * Batch-present multiple documents, fetching all related FileOperation records - * in a single query instead of one per document. + * Batch-present multiple documents, fetching all related FileOperation and + * Collection records in a single query each instead of one per document. * * @param ctx the API context. * @param documents the documents to present. @@ -168,6 +169,40 @@ export async function presentDocuments( } } } + + // Preload the collections needed by isCollectionDeleted so that deleted + // and archived listings don't trigger one Collection query per document. + const collectionIds = [ + ...new Set( + documents + .filter( + (doc) => + (doc.deletedAt || doc.archivedAt) && + doc.collectionId && + !doc.collection + ) + .map((doc) => doc.collectionId!) + ), + ]; + + if (collectionIds.length > 0) { + const collections = await Collection.unscoped().findAll({ + attributes: ["id", "deletedAt"], + where: { id: { [Op.in]: collectionIds } }, + paranoid: false, + }); + const collectionMap = new Map(collections.map((c) => [c.id, c])); + + for (const doc of documents) { + if ( + (doc.deletedAt || doc.archivedAt) && + doc.collectionId && + !doc.collection + ) { + doc.collection = collectionMap.get(doc.collectionId) ?? null; + } + } + } } return Promise.all( diff --git a/server/presenters/revision.ts b/server/presenters/revision.ts index eaf88f3954..c4a9b8bfd4 100644 --- a/server/presenters/revision.ts +++ b/server/presenters/revision.ts @@ -1,6 +1,6 @@ import parseTitle from "@shared/utils/parseTitle"; import { traceFunction } from "@server/logging/tracing"; -import type { Revision } from "@server/models"; +import type { Revision, User } from "@server/models"; import { DocumentHelper } from "@server/models/helpers/DocumentHelper"; import presentUser from "./user"; @@ -9,6 +9,11 @@ type PresentRevisionOptions = { * Whether to include the document content. Defaults to true. */ includeContent?: boolean; + /** + * Preloaded collaborators for the revision. When presenting many revisions + * pass these to avoid one User query per revision. + */ + collaborators?: User[]; }; async function presentRevision( @@ -23,7 +28,7 @@ async function presentRevision( const [data, text, collaborators] = await Promise.all([ includeContent ? DocumentHelper.toJSON(revision) : undefined, includeContent ? DocumentHelper.toMarkdown(revision) : undefined, - revision.collaborators, + options.collaborators ?? revision.collaborators, ]); return { diff --git a/server/queues/processors/BacklinksProcessor.ts b/server/queues/processors/BacklinksProcessor.ts index e3d8edb5fe..caa722e8fb 100644 --- a/server/queues/processors/BacklinksProcessor.ts +++ b/server/queues/processors/BacklinksProcessor.ts @@ -1,4 +1,6 @@ import { Op } from "sequelize"; +import isUUID from "validator/lib/isUUID"; +import { UrlHelper } from "@shared/utils/UrlHelper"; import { Document, Relationship } from "@server/models"; import { RelationshipType } from "@server/models/Relationship"; import { DocumentHelper } from "@server/models/helpers/DocumentHelper"; @@ -22,39 +24,12 @@ export default class BacklinksProcessor extends BaseProcessor { // Note: These can be UUID or slugs const linkIds = DocumentHelper.parseDocumentIds(document); - - await Promise.all( - linkIds.map(async (linkId) => { - const linkedDocument = await Document.findByPk(linkId, { - attributes: ["id", "teamId"], - }); - - if ( - !linkedDocument || - linkedDocument.id === event.documentId || - linkedDocument.teamId !== document.teamId - ) { - return; - } - - const existing = await Relationship.findOne({ - where: { - documentId: linkedDocument.id, - reverseDocumentId: event.documentId, - type: RelationshipType.Backlink, - }, - }); - - if (!existing) { - await Relationship.create({ - documentId: linkedDocument.id, - reverseDocumentId: event.documentId, - userId: document.lastModifiedById, - type: RelationshipType.Backlink, - }); - } - }) + const linkedDocuments = await this.findLinkedDocuments( + linkIds, + document ); + + await this.createMissingBacklinks(linkedDocuments, document); break; } @@ -70,48 +45,19 @@ export default class BacklinksProcessor extends BaseProcessor { } const linkIds = DocumentHelper.parseDocumentIds(document); - const linkedDocumentIds: string[] = []; + const linkedDocuments = await this.findLinkedDocuments( + linkIds, + document + ); // create or find existing backlink records for referenced docs - await Promise.all( - linkIds.map(async (linkId) => { - const linkedDocument = await Document.findByPk(linkId, { - attributes: ["id", "teamId"], - }); - - if ( - !linkedDocument || - linkedDocument.id === event.documentId || - linkedDocument.teamId !== document.teamId - ) { - return; - } - - const existing = await Relationship.findOne({ - where: { - documentId: linkedDocument.id, - reverseDocumentId: event.documentId, - type: RelationshipType.Backlink, - }, - }); - - if (!existing) { - await Relationship.create({ - documentId: linkedDocument.id, - reverseDocumentId: event.documentId, - userId: document.lastModifiedById, - type: RelationshipType.Backlink, - }); - } - linkedDocumentIds.push(linkedDocument.id); - }) - ); + await this.createMissingBacklinks(linkedDocuments, document); // delete any backlinks that no longer exist await Relationship.destroy({ where: { documentId: { - [Op.notIn]: linkedDocumentIds, + [Op.notIn]: linkedDocuments.map((doc) => doc.id), }, reverseDocumentId: event.documentId, type: RelationshipType.Backlink, @@ -140,4 +86,88 @@ export default class BacklinksProcessor extends BaseProcessor { default: } } + + /** + * Load all documents referenced by the given link identifiers in a single + * query, restricted to the same team and excluding the source document. + * + * @param linkIds document identifiers parsed from the document body, either + * UUIDs or url slugs. + * @param document the source document containing the links. + * @returns array of linked documents with minimal attributes. + */ + private async findLinkedDocuments(linkIds: string[], document: Document) { + const ids: string[] = []; + const urlIds: string[] = []; + + for (const linkId of linkIds) { + if (isUUID(linkId)) { + ids.push(linkId); + } else { + const match = linkId.match(UrlHelper.SLUG_URL_REGEX); + if (match) { + urlIds.push(match[1]); + } + } + } + + if (ids.length === 0 && urlIds.length === 0) { + return []; + } + + return Document.unscoped().findAll({ + attributes: ["id", "teamId"], + where: { + teamId: document.teamId, + id: { + [Op.ne]: document.id, + }, + [Op.or]: [ + ...(ids.length > 0 ? [{ id: { [Op.in]: ids } }] : []), + ...(urlIds.length > 0 ? [{ urlId: { [Op.in]: urlIds } }] : []), + ], + }, + }); + } + + /** + * Create backlink relationships from the source document to each linked + * document that does not already have one, using bulk queries. + * + * @param linkedDocuments the documents referenced by the source document. + * @param document the source document containing the links. + */ + private async createMissingBacklinks( + linkedDocuments: Document[], + document: Document + ) { + if (linkedDocuments.length === 0) { + return; + } + + const existing = await Relationship.findAll({ + attributes: ["documentId"], + where: { + documentId: { + [Op.in]: linkedDocuments.map((doc) => doc.id), + }, + reverseDocumentId: document.id, + type: RelationshipType.Backlink, + }, + }); + const existingIds = new Set(existing.map((rel) => rel.documentId)); + + const missing = linkedDocuments + .filter((doc) => !existingIds.has(doc.id)) + .map((doc) => ({ + documentId: doc.id, + reverseDocumentId: document.id, + userId: document.lastModifiedById, + type: RelationshipType.Backlink, + })); + + if (missing.length > 0) { + await Relationship.bulkCreate(missing); + } + } } diff --git a/server/routes/api/documents/documents.ts b/server/routes/api/documents/documents.ts index a2ca5521de..3441c9e01d 100644 --- a/server/routes/api/documents/documents.ts +++ b/server/routes/api/documents/documents.ts @@ -1159,15 +1159,14 @@ router.post( const { results, total } = response; const documents = results.map((result) => result.document); - const data = await Promise.all( - results.map(async (result) => { - const document = await presentDocument(ctx, result.document, { - isPublic, - shareId, - }); - return { ...result, document }; - }) - ); + const presented = await presentDocuments(ctx, documents, { + isPublic, + shareId, + }); + const data = results.map((result, index) => ({ + ...result, + document: presented[index], + })); // When requesting subsequent pages of search results we don't want to record // duplicate search query records diff --git a/server/routes/api/revisions/revisions.ts b/server/routes/api/revisions/revisions.ts index 6d8ef2dbdd..cc74259a85 100644 --- a/server/routes/api/revisions/revisions.ts +++ b/server/routes/api/revisions/revisions.ts @@ -3,6 +3,7 @@ import Router from "koa-router"; import contentDisposition from "content-disposition"; import { escapeRegExp } from "es-toolkit/compat"; import mime from "mime-types"; +import { Op } from "sequelize"; import { errToString } from "@shared/utils/error"; import { UserRole } from "@shared/types"; import { RevisionHelper } from "@shared/utils/RevisionHelper"; @@ -13,7 +14,7 @@ import auth from "@server/middlewares/authentication"; import { rateLimiter } from "@server/middlewares/rateLimiter"; import { transaction } from "@server/middlewares/transaction"; import validate from "@server/middlewares/validate"; -import { Attachment, Document, Revision } from "@server/models"; +import { Attachment, Document, Revision, User } from "@server/models"; import { DocumentHelper } from "@server/models/helpers/DocumentHelper"; import { ProsemirrorHelper } from "@server/models/helpers/ProsemirrorHelper"; import { authorize } from "@server/policies"; @@ -246,15 +247,49 @@ router.post( where: { documentId: document.id, }, + attributes: { + exclude: ["content", "text"], + }, order: [[sort, direction]], offset: ctx.state.pagination.offset, limit: ctx.state.pagination.limit, paranoid: false, }); + // Load the collaborators for the whole page of revisions in a single + // query rather than one query per revision. + const collaboratorIds = new Set(); + for (const revision of revisions) { + for (const collaboratorId of revision.collaboratorIds ?? []) { + if (collaboratorId !== revision.userId) { + collaboratorIds.add(collaboratorId); + } + } + } + const collaborators = collaboratorIds.size + ? await User.findAll({ + where: { + id: { + [Op.in]: [...collaboratorIds], + }, + }, + paranoid: false, + }) + : []; + const collaboratorMap = new Map(collaborators.map((u) => [u.id, u])); + const data = await Promise.all( revisions.map((revision) => - presentRevision(revision, { includeContent: false }) + presentRevision(revision, { + includeContent: false, + collaborators: [ + revision.user, + ...(revision.collaboratorIds ?? []) + .filter((collaboratorId) => collaboratorId !== revision.userId) + .map((collaboratorId) => collaboratorMap.get(collaboratorId)) + .filter((u): u is User => !!u), + ], + }) ) );