diff --git a/app/utils/developer.ts b/app/utils/developer.ts index 1405875ecc..5cad8e11b6 100644 --- a/app/utils/developer.ts +++ b/app/utils/developer.ts @@ -1,4 +1,8 @@ import { flatten } from "es-toolkit/compat"; +import { + MultiplayerEntityType, + toMultiplayerName, +} from "@shared/collaboration/EntityName"; import stores from "~/stores"; import { flattenTree } from "@shared/utils/tree"; @@ -27,11 +31,15 @@ export async function deleteAllDatabases() { // by iterating over all known collections and documents. await Promise.all( stores.collections.orderedData.map(async (collection) => { - window.indexedDB.deleteDatabase(`collection.${collection.id}`); + window.indexedDB.deleteDatabase( + toMultiplayerName(MultiplayerEntityType.Collection, collection.id) + ); const nodes = flatten(collection.documents?.map(flattenTree)); return nodes.map(async (node) => { - window.indexedDB.deleteDatabase(`document.${node.id}`); + window.indexedDB.deleteDatabase( + toMultiplayerName(MultiplayerEntityType.Document, node.id) + ); }); }) ); diff --git a/server/collaboration/APIUpdateExtension.ts b/server/collaboration/APIUpdateExtension.ts index c6b2b5391b..8a5c101ed9 100644 --- a/server/collaboration/APIUpdateExtension.ts +++ b/server/collaboration/APIUpdateExtension.ts @@ -16,6 +16,7 @@ import Logger from "@server/logging/Logger"; import { trace } from "@server/logging/tracing"; import Collection from "@server/models/Collection"; import Document from "@server/models/Document"; +import type { HookContext } from "@server/models/base/Model"; import RedisAdapter from "@server/storage/redis"; /** @@ -193,6 +194,30 @@ export class APIUpdateExtension implements Extension { } }; + /** + * Publish a notification that an entity was updated via the API once the + * current transaction, if any, has committed. This should be called from + * the entity's update hook. + * + * @param documentName - the multiplayer name of the entity that was updated, eg "document.123". + * @param ctx - the hook context of the update. + */ + static notifyUpdateAfterCommit(documentName: string, ctx: HookContext) { + const actorId = ctx.auth?.user?.id; + if (!actorId) { + return; + } + + const notify = () => APIUpdateExtension.notifyUpdate(documentName, actorId); + + if (ctx.transaction) { + const transaction = ctx.transaction.parent || ctx.transaction; + transaction.afterCommit(notify); + } else { + void notify(); + } + } + /** * Publish a notification that an entity was updated via the API. * This should be called from the entity's update command. diff --git a/server/collaboration/PersistenceExtension.ts b/server/collaboration/PersistenceExtension.ts index e26be685c4..6e4e51e927 100644 --- a/server/collaboration/PersistenceExtension.ts +++ b/server/collaboration/PersistenceExtension.ts @@ -17,6 +17,7 @@ import Document from "@server/models/Document"; import { ProsemirrorHelper } from "@server/models/helpers/ProsemirrorHelper"; import { sequelize } from "@server/storage/database"; import Redis from "@server/storage/redis"; +import { RedisPrefixHelper } from "@server/utils/RedisPrefixHelper"; import collectionCollaborativeUpdater from "../commands/collectionCollaborativeUpdater"; import documentCollaborativeUpdater from "../commands/documentCollaborativeUpdater"; import type { withContext } from "./types"; @@ -44,7 +45,7 @@ export default class PersistenceExtension implements Extension { } async onChange({ context, documentName }: withContext) { - const { type, id } = parseMultiplayerName(documentName); + const { id } = parseMultiplayerName(documentName); if (context.user) { Logger.debug( @@ -52,10 +53,7 @@ export default class PersistenceExtension implements Extension { `${context.user.name} changed ${documentName}` ); - const key = - type === MultiplayerEntityType.Collection - ? Collection.getCollaboratorKey(id) - : Document.getCollaboratorKey(id); + const key = RedisPrefixHelper.getCollaboratorsKey(id); await Redis.defaultClient.sadd(key, context.user.id); } } @@ -70,10 +68,7 @@ export default class PersistenceExtension implements Extension { const { type, id } = parseMultiplayerName(documentName); const clientVersion = requestParameters.get("editorVersion"); - const key = - type === MultiplayerEntityType.Collection - ? Collection.getCollaboratorKey(id) - : Document.getCollaboratorKey(id); + const key = RedisPrefixHelper.getCollaboratorsKey(id); const sessionCollaboratorIds = await Redis.defaultClient.smembers(key); if (!sessionCollaboratorIds || sessionCollaboratorIds.length === 0) { Logger.debug("multiplayer", `No changes for ${documentName}`); @@ -105,6 +100,20 @@ export default class PersistenceExtension implements Extension { } } + /** + * Hydrates a YJS document from stored collaborative state. + * + * @param name A label for the entity used in logging. + * @param state The stored collaborative state. + * @returns the hydrated YJS document. + */ + private hydrateFromState(name: string, state: Uint8Array): Y.Doc { + Logger.info("database", `${name} is in database state`); + const ydoc = new Y.Doc(); + Y.applyUpdate(ydoc, state); + return ydoc; + } + /** * Loads the collaborative state for a document, creating it from the * content or text if it does not exist yet. @@ -125,10 +134,10 @@ export default class PersistenceExtension implements Extension { // If the document already has state, we can return it without needing a transaction if (documentWithoutLock.state) { - const ydoc = new Y.Doc(); - Logger.info("database", `Document ${documentId} is in database state`); - Y.applyUpdate(ydoc, documentWithoutLock.state); - return ydoc; + return this.hydrateFromState( + `Document ${documentId}`, + documentWithoutLock.state + ); } // If the document doesn't have state yet, we need to acquire a lock and create it @@ -146,10 +155,7 @@ export default class PersistenceExtension implements Extension { // Double-check the state in case another process created it if (document.state) { - ydoc = new Y.Doc(); - Logger.info("database", `Document ${documentId} is in database state`); - Y.applyUpdate(ydoc, document.state); - return ydoc; + return this.hydrateFromState(`Document ${documentId}`, document.state); } if (document.content) { @@ -200,13 +206,10 @@ export default class PersistenceExtension implements Extension { // If the collection already has state, we can return it without needing a transaction if (collectionWithoutLock.state) { - const ydoc = new Y.Doc(); - Logger.info( - "database", - `Collection ${collectionId} is in database state` + return this.hydrateFromState( + `Collection ${collectionId}`, + collectionWithoutLock.state ); - Y.applyUpdate(ydoc, collectionWithoutLock.state); - return ydoc; } // If the collection doesn't have state yet, we need to acquire a lock and create it @@ -223,13 +226,10 @@ export default class PersistenceExtension implements Extension { // Double-check the state in case another process created it if (collection.state) { - const ydoc = new Y.Doc(); - Logger.info( - "database", - `Collection ${collectionId} is in database state` + return this.hydrateFromState( + `Collection ${collectionId}`, + collection.state ); - Y.applyUpdate(ydoc, collection.state); - return ydoc; } Logger.info( diff --git a/server/collaboration/ViewsExtension.ts b/server/collaboration/ViewsExtension.ts index 80d4ccf56a..32145bc489 100644 --- a/server/collaboration/ViewsExtension.ts +++ b/server/collaboration/ViewsExtension.ts @@ -47,9 +47,9 @@ export class ViewsExtension implements Extension { ); await Promise.all([ // views are only recorded for documents - ...(type === MultiplayerEntityType.Document - ? [View.touch(id, context.user.id, true)] - : []), + type === MultiplayerEntityType.Document + ? View.touch(id, context.user.id, true) + : undefined, context.user.update({ lastActiveAt: new Date() }), ]); } diff --git a/server/commands/collectionCollaborativeUpdater.ts b/server/commands/collectionCollaborativeUpdater.ts index cf8d3a9478..c91b8bf704 100644 --- a/server/commands/collectionCollaborativeUpdater.ts +++ b/server/commands/collectionCollaborativeUpdater.ts @@ -39,8 +39,9 @@ export default async function collectionCollaborativeUpdater({ transaction, }); - const collection = await Collection.unscoped().findOne({ - attributes: { exclude: ["state"] }, + // the default scope excludes the large state and documentStructure + // columns, neither of which need to be read here. + const collection = await Collection.findOne({ where: { id: collectionId, }, @@ -75,16 +76,12 @@ export default async function collectionCollaborativeUpdater({ `Persisting collection ${collectionId}, attributed to ${actorId}` ); - collection.content = content; - await collection.update( { content, description: ProsemirrorHelper.isEmptyData(content) ? null - : await DocumentHelper.toMarkdown(collection, { - includeTitle: false, - }), + : await DocumentHelper.toMarkdown(content), state: Buffer.from(state), }, { diff --git a/server/models/Collection.ts b/server/models/Collection.ts index 19b171febc..4b0b8c1698 100644 --- a/server/models/Collection.ts +++ b/server/models/Collection.ts @@ -372,16 +372,6 @@ class Collection extends ParanoidModel< return `/collection/${slugifiedName}-${urlId}`; } - /** - * Returns the key used to store the collaborators of a collection in Redis. - * - * @param collectionId The ID of the collection. - * @returns Redis key for collaborators. - */ - static getCollaboratorKey(collectionId: string) { - return `collaborators:${collectionId}`; - } - /** * Whether this collection is considered active or not. A collection is active if * it has not been archived or deleted. @@ -461,21 +451,11 @@ class Collection extends ParanoidModel< @AfterUpdate static notifyCollaborationServer(model: Collection, ctx: HookContext) { - if (model.changed("state") && ctx.auth?.user?.id) { - const actorId = ctx.auth.user.id; - const notify = async () => { - await APIUpdateExtension.notifyUpdate( - toMultiplayerName(MultiplayerEntityType.Collection, model.id), - actorId - ); - }; - - if (ctx.transaction) { - const transaction = ctx.transaction.parent || ctx.transaction; - transaction.afterCommit(notify); - } else { - void notify(); - } + if (model.changed("state")) { + APIUpdateExtension.notifyUpdateAfterCommit( + toMultiplayerName(MultiplayerEntityType.Collection, model.id), + ctx + ); } } diff --git a/server/models/Document.ts b/server/models/Document.ts index 30f2578713..d9690cbe13 100644 --- a/server/models/Document.ts +++ b/server/models/Document.ts @@ -422,15 +422,6 @@ class Document extends ArchivableModel< ); } - /** - * Returns the key used to store the collaborators of a document in Redis. - * @param documentId The ID of the document. - * @returns Redis key for collaborators - */ - static getCollaboratorKey(documentId: string) { - return `collaborators:${documentId}`; - } - static getPath({ title, urlId }: { title: string; urlId: string }) { if (!title.length) { return `/doc/untitled-${urlId}`; @@ -589,21 +580,11 @@ class Document extends ArchivableModel< @AfterUpdate static notifyCollaborationServer(model: Document, ctx: HookContext) { - if (model.changed("state") && ctx.auth?.user?.id) { - const actorId = ctx.auth.user.id; - const notify = async () => { - await APIUpdateExtension.notifyUpdate( - toMultiplayerName(MultiplayerEntityType.Document, model.id), - actorId - ); - }; - - if (ctx.transaction) { - const transaction = ctx.transaction.parent || ctx.transaction; - transaction.afterCommit(notify); - } else { - void notify(); - } + if (model.changed("state")) { + APIUpdateExtension.notifyUpdateAfterCommit( + toMultiplayerName(MultiplayerEntityType.Document, model.id), + ctx + ); } } diff --git a/server/queues/processors/RevisionsProcessor.ts b/server/queues/processors/RevisionsProcessor.ts index 9ff53e1a81..ffdf67d9ec 100644 --- a/server/queues/processors/RevisionsProcessor.ts +++ b/server/queues/processors/RevisionsProcessor.ts @@ -3,6 +3,7 @@ import Redis from "@server/storage/redis"; import revisionCreator from "@server/commands/revisionCreator"; import { Revision, Document, User } from "@server/models"; import type { DocumentEvent, RevisionEvent, Event } from "@server/types"; +import { RedisPrefixHelper } from "@server/utils/RedisPrefixHelper"; import DocumentUpdateTextTask from "../tasks/DocumentUpdateTextTask"; import BaseProcessor from "./BaseProcessor"; @@ -23,7 +24,7 @@ export default class RevisionsProcessor extends BaseProcessor { } // Get collaborator IDs since last revision was written. - const key = Document.getCollaboratorKey(event.documentId); + const key = RedisPrefixHelper.getCollaboratorsKey(event.documentId); const collaboratorIds = await Redis.defaultClient.smembers(key); await Redis.defaultClient.del(key); diff --git a/server/routes/api/collections/collections.ts b/server/routes/api/collections/collections.ts index c22619c902..084d334d8e 100644 --- a/server/routes/api/collections/collections.ts +++ b/server/routes/api/collections/collections.ts @@ -593,7 +593,7 @@ router.post( userId: user.id, // required so that edits to the description are synced to any active // collaborative editing session. - includeState: true, + includeState: description !== undefined || data !== undefined, transaction, }); authorize(user, "update", collection); diff --git a/server/tools/collections.ts b/server/tools/collections.ts index a3dbf136bb..b2c8c479b1 100644 --- a/server/tools/collections.ts +++ b/server/tools/collections.ts @@ -245,7 +245,7 @@ export function collectionTools(server: McpServer, scopes: string[]) { userId: user.id, // required so that edits to the description are synced to any // active collaborative editing session. - includeState: true, + includeState: input.description !== undefined, rejectOnEmpty: true, }); authorize(user, "update", collection); diff --git a/server/utils/RedisPrefixHelper.ts b/server/utils/RedisPrefixHelper.ts index 5510bdb26c..e05c1c3b73 100644 --- a/server/utils/RedisPrefixHelper.ts +++ b/server/utils/RedisPrefixHelper.ts @@ -22,6 +22,17 @@ export class RedisPrefixHelper { return `cd:${collectionId}`; } + /** + * Gets key against which the collaborators of a multiplayer editing session + * are stored, shared by all entity types. + * + * @param id The ID of the entity being edited. + * @returns the cache key string. + */ + public static getCollaboratorsKey(id: string) { + return `collaborators:${id}`; + } + /** * Gets key for caching embed check results. This is a global cache key * (not team-specific) since embed headers are the same for all users.