Files
Tom Moor f95761dddf perf: Use FOR NO KEY UPDATE where locked rows are only updated (#14559)
* perf: Use FOR NO KEY UPDATE where locked rows are only updated

Rows that other tables reference by foreign key are now locked with
FOR NO KEY UPDATE when the transaction only updates or soft-deletes
them, so concurrent inserts that reference the row are not blocked.

* test: Collaborative save does not wait for foreign key inserts
2026-11-12 18:54:37 -05:00

263 lines
8.7 KiB
TypeScript

import type {
onStoreDocumentPayload,
onLoadDocumentPayload,
afterLoadDocumentPayload,
Connection,
Extension,
} from "@hocuspocus/server";
import * as Y from "yjs";
import { Day } from "@shared/utils/time";
import { toError } from "@shared/utils/error";
import Logger from "@server/logging/Logger";
import { trace } from "@server/logging/tracing";
import Document from "@server/models/Document";
import { ProsemirrorHelper } from "@server/models/helpers/ProsemirrorHelper";
import { sequelize } from "@server/storage/database";
import Redis from "@server/storage/redis";
import documentCollaborativeUpdater from "../commands/documentCollaborativeUpdater";
import type { withContext } from "./types";
@trace()
export default class PersistenceExtension implements Extension {
/** The maximum number of times persisting a single document will be retried. */
private static maxPersistFailures = 5;
/** The names of documents that have changed since they were last persisted. */
private unsavedDocumentNames = new Set<string>();
/** The number of consecutive persistence failures, keyed by document name. */
private persistFailureCounts = new Map<string, number>();
/** Attribution captured synchronously with each document update. */
private lastEditors = new WeakMap<Y.Doc, Promise<CollaborativeEdit>>();
async onLoadDocument({
documentName,
...data
}: withContext<onLoadDocumentPayload>) {
const [, documentId] = documentName.split(".");
const fieldName = "default";
// Check if the given field already exists in the given y-doc. This is import
// so we don't import a document fresh if it exists already.
if (!data.document.isEmpty(fieldName)) {
return;
}
// First, try to find the document without a lock to check if it has state
const documentWithoutLock = await Document.unscoped().findOne({
attributes: ["state"],
rejectOnEmpty: true,
where: {
id: documentId,
},
});
// If the document already has state, we can return it without needing a transaction
if (documentWithoutLock.state) {
const ydoc = new Y.Doc();
Logger.info("database", `Document ${documentId} is in database state`);
Y.applyUpdate(ydoc, documentWithoutLock.state);
return ydoc;
}
// If the document doesn't have state yet, we need to acquire a lock and create it
return await sequelize.transaction(async (transaction) => {
const document = await Document.unscoped().findOne({
attributes: ["id", "state", "content", "text"],
transaction,
lock: transaction.LOCK.NO_KEY_UPDATE,
rejectOnEmpty: true,
where: {
id: documentId,
},
});
let ydoc;
// Double-check the state in case another process created it
if (document.state) {
ydoc = new Y.Doc();
Logger.info("database", `Document ${documentId} is in database state`);
Y.applyUpdate(ydoc, document.state);
return ydoc;
}
if (document.content) {
Logger.info(
"database",
`Document ${documentId} is not in state, creating from content`
);
ydoc = ProsemirrorHelper.toYDoc(document.content, fieldName);
} else {
Logger.info(
"database",
`Document ${documentId} is not in state, creating from text`
);
ydoc = ProsemirrorHelper.toYDoc(document.text, fieldName);
}
const state = ProsemirrorHelper.toState(ydoc);
await document.update(
{
state,
},
{
silent: true,
hooks: false,
transaction,
}
);
return ydoc;
});
}
/**
* Track edits and their authenticated origin before asynchronous hooks run.
*
* @param data the loaded collaborative document.
* @returns a promise resolving when tracking is installed.
*/
async afterLoadDocument({
documentName,
document,
}: afterLoadDocumentPayload) {
// Track changes from the ydoc itself rather than the onChange hook, which
// runs behind other extensions in an async chain and so may not have
// recorded the change by the time the document is stored on disconnect.
const [, documentId] = documentName.split(".");
document.on("update", (_update: Uint8Array, origin?: Connection) => {
this.unsavedDocumentNames.add(documentName);
const context:
| withContext<afterLoadDocumentPayload>["context"]
| undefined = origin?.context;
const userId = context?.user?.id;
const key = Document.getCollaboratorKey(documentId);
const editor = userId
? Redis.defaultClient
.zaddWithSequence(key, userId, Day.seconds)
.then((sequence) => ({ userId, sequence }))
: Redis.defaultClient.zlatestWithSequence(key).then((latest) => ({
userId: latest?.member,
sequence: latest?.sequence,
}));
// Updates from Redis have no connection origin. Resolve their editor
// when received, rather than reading a potentially newer editor at save
// time or retaining the previous local editor.
this.lastEditors.set(
document,
editor.catch((err) => {
Logger.warn("Unable to track document editor", {
documentId,
message: toError(err).message,
});
return { userId };
})
);
});
}
/**
* Persist the pending document snapshot with its editing user.
*
* @param data the document and connection requesting persistence.
* @returns a promise resolving when the persistence attempt completes.
*/
async onStoreDocument({
document,
context,
documentName,
clientsCount,
requestParameters,
}: onStoreDocumentPayload) {
const [, documentId] = documentName.split(".");
const clientVersion = requestParameters.get("editorVersion");
// Nothing to do if the document hasn't changed since it was last persisted.
// Note the flag is cleared before writing so that changes received while
// persisting will schedule another store.
if (!this.unsavedDocumentNames.delete(documentName)) {
Logger.debug("multiplayer", `No changes for ${documentName}`);
return;
}
// Capture both the content and local author before yielding. New edits
// received during Redis or database I/O belong to a subsequent save.
const snapshot = new Y.Doc();
Y.applyUpdate(snapshot, Y.encodeStateAsUpdate(document));
const editor = this.lastEditors.get(document);
let edit: CollaborativeEdit | undefined;
// Collaborators are used for attribution only, failure to load them must
// not prevent the document itself from being persisted.
let sessionCollaboratorIds: string[] = [];
try {
edit = await editor;
if (edit?.sequence !== undefined) {
sessionCollaboratorIds = await Redis.defaultClient.zrangebyscore(
Document.getCollaboratorKey(documentId),
"-inf",
edit.sequence
);
}
} catch (err) {
Logger.warn("Unable to load collaborators for document", {
documentId,
message: toError(err).message,
});
}
// Keep the captured editor last even if Redis failed, revision cleanup
// removed their entry, or a later edit moved it beyond this snapshot's sequence.
const editingUserId = edit?.userId;
if (editingUserId) {
sessionCollaboratorIds = [
...sessionCollaboratorIds.filter((id) => id !== editingUserId),
editingUserId,
];
}
try {
await documentCollaborativeUpdater({
documentId,
ydoc: snapshot,
collaborators: {
ids: sessionCollaboratorIds,
sequence: edit?.sequence,
},
isLastConnection: clientsCount === 0,
clientVersion,
});
this.persistFailureCounts.delete(documentName);
} catch (err) {
const failures = (this.persistFailureCounts.get(documentName) ?? 0) + 1;
const giveUp = failures >= PersistenceExtension.maxPersistFailures;
if (giveUp) {
// Stop retrying, the error is unlikely to be transient. Further changes
// to the document will set the flag again and restart the count.
this.persistFailureCounts.delete(documentName);
} else {
// Restore the flag so that a subsequent store will retry the write.
this.persistFailureCounts.set(documentName, failures);
this.unsavedDocumentNames.add(documentName);
}
Logger.error("Unable to persist document", toError(err), {
documentId,
userId: context.user?.id,
failures,
giveUp,
});
} finally {
snapshot.destroy();
}
}
}
interface CollaborativeEdit {
userId?: string;
sequence?: number;
}