Files
outline/server/queues/tasks/ExportTask.ts
T
Tom Moor bc992fb22a refactor: Add jobId hook to BaseTask for deduplicating queued jobs (#13890)
* refactor: Add jobId hook to BaseTask for deduplicating queued jobs

* fix: Drop jobId from APIImportTask, it re-schedules itself while active
2026-09-25 23:44:16 +00:00

318 lines
9.3 KiB
TypeScript

import { Readable } from "node:stream";
import fs from "fs-extra";
import { truncate } from "es-toolkit/compat";
import { Op } from "sequelize";
import type { WhereOptions } from "sequelize";
import type { ZipFile } from "yazl";
import { toError } from "@shared/utils/error";
import type { NavigationNode } from "@shared/types";
import { FileOperationState, NotificationEventType } from "@shared/types";
import { bytesToHumanReadable } from "@shared/utils/files";
import ExportFailureEmail from "@server/emails/templates/ExportFailureEmail";
import ExportSuccessEmail from "@server/emails/templates/ExportSuccessEmail";
import env from "@server/env";
import { ValidationError } from "@server/errors";
import Logger from "@server/logging/Logger";
import {
Attachment,
Collection,
Document,
Event,
FileOperation,
Team,
User,
} from "@server/models";
import fileOperationPresenter from "@server/presenters/fileOperation";
import { sequelizeReadOnly } from "@server/storage/database";
import FileStorage from "@server/storage/files";
import { BaseTask, TaskPriority } from "./base/BaseTask";
type Props = {
fileOperationId: string;
};
export default abstract class ExportTask extends BaseTask<Props> {
protected jobId({ fileOperationId }: Props) {
return `export:${fileOperationId}`;
}
/**
* Transforms the data to be exported, uploads, and notifies user.
*
* @param props The props
*/
public async perform({ fileOperationId }: Props) {
Logger.info("task", `ExportTask fetching data for ${fileOperationId}`);
const fileOperation = await FileOperation.findByPk(fileOperationId, {
rejectOnEmpty: true,
});
const [team, user] = await Promise.all([
Team.findByPk(fileOperation.teamId, { rejectOnEmpty: true }),
User.findByPk(fileOperation.userId, { rejectOnEmpty: true }),
]);
let filePath: string | undefined;
let readStream: fs.ReadStream | undefined;
try {
Logger.info("task", `ExportTask processing data for ${fileOperationId}`, {
options: fileOperation.options,
});
await this.updateFileOperation(fileOperation, {
state: FileOperationState.Creating,
});
filePath = await this.loadDataAndExport(fileOperation, user);
Logger.info("task", `ExportTask uploading data for ${fileOperationId}`);
await this.updateFileOperation(fileOperation, {
state: FileOperationState.Uploading,
});
const stat = await fs.stat(filePath);
readStream = fs.createReadStream(filePath);
const url = await FileStorage.store({
body: readStream,
contentLength: stat.size,
contentType: "application/zip",
key: fileOperation.key,
acl: "private",
});
await this.updateFileOperation(fileOperation, {
size: stat.size,
state: FileOperationState.Complete,
url,
});
if (user.subscribedToEventType(NotificationEventType.ExportCompleted)) {
await new ExportSuccessEmail({
to: user.email,
language: user.language,
userId: user.id,
id: fileOperation.id,
teamUrl: team.url,
teamId: team.id,
}).schedule();
}
} catch (error) {
await this.updateFileOperation(fileOperation, {
state: FileOperationState.Error,
error: toError(error),
});
if (user.subscribedToEventType(NotificationEventType.ExportCompleted)) {
await new ExportFailureEmail({
to: user.email,
language: user.language,
userId: user.id,
teamUrl: team.url,
teamId: team.id,
}).schedule();
}
throw error;
} finally {
// Destroy the read stream first to release the file handle before deletion
if (readStream) {
readStream.destroy();
}
if (filePath) {
void fs.unlink(filePath).catch((error) => {
Logger.error(`Failed to delete temporary file ${filePath}`, error);
});
}
}
}
public async loadDataAndExport(
fileOperation: FileOperation,
user: User
): Promise<string> {
if (fileOperation.documentId) {
const document = await sequelizeReadOnly.transaction((transaction) =>
Document.findByPk(fileOperation.documentId!, {
include: {
model: Collection.scope("withDocumentStructure"),
as: "collection",
},
rejectOnEmpty: true,
transaction,
})
);
const documentStructure = document.collection?.getDocumentTree(
document.id
);
if (!documentStructure) {
throw new Error("Document not found in collection tree");
}
return this.exportDocument(
document,
documentStructure.children ?? [],
fileOperation.options?.includeAttachments ?? true
);
}
// ensure attachment size is within limits
if (
!fileOperation.collectionId &&
fileOperation.options?.includeAttachments &&
env.MAXIMUM_EXPORT_SIZE
) {
const totalAttachmentsSize = await Attachment.getTotalSizeForTeam(
sequelizeReadOnly,
user.teamId
);
if (totalAttachmentsSize > env.MAXIMUM_EXPORT_SIZE) {
throw ValidationError(
`${bytesToHumanReadable(
totalAttachmentsSize
)} of attachments in workspace is larger than maximum export size of ${bytesToHumanReadable(
env.MAXIMUM_EXPORT_SIZE
)}.`
);
}
}
const where: WhereOptions<Collection> = {
teamId: user.teamId,
};
if (!fileOperation.options?.includePrivate) {
where.permission = {
[Op.ne]: null,
};
}
if (fileOperation.collectionId) {
where.id = fileOperation.collectionId;
} else {
where.archivedAt = {
[Op.eq]: null,
};
}
const collections = await sequelizeReadOnly.transaction((transaction) =>
Collection.scope("withDocumentStructure").findAll({
where,
transaction,
})
);
return this.exportCollections(collections, fileOperation);
}
/**
* Transform the data in all of the passed collections into a single Buffer.
*
* @param collections The collections to export
* @returns A promise that resolves to a temporary file path
*/
protected abstract exportCollections(
collections: Collection[],
fileOperation: FileOperation
): Promise<string>;
/**
* Transform the data in the document into a single Buffer.
*
* @param document The document to export
* @param documentStructure Structure of document's children
* @param includeAttachments Whether to include attachments in the export
* @returns A promise that resolves to a temporary file path
*/
protected abstract exportDocument(
document: Document,
documentStructure: NavigationNode[],
includeAttachments: boolean
): Promise<string>;
/**
* Add an attachment to the archive, streaming its contents from storage as
* the archive is written so that the file is never held in memory whole.
*
* A read that fails, or is interrupted part way through, fails the export —
* a silently truncated archive is worse than none, as it looks complete.
* An attachment that is simply absent from storage is written as an empty
* entry, since a row outliving its file must not make export impossible.
*
* @param zip The archive to add the attachment to
* @param attachment The attachment to add
* @param pathInZip The path to store the attachment at within the archive
*/
protected addAttachmentToArchive(
zip: ZipFile,
attachment: Attachment,
pathInZip: string
) {
zip.addReadStreamLazy(
pathInZip,
{ mtime: attachment.updatedAt },
(callback) => {
attachment.stream.then(
(stream) => {
if (!stream) {
Logger.warn(`Attachment is missing from storage`, {
attachmentId: attachment.id,
teamId: attachment.teamId,
});
return callback(null, Readable.from([]));
}
// yazl does not watch the streams it is handed, so an interrupted
// read would otherwise stall the archive rather than fail it.
stream.on("error", (err) => zip.emit("error", toError(err)));
return callback(null, stream);
},
(err) => callback(toError(err), Readable.from([]))
);
}
);
}
/**
* Update the state of the underlying FileOperation in the database and send
* an event to the client.
*
* @param fileOperation The FileOperation to update
*/
private async updateFileOperation(
fileOperation: FileOperation,
options: Omit<Partial<FileOperation>, "error"> & { error?: Error }
) {
await fileOperation.update(
{
...options,
error: options.error
? truncate(options.error.message, { length: 255 })
: undefined,
},
{
hooks: false,
}
);
await Event.schedule({
name: "fileOperations.update",
modelId: fileOperation.id,
teamId: fileOperation.teamId,
actorId: fileOperation.userId,
data: fileOperationPresenter(fileOperation),
});
}
/**
* Job options such as priority and retry strategy, as defined by Bull.
*/
public get options() {
return {
priority: TaskPriority.Background,
attempts: 1,
};
}
}