import { Readable } from "node:stream"; import fs from "fs-extra"; import { truncate } from "es-toolkit/compat"; import { Op } from "sequelize"; import type { WhereOptions } from "sequelize"; import type { ZipFile } from "yazl"; import { toError } from "@shared/utils/error"; import type { NavigationNode } from "@shared/types"; import { FileOperationState, NotificationEventType } from "@shared/types"; import { bytesToHumanReadable } from "@shared/utils/files"; import env from "@server/env"; import { ValidationError } from "@server/errors"; import Logger from "@server/logging/Logger"; import { Attachment, Collection, Document, Event, FileOperation, Notification, User, } from "@server/models"; import fileOperationPresenter from "@server/presenters/fileOperation"; import { sequelizeReadOnly } from "@server/storage/database"; import FileStorage from "@server/storage/files"; import { BaseTask, TaskPriority } from "./base/BaseTask"; type Props = { fileOperationId: string; }; export default abstract class ExportTask extends BaseTask { protected jobId({ fileOperationId }: Props) { return `export:${fileOperationId}`; } /** * Transforms the data to be exported, uploads, and notifies user. * * @param props The props */ public async perform({ fileOperationId }: Props) { Logger.info("task", `ExportTask fetching data for ${fileOperationId}`); const fileOperation = await FileOperation.findByPk(fileOperationId, { rejectOnEmpty: true, }); const user = await User.findByPk(fileOperation.userId, { rejectOnEmpty: true, }); let filePath: string | undefined; let readStream: fs.ReadStream | undefined; try { Logger.info("task", `ExportTask processing data for ${fileOperationId}`, { options: fileOperation.options, }); await this.updateFileOperation(fileOperation, { state: FileOperationState.Creating, }); filePath = await this.loadDataAndExport(fileOperation, user); Logger.info("task", `ExportTask uploading data for ${fileOperationId}`); await this.updateFileOperation(fileOperation, { state: FileOperationState.Uploading, }); const stat = await fs.stat(filePath); readStream = fs.createReadStream(filePath); const url = await FileStorage.store({ body: readStream, contentLength: stat.size, contentType: "application/zip", key: fileOperation.key, acl: "private", }); await this.updateFileOperation(fileOperation, { size: stat.size, state: FileOperationState.Complete, url, }); await this.createNotification(fileOperation, user); } catch (error) { await this.updateFileOperation(fileOperation, { state: FileOperationState.Error, error: toError(error), }); await this.createNotification(fileOperation, user); throw error; } finally { // Destroy the read stream first to release the file handle before deletion if (readStream) { readStream.destroy(); } if (filePath) { void fs.unlink(filePath).catch((error) => { Logger.error(`Failed to delete temporary file ${filePath}`, error); }); } } } public async loadDataAndExport( fileOperation: FileOperation, user: User ): Promise { if (fileOperation.documentId) { const document = await sequelizeReadOnly.transaction((transaction) => Document.findByPk(fileOperation.documentId!, { include: { model: Collection.scope("withDocumentStructure"), as: "collection", }, rejectOnEmpty: true, transaction, }) ); const documentStructure = document.collection?.getDocumentTree( document.id ); if (!documentStructure) { throw new Error("Document not found in collection tree"); } return this.exportDocument( document, documentStructure.children ?? [], fileOperation.options?.includeAttachments ?? true ); } // ensure attachment size is within limits if ( !fileOperation.collectionId && fileOperation.options?.includeAttachments && env.MAXIMUM_EXPORT_SIZE ) { const totalAttachmentsSize = await Attachment.getTotalSizeForTeam( sequelizeReadOnly, user.teamId ); if (totalAttachmentsSize > env.MAXIMUM_EXPORT_SIZE) { throw ValidationError( `${bytesToHumanReadable( totalAttachmentsSize )} of attachments in workspace is larger than maximum export size of ${bytesToHumanReadable( env.MAXIMUM_EXPORT_SIZE )}.` ); } } const where: WhereOptions = { teamId: user.teamId, }; if (!fileOperation.options?.includePrivate) { where.permission = { [Op.ne]: null, }; } if (fileOperation.collectionId) { where.id = fileOperation.collectionId; // Only the owner can export a personal collection. where.ownerId = { [Op.or]: [{ [Op.is]: null }, { [Op.eq]: user.id }] }; } else { where.archivedAt = { [Op.eq]: null, }; // A workspace export never includes personal collections. where.ownerId = { [Op.is]: null }; } const collections = await sequelizeReadOnly.transaction((transaction) => Collection.scope("withDocumentStructure").findAll({ where, transaction, }) ); return this.exportCollections(collections, fileOperation); } /** * Transform the data in all of the passed collections into a single Buffer. * * @param collections The collections to export * @returns A promise that resolves to a temporary file path */ protected abstract exportCollections( collections: Collection[], fileOperation: FileOperation ): Promise; /** * Transform the data in the document into a single Buffer. * * @param document The document to export * @param documentStructure Structure of document's children * @param includeAttachments Whether to include attachments in the export * @returns A promise that resolves to a temporary file path */ protected abstract exportDocument( document: Document, documentStructure: NavigationNode[], includeAttachments: boolean ): Promise; /** * Add an attachment to the archive, streaming its contents from storage as * the archive is written so that the file is never held in memory whole. * * A read that fails, or is interrupted part way through, fails the export — * a silently truncated archive is worse than none, as it looks complete. * An attachment that is simply absent from storage is written as an empty * entry, since a row outliving its file must not make export impossible. * * @param zip The archive to add the attachment to * @param attachment The attachment to add * @param pathInZip The path to store the attachment at within the archive */ protected addAttachmentToArchive( zip: ZipFile, attachment: Attachment, pathInZip: string ) { zip.addReadStreamLazy( pathInZip, { mtime: attachment.updatedAt }, (callback) => { attachment.stream.then( (stream) => { if (!stream) { Logger.warn(`Attachment is missing from storage`, { attachmentId: attachment.id, teamId: attachment.teamId, }); return callback(null, Readable.from([])); } // yazl does not watch the streams it is handed, so an interrupted // read would otherwise stall the archive rather than fail it. stream.on("error", (err) => zip.emit("error", toError(err))); return callback(null, stream); }, (err) => callback(toError(err), Readable.from([])) ); } ); } /** * Notify the user that requested the export, in the app and by email. A * failure is logged and does not change the result of the export. * * @param fileOperation The FileOperation that finished. * @param user The user that requested the export. */ private async createNotification(fileOperation: FileOperation, user: User) { if (!user.subscribedToEventType(NotificationEventType.ExportCompleted)) { return; } try { await Notification.create({ event: NotificationEventType.ExportCompleted, userId: user.id, actorId: user.id, teamId: user.teamId, fileOperationId: fileOperation.id, // The source lets the notification be revoked if access is lost. documentId: fileOperation.documentId ?? undefined, collectionId: fileOperation.collectionId ?? undefined, }); } catch (error) { Logger.error("Failed to create export notification", toError(error), { fileOperationId: fileOperation.id, }); } } /** * Update the state of the underlying FileOperation in the database and send * an event to the client. * * @param fileOperation The FileOperation to update */ private async updateFileOperation( fileOperation: FileOperation, options: Omit, "error"> & { error?: Error } ) { await fileOperation.update( { ...options, error: options.error ? truncate(options.error.message, { length: 255 }) : undefined, }, { hooks: false, } ); await Event.schedule({ name: "fileOperations.update", modelId: fileOperation.id, teamId: fileOperation.teamId, actorId: fileOperation.userId, data: fileOperationPresenter(fileOperation), }); } /** * Job options such as priority and retry strategy, as defined by Bull. */ public get options() { return { priority: TaskPriority.Background, attempts: 1, }; } }