Files
Tom Moor 1465b7ea80 feat: Show export completion in the notifications center (#14673)
* feat: Show export completion in the notifications center

Exports now create an in-app notification with the export name, status,
format and a download button. Export emails are sent from the notification
with a one-minute delay, so they are skipped once the notification is read.

* fix: Avoid nested button in export notification and keep it after delete

* fix: Revoke export notifications when access to the source is lost

* fix: Remove icon from export notification download cue
2026-11-20 11:17:15 -05:00

332 lines
9.8 KiB
TypeScript

import { Readable } from "node:stream";
import fs from "fs-extra";
import { truncate } from "es-toolkit/compat";
import { Op } from "sequelize";
import type { WhereOptions } from "sequelize";
import type { ZipFile } from "yazl";
import { toError } from "@shared/utils/error";
import type { NavigationNode } from "@shared/types";
import { FileOperationState, NotificationEventType } from "@shared/types";
import { bytesToHumanReadable } from "@shared/utils/files";
import env from "@server/env";
import { ValidationError } from "@server/errors";
import Logger from "@server/logging/Logger";
import {
Attachment,
Collection,
Document,
Event,
FileOperation,
Notification,
User,
} from "@server/models";
import fileOperationPresenter from "@server/presenters/fileOperation";
import { sequelizeReadOnly } from "@server/storage/database";
import FileStorage from "@server/storage/files";
import { BaseTask, TaskPriority } from "./base/BaseTask";
type Props = {
fileOperationId: string;
};
export default abstract class ExportTask extends BaseTask<Props> {
protected jobId({ fileOperationId }: Props) {
return `export:${fileOperationId}`;
}
/**
* Transforms the data to be exported, uploads, and notifies user.
*
* @param props The props
*/
public async perform({ fileOperationId }: Props) {
Logger.info("task", `ExportTask fetching data for ${fileOperationId}`);
const fileOperation = await FileOperation.findByPk(fileOperationId, {
rejectOnEmpty: true,
});
const user = await User.findByPk(fileOperation.userId, {
rejectOnEmpty: true,
});
let filePath: string | undefined;
let readStream: fs.ReadStream | undefined;
try {
Logger.info("task", `ExportTask processing data for ${fileOperationId}`, {
options: fileOperation.options,
});
await this.updateFileOperation(fileOperation, {
state: FileOperationState.Creating,
});
filePath = await this.loadDataAndExport(fileOperation, user);
Logger.info("task", `ExportTask uploading data for ${fileOperationId}`);
await this.updateFileOperation(fileOperation, {
state: FileOperationState.Uploading,
});
const stat = await fs.stat(filePath);
readStream = fs.createReadStream(filePath);
const url = await FileStorage.store({
body: readStream,
contentLength: stat.size,
contentType: "application/zip",
key: fileOperation.key,
acl: "private",
});
await this.updateFileOperation(fileOperation, {
size: stat.size,
state: FileOperationState.Complete,
url,
});
await this.createNotification(fileOperation, user);
} catch (error) {
await this.updateFileOperation(fileOperation, {
state: FileOperationState.Error,
error: toError(error),
});
await this.createNotification(fileOperation, user);
throw error;
} finally {
// Destroy the read stream first to release the file handle before deletion
if (readStream) {
readStream.destroy();
}
if (filePath) {
void fs.unlink(filePath).catch((error) => {
Logger.error(`Failed to delete temporary file ${filePath}`, error);
});
}
}
}
public async loadDataAndExport(
fileOperation: FileOperation,
user: User
): Promise<string> {
if (fileOperation.documentId) {
const document = await sequelizeReadOnly.transaction((transaction) =>
Document.findByPk(fileOperation.documentId!, {
include: {
model: Collection.scope("withDocumentStructure"),
as: "collection",
},
rejectOnEmpty: true,
transaction,
})
);
const documentStructure = document.collection?.getDocumentTree(
document.id
);
if (!documentStructure) {
throw new Error("Document not found in collection tree");
}
return this.exportDocument(
document,
documentStructure.children ?? [],
fileOperation.options?.includeAttachments ?? true
);
}
// ensure attachment size is within limits
if (
!fileOperation.collectionId &&
fileOperation.options?.includeAttachments &&
env.MAXIMUM_EXPORT_SIZE
) {
const totalAttachmentsSize = await Attachment.getTotalSizeForTeam(
sequelizeReadOnly,
user.teamId
);
if (totalAttachmentsSize > env.MAXIMUM_EXPORT_SIZE) {
throw ValidationError(
`${bytesToHumanReadable(
totalAttachmentsSize
)} of attachments in workspace is larger than maximum export size of ${bytesToHumanReadable(
env.MAXIMUM_EXPORT_SIZE
)}.`
);
}
}
const where: WhereOptions<Collection> = {
teamId: user.teamId,
};
if (!fileOperation.options?.includePrivate) {
where.permission = {
[Op.ne]: null,
};
}
if (fileOperation.collectionId) {
where.id = fileOperation.collectionId;
// Only the owner can export a personal collection.
where.ownerId = { [Op.or]: [{ [Op.is]: null }, { [Op.eq]: user.id }] };
} else {
where.archivedAt = {
[Op.eq]: null,
};
// A workspace export never includes personal collections.
where.ownerId = { [Op.is]: null };
}
const collections = await sequelizeReadOnly.transaction((transaction) =>
Collection.scope("withDocumentStructure").findAll({
where,
transaction,
})
);
return this.exportCollections(collections, fileOperation);
}
/**
* Transform the data in all of the passed collections into a single Buffer.
*
* @param collections The collections to export
* @returns A promise that resolves to a temporary file path
*/
protected abstract exportCollections(
collections: Collection[],
fileOperation: FileOperation
): Promise<string>;
/**
* Transform the data in the document into a single Buffer.
*
* @param document The document to export
* @param documentStructure Structure of document's children
* @param includeAttachments Whether to include attachments in the export
* @returns A promise that resolves to a temporary file path
*/
protected abstract exportDocument(
document: Document,
documentStructure: NavigationNode[],
includeAttachments: boolean
): Promise<string>;
/**
* Add an attachment to the archive, streaming its contents from storage as
* the archive is written so that the file is never held in memory whole.
*
* A read that fails, or is interrupted part way through, fails the export —
* a silently truncated archive is worse than none, as it looks complete.
* An attachment that is simply absent from storage is written as an empty
* entry, since a row outliving its file must not make export impossible.
*
* @param zip The archive to add the attachment to
* @param attachment The attachment to add
* @param pathInZip The path to store the attachment at within the archive
*/
protected addAttachmentToArchive(
zip: ZipFile,
attachment: Attachment,
pathInZip: string
) {
zip.addReadStreamLazy(
pathInZip,
{ mtime: attachment.updatedAt },
(callback) => {
attachment.stream.then(
(stream) => {
if (!stream) {
Logger.warn(`Attachment is missing from storage`, {
attachmentId: attachment.id,
teamId: attachment.teamId,
});
return callback(null, Readable.from([]));
}
// yazl does not watch the streams it is handed, so an interrupted
// read would otherwise stall the archive rather than fail it.
stream.on("error", (err) => zip.emit("error", toError(err)));
return callback(null, stream);
},
(err) => callback(toError(err), Readable.from([]))
);
}
);
}
/**
* Notify the user that requested the export, in the app and by email. A
* failure is logged and does not change the result of the export.
*
* @param fileOperation The FileOperation that finished.
* @param user The user that requested the export.
*/
private async createNotification(fileOperation: FileOperation, user: User) {
if (!user.subscribedToEventType(NotificationEventType.ExportCompleted)) {
return;
}
try {
await Notification.create({
event: NotificationEventType.ExportCompleted,
userId: user.id,
actorId: user.id,
teamId: user.teamId,
fileOperationId: fileOperation.id,
// The source lets the notification be revoked if access is lost.
documentId: fileOperation.documentId ?? undefined,
collectionId: fileOperation.collectionId ?? undefined,
});
} catch (error) {
Logger.error("Failed to create export notification", toError(error), {
fileOperationId: fileOperation.id,
});
}
}
/**
* Update the state of the underlying FileOperation in the database and send
* an event to the client.
*
* @param fileOperation The FileOperation to update
*/
private async updateFileOperation(
fileOperation: FileOperation,
options: Omit<Partial<FileOperation>, "error"> & { error?: Error }
) {
await fileOperation.update(
{
...options,
error: options.error
? truncate(options.error.message, { length: 255 })
: undefined,
},
{
hooks: false,
}
);
await Event.schedule({
name: "fileOperations.update",
modelId: fileOperation.id,
teamId: fileOperation.teamId,
actorId: fileOperation.userId,
data: fileOperationPresenter(fileOperation),
});
}
/**
* Job options such as priority and retry strategy, as defined by Bull.
*/
public get options() {
return {
priority: TaskPriority.Background,
attempts: 1,
};
}
}