mirror of
https://github.com/outline/outline.git
synced 2026-11-20 21:23:06 +03:00
* feat: Show export completion in the notifications center Exports now create an in-app notification with the export name, status, format and a download button. Export emails are sent from the notification with a one-minute delay, so they are skipped once the notification is read. * fix: Avoid nested button in export notification and keep it after delete * fix: Revoke export notifications when access to the source is lost * fix: Remove icon from export notification download cue
332 lines
9.8 KiB
TypeScript
332 lines
9.8 KiB
TypeScript
import { Readable } from "node:stream";
|
|
import fs from "fs-extra";
|
|
import { truncate } from "es-toolkit/compat";
|
|
import { Op } from "sequelize";
|
|
import type { WhereOptions } from "sequelize";
|
|
import type { ZipFile } from "yazl";
|
|
import { toError } from "@shared/utils/error";
|
|
import type { NavigationNode } from "@shared/types";
|
|
import { FileOperationState, NotificationEventType } from "@shared/types";
|
|
import { bytesToHumanReadable } from "@shared/utils/files";
|
|
import env from "@server/env";
|
|
import { ValidationError } from "@server/errors";
|
|
import Logger from "@server/logging/Logger";
|
|
import {
|
|
Attachment,
|
|
Collection,
|
|
Document,
|
|
Event,
|
|
FileOperation,
|
|
Notification,
|
|
User,
|
|
} from "@server/models";
|
|
import fileOperationPresenter from "@server/presenters/fileOperation";
|
|
import { sequelizeReadOnly } from "@server/storage/database";
|
|
import FileStorage from "@server/storage/files";
|
|
import { BaseTask, TaskPriority } from "./base/BaseTask";
|
|
|
|
type Props = {
|
|
fileOperationId: string;
|
|
};
|
|
|
|
export default abstract class ExportTask extends BaseTask<Props> {
|
|
protected jobId({ fileOperationId }: Props) {
|
|
return `export:${fileOperationId}`;
|
|
}
|
|
|
|
/**
|
|
* Transforms the data to be exported, uploads, and notifies user.
|
|
*
|
|
* @param props The props
|
|
*/
|
|
public async perform({ fileOperationId }: Props) {
|
|
Logger.info("task", `ExportTask fetching data for ${fileOperationId}`);
|
|
const fileOperation = await FileOperation.findByPk(fileOperationId, {
|
|
rejectOnEmpty: true,
|
|
});
|
|
const user = await User.findByPk(fileOperation.userId, {
|
|
rejectOnEmpty: true,
|
|
});
|
|
|
|
let filePath: string | undefined;
|
|
let readStream: fs.ReadStream | undefined;
|
|
|
|
try {
|
|
Logger.info("task", `ExportTask processing data for ${fileOperationId}`, {
|
|
options: fileOperation.options,
|
|
});
|
|
|
|
await this.updateFileOperation(fileOperation, {
|
|
state: FileOperationState.Creating,
|
|
});
|
|
|
|
filePath = await this.loadDataAndExport(fileOperation, user);
|
|
|
|
Logger.info("task", `ExportTask uploading data for ${fileOperationId}`);
|
|
|
|
await this.updateFileOperation(fileOperation, {
|
|
state: FileOperationState.Uploading,
|
|
});
|
|
|
|
const stat = await fs.stat(filePath);
|
|
readStream = fs.createReadStream(filePath);
|
|
const url = await FileStorage.store({
|
|
body: readStream,
|
|
contentLength: stat.size,
|
|
contentType: "application/zip",
|
|
key: fileOperation.key,
|
|
acl: "private",
|
|
});
|
|
|
|
await this.updateFileOperation(fileOperation, {
|
|
size: stat.size,
|
|
state: FileOperationState.Complete,
|
|
url,
|
|
});
|
|
|
|
await this.createNotification(fileOperation, user);
|
|
} catch (error) {
|
|
await this.updateFileOperation(fileOperation, {
|
|
state: FileOperationState.Error,
|
|
error: toError(error),
|
|
});
|
|
await this.createNotification(fileOperation, user);
|
|
throw error;
|
|
} finally {
|
|
// Destroy the read stream first to release the file handle before deletion
|
|
if (readStream) {
|
|
readStream.destroy();
|
|
}
|
|
if (filePath) {
|
|
void fs.unlink(filePath).catch((error) => {
|
|
Logger.error(`Failed to delete temporary file ${filePath}`, error);
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
public async loadDataAndExport(
|
|
fileOperation: FileOperation,
|
|
user: User
|
|
): Promise<string> {
|
|
if (fileOperation.documentId) {
|
|
const document = await sequelizeReadOnly.transaction((transaction) =>
|
|
Document.findByPk(fileOperation.documentId!, {
|
|
include: {
|
|
model: Collection.scope("withDocumentStructure"),
|
|
as: "collection",
|
|
},
|
|
rejectOnEmpty: true,
|
|
transaction,
|
|
})
|
|
);
|
|
|
|
const documentStructure = document.collection?.getDocumentTree(
|
|
document.id
|
|
);
|
|
|
|
if (!documentStructure) {
|
|
throw new Error("Document not found in collection tree");
|
|
}
|
|
|
|
return this.exportDocument(
|
|
document,
|
|
documentStructure.children ?? [],
|
|
fileOperation.options?.includeAttachments ?? true
|
|
);
|
|
}
|
|
|
|
// ensure attachment size is within limits
|
|
if (
|
|
!fileOperation.collectionId &&
|
|
fileOperation.options?.includeAttachments &&
|
|
env.MAXIMUM_EXPORT_SIZE
|
|
) {
|
|
const totalAttachmentsSize = await Attachment.getTotalSizeForTeam(
|
|
sequelizeReadOnly,
|
|
user.teamId
|
|
);
|
|
|
|
if (totalAttachmentsSize > env.MAXIMUM_EXPORT_SIZE) {
|
|
throw ValidationError(
|
|
`${bytesToHumanReadable(
|
|
totalAttachmentsSize
|
|
)} of attachments in workspace is larger than maximum export size of ${bytesToHumanReadable(
|
|
env.MAXIMUM_EXPORT_SIZE
|
|
)}.`
|
|
);
|
|
}
|
|
}
|
|
|
|
const where: WhereOptions<Collection> = {
|
|
teamId: user.teamId,
|
|
};
|
|
|
|
if (!fileOperation.options?.includePrivate) {
|
|
where.permission = {
|
|
[Op.ne]: null,
|
|
};
|
|
}
|
|
|
|
if (fileOperation.collectionId) {
|
|
where.id = fileOperation.collectionId;
|
|
// Only the owner can export a personal collection.
|
|
where.ownerId = { [Op.or]: [{ [Op.is]: null }, { [Op.eq]: user.id }] };
|
|
} else {
|
|
where.archivedAt = {
|
|
[Op.eq]: null,
|
|
};
|
|
// A workspace export never includes personal collections.
|
|
where.ownerId = { [Op.is]: null };
|
|
}
|
|
|
|
const collections = await sequelizeReadOnly.transaction((transaction) =>
|
|
Collection.scope("withDocumentStructure").findAll({
|
|
where,
|
|
transaction,
|
|
})
|
|
);
|
|
|
|
return this.exportCollections(collections, fileOperation);
|
|
}
|
|
|
|
/**
|
|
* Transform the data in all of the passed collections into a single Buffer.
|
|
*
|
|
* @param collections The collections to export
|
|
* @returns A promise that resolves to a temporary file path
|
|
*/
|
|
protected abstract exportCollections(
|
|
collections: Collection[],
|
|
fileOperation: FileOperation
|
|
): Promise<string>;
|
|
|
|
/**
|
|
* Transform the data in the document into a single Buffer.
|
|
*
|
|
* @param document The document to export
|
|
* @param documentStructure Structure of document's children
|
|
* @param includeAttachments Whether to include attachments in the export
|
|
* @returns A promise that resolves to a temporary file path
|
|
*/
|
|
protected abstract exportDocument(
|
|
document: Document,
|
|
documentStructure: NavigationNode[],
|
|
includeAttachments: boolean
|
|
): Promise<string>;
|
|
|
|
/**
|
|
* Add an attachment to the archive, streaming its contents from storage as
|
|
* the archive is written so that the file is never held in memory whole.
|
|
*
|
|
* A read that fails, or is interrupted part way through, fails the export —
|
|
* a silently truncated archive is worse than none, as it looks complete.
|
|
* An attachment that is simply absent from storage is written as an empty
|
|
* entry, since a row outliving its file must not make export impossible.
|
|
*
|
|
* @param zip The archive to add the attachment to
|
|
* @param attachment The attachment to add
|
|
* @param pathInZip The path to store the attachment at within the archive
|
|
*/
|
|
protected addAttachmentToArchive(
|
|
zip: ZipFile,
|
|
attachment: Attachment,
|
|
pathInZip: string
|
|
) {
|
|
zip.addReadStreamLazy(
|
|
pathInZip,
|
|
{ mtime: attachment.updatedAt },
|
|
(callback) => {
|
|
attachment.stream.then(
|
|
(stream) => {
|
|
if (!stream) {
|
|
Logger.warn(`Attachment is missing from storage`, {
|
|
attachmentId: attachment.id,
|
|
teamId: attachment.teamId,
|
|
});
|
|
return callback(null, Readable.from([]));
|
|
}
|
|
|
|
// yazl does not watch the streams it is handed, so an interrupted
|
|
// read would otherwise stall the archive rather than fail it.
|
|
stream.on("error", (err) => zip.emit("error", toError(err)));
|
|
return callback(null, stream);
|
|
},
|
|
(err) => callback(toError(err), Readable.from([]))
|
|
);
|
|
}
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Notify the user that requested the export, in the app and by email. A
|
|
* failure is logged and does not change the result of the export.
|
|
*
|
|
* @param fileOperation The FileOperation that finished.
|
|
* @param user The user that requested the export.
|
|
*/
|
|
private async createNotification(fileOperation: FileOperation, user: User) {
|
|
if (!user.subscribedToEventType(NotificationEventType.ExportCompleted)) {
|
|
return;
|
|
}
|
|
|
|
try {
|
|
await Notification.create({
|
|
event: NotificationEventType.ExportCompleted,
|
|
userId: user.id,
|
|
actorId: user.id,
|
|
teamId: user.teamId,
|
|
fileOperationId: fileOperation.id,
|
|
// The source lets the notification be revoked if access is lost.
|
|
documentId: fileOperation.documentId ?? undefined,
|
|
collectionId: fileOperation.collectionId ?? undefined,
|
|
});
|
|
} catch (error) {
|
|
Logger.error("Failed to create export notification", toError(error), {
|
|
fileOperationId: fileOperation.id,
|
|
});
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Update the state of the underlying FileOperation in the database and send
|
|
* an event to the client.
|
|
*
|
|
* @param fileOperation The FileOperation to update
|
|
*/
|
|
private async updateFileOperation(
|
|
fileOperation: FileOperation,
|
|
options: Omit<Partial<FileOperation>, "error"> & { error?: Error }
|
|
) {
|
|
await fileOperation.update(
|
|
{
|
|
...options,
|
|
error: options.error
|
|
? truncate(options.error.message, { length: 255 })
|
|
: undefined,
|
|
},
|
|
{
|
|
hooks: false,
|
|
}
|
|
);
|
|
|
|
await Event.schedule({
|
|
name: "fileOperations.update",
|
|
modelId: fileOperation.id,
|
|
teamId: fileOperation.teamId,
|
|
actorId: fileOperation.userId,
|
|
data: fileOperationPresenter(fileOperation),
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Job options such as priority and retry strategy, as defined by Bull.
|
|
*/
|
|
public get options() {
|
|
return {
|
|
priority: TaskPriority.Background,
|
|
attempts: 1,
|
|
};
|
|
}
|
|
}
|