Files
outline/server/queues/tasks/ExportDocumentTreeTask.ts

475 lines
14 KiB
TypeScript

import path from "node:path";
import { escapeRegExp } from "es-toolkit/compat";
import type { ZipFile } from "yazl";
import { errToString } from "@shared/utils/error";
import type { NavigationNode } from "@shared/types";
import { FileOperationFormat } from "@shared/types";
import Logger from "@server/logging/Logger";
import type { Collection } from "@server/models";
import Attachment from "@server/models/Attachment";
import Document from "@server/models/Document";
import Team from "@server/models/Team";
import { DocumentHelper } from "@server/models/helpers/DocumentHelper";
import HTMLHelper from "@server/models/helpers/HTMLHelper";
import OKFHelper from "@server/models/helpers/OKFHelper";
import { ProsemirrorHelper } from "@server/models/helpers/ProsemirrorHelper";
import TextBundleHelper from "@server/models/helpers/TextBundleHelper";
import { sequelizeReadOnly } from "@server/storage/database";
import ZipHelper from "@server/utils/ZipHelper";
import { serializeFilename } from "@server/utils/fs";
import ExportTask from "./ExportTask";
export default abstract class ExportDocumentTreeTask extends ExportTask {
/**
* The extension given to each document's entry in the archive. For TextBundle
* this names a directory rather than a file.
*
* @param format The format being exported.
* @returns The extension, without a leading dot.
*/
private static extensionForFormat(format: FileOperationFormat): string {
switch (format) {
case FileOperationFormat.HTMLZip:
return "html";
case FileOperationFormat.TextBundleZip:
return TextBundleHelper.bundleExtension;
default:
return "md";
}
}
/**
* Exports the document tree to the given zip instance.
*
* @param zip The yazl ZipFile to add files to
* @param documentId The document ID to export
* @param pathInZip The path in the zip to add the document to. For TextBundle
* this is the bundle directory rather than a file.
* @param format The format to export in
*/
protected async processDocument({
zip,
pathInZip,
documentId,
format,
includeAttachments,
pathMap,
origin,
}: {
zip: ZipFile;
pathInZip: string;
documentId: string;
format: FileOperationFormat;
includeAttachments: boolean;
pathMap: Map<string, string>;
origin?: string;
}) {
Logger.debug("task", `Adding document to archive`, { documentId });
const result = await sequelizeReadOnly.transaction(async (transaction) => {
const document = await Document.findByPk(documentId, { transaction });
if (!document) {
return;
}
const attachmentIds = includeAttachments
? ProsemirrorHelper.parseAttachmentIds(
DocumentHelper.toProsemirror(document)
)
: [];
const attachments = attachmentIds.length
? await Attachment.findAll({
where: {
teamId: document.teamId,
id: attachmentIds,
},
transaction,
})
: [];
return { attachments, document };
});
if (!result) {
return;
}
const { attachments, document } = result;
let text =
format === FileOperationFormat.HTMLZip
? await DocumentHelper.toHTML(document, { centered: true })
: await DocumentHelper.toMarkdown(document, { commonMark: true });
const isTextBundle = format === FileOperationFormat.TextBundleZip;
// A TextBundle is a directory, so its text and assets sit inside the path
// reserved for the document rather than beside it.
const textPathInZip = isTextBundle
? path.join(pathInZip, TextBundleHelper.textFileName)
: pathInZip;
const usedAssetNames = new Set<string>();
// Add any referenced attachments to the zip file and replace the
// reference in the document with the path to the attachment in the zip
for (const attachment of attachments) {
Logger.debug("task", `Adding attachment to archive`, {
documentId,
key: attachment.key,
});
// Skip attachments with a malformed key that has no filename component,
// as yazl rejects entries whose path ends with a slash.
if (!attachment.key || attachment.key.endsWith("/")) {
Logger.warn(`Skipping attachment with invalid key`, {
attachmentId: attachment.id,
teamId: attachment.teamId,
key: attachment.key,
});
continue;
}
const dir = path.dirname(pathInZip);
// Inline small images referenced a single time as base64 data URIs
// rather than writing an external file. PDF export renders from HTML too.
// Only these are read into memory; the rest are streamed into the archive.
if (
(format === FileOperationFormat.HTMLZip ||
format === FileOperationFormat.PDF) &&
HTMLHelper.canInlineImage(
text,
attachment.redirectUrl,
attachment.contentType,
attachment.size
)
) {
let buffer: Buffer | undefined;
try {
buffer = await attachment.buffer;
} catch (err) {
Logger.warn(`Failed to read attachment from storage`, {
attachmentId: attachment.id,
teamId: attachment.teamId,
error: errToString(err),
});
}
const inlined = buffer
? HTMLHelper.inlineImage(
text,
attachment.redirectUrl,
attachment.contentType,
buffer
)
: null;
if (inlined !== null) {
text = inlined;
continue;
}
}
// TextBundle requires assets to live in the bundle's own assets folder,
// referenced relative to the text file, rather than at their storage key.
const reference = isTextBundle
? TextBundleHelper.assetPath(attachment.name, usedAssetNames)
: attachment.key;
this.addAttachmentToArchive(
zip,
attachment,
isTextBundle
? path.join(pathInZip, reference)
: path.join(dir, reference)
);
text = text.replace(
new RegExp(escapeRegExp(attachment.redirectUrl), "g"),
encodeURI(reference)
);
}
// Replace any internal links with relative paths to the document in the zip
const internalLinks = [
...text.matchAll(/\/doc\/(?:[0-9a-zA-Z-_~]*-)?([a-zA-Z0-9]{10,15})/g),
];
internalLinks.forEach((match) => {
const matchedLink = match[0];
const matchedDocPath = pathMap.get(matchedLink);
if (matchedDocPath) {
const relativePath = path.relative(textPathInZip, matchedDocPath);
if (relativePath.startsWith(".")) {
text = text.replace(
matchedLink,
encodeURI(relativePath.substring(1))
);
}
}
});
if (isTextBundle) {
zip.addBuffer(
Buffer.from(TextBundleHelper.info(document)),
path.join(pathInZip, TextBundleHelper.infoFileName),
{ mtime: document.updatedAt }
);
}
// OKF frontmatter carries the document's canonical URL, so it is added
// only after internal links have been rewritten to relative paths.
if (format === FileOperationFormat.OKFZip && origin) {
text = OKFHelper.frontmatter(document, origin) + text;
}
// Finally, add the document to the zip file
zip.addBuffer(Buffer.from(text), textPathInZip, {
mtime: document.updatedAt,
fileComment: JSON.stringify({
createdAt: document.createdAt,
updatedAt: document.updatedAt,
}),
});
}
/**
* Exports the documents and attachments in the given collections to a zip file
* and returns the path to the zip file in tmp.
*
* @param zip The yazl ZipFile to add files to
* @param collections The collections to export
* @param format The format to export in
* @param includeAttachments Whether to include attachments in the export
*
* @returns The path to the zip file in tmp.
*/
protected async addCollectionsToArchive(
collections: Collection[],
format: FileOperationFormat,
includeAttachments = true
) {
const { pathMap, roots } = this.createPathMap(collections, format);
const origin = await this.originForFormat(format, collections[0]?.teamId);
return await ZipHelper.toTmpFile(async (zip) => {
await this.addDocumentsToArchive({
zip,
pathMap,
format,
includeAttachments,
origin,
});
if (format === FileOperationFormat.OKFZip) {
zip.addBuffer(
Buffer.from(
OKFHelper.rootIndex(
"Collections",
roots.map(({ root, collection }) => ({
title: collection.name,
path: `${root}/`,
description: collection.description,
}))
)
),
OKFHelper.indexFileName
);
}
});
}
protected async addDocumentToArchive({
document,
format,
documentStructure,
includeAttachments = true,
}: {
document: Document;
format: FileOperationFormat;
documentStructure: NavigationNode[];
includeAttachments?: boolean;
}) {
const pathMap = new Map<string, string>();
const rootFolderName = serializeFilename(document.displayTitle);
const rootPath = `${rootFolderName}.${ExportDocumentTreeTask.extensionForFormat(format)}`;
// entry for root document
pathMap.set(document.path, rootPath);
this.addDocumentTreeToPathMap(
pathMap,
documentStructure,
rootFolderName,
format
);
const origin = await this.originForFormat(format, document.teamId);
return await ZipHelper.toTmpFile(async (zip) => {
await this.addDocumentsToArchive({
zip,
pathMap,
format,
includeAttachments,
origin,
});
if (format === FileOperationFormat.OKFZip) {
zip.addBuffer(
Buffer.from(
OKFHelper.rootIndex("Documents", [
{
title: document.displayTitle,
path: rootPath,
description: document.getSummary(),
},
])
),
OKFHelper.indexFileName
);
}
});
}
/**
* Resolves the origin used for canonical document URLs, which only the OKF
* format writes into its output.
*
* @param format The format to export in
* @param teamId The team that owns the exported documents
* @returns The team's URL origin, or undefined when the format has no use for it.
*/
private async originForFormat(
format: FileOperationFormat,
teamId?: string
): Promise<string | undefined> {
if (format !== FileOperationFormat.OKFZip || !teamId) {
return undefined;
}
const team = await Team.findByPk(teamId, { rejectOnEmpty: true });
return team.url;
}
/**
* Processes each unique document in the path map and adds it to the zip.
*
* @param zip The yazl ZipFile to add files to
* @param pathMap Map of document urls to their path in the zip
* @param format The format to export in
* @param includeAttachments Whether to include attachments in the export
* @param origin The origin for canonical document URLs, when the format needs one
*/
private async addDocumentsToArchive({
zip,
pathMap,
format,
includeAttachments,
origin,
}: {
zip: ZipFile;
pathMap: Map<string, string>;
format: FileOperationFormat;
includeAttachments: boolean;
origin?: string;
}) {
const processedPaths = new Set<string>();
Logger.debug("task", `Start adding documents to archive`);
for (const [url, pathInZip] of pathMap) {
// A document may be keyed by multiple urls in the path map, only
// process each file in the zip once.
if (processedPaths.has(pathInZip)) {
continue;
}
processedPaths.add(pathInZip);
await this.processDocument({
zip,
pathInZip,
documentId: url.replace("/doc/", ""),
includeAttachments,
format,
pathMap,
origin,
});
}
Logger.debug("task", "Completed adding documents to archive");
}
/**
* Generates a map of document urls to their path in the zip file.
*
* @param collections The collections to generate the path map for.
* @param format The format of the exported documents.
* @returns The path map and the root directory chosen for each collection.
*/
private createPathMap(
collections: Collection[],
format: FileOperationFormat
) {
const pathMap = new Map<string, string>();
const roots: { root: string; collection: Collection }[] = [];
for (const collection of collections) {
if (collection.documentStructure) {
let root = serializeFilename(collection.name);
let i = 0;
while (roots.some((entry) => entry.root === root)) {
root = `${serializeFilename(collection.name)} (${++i})`;
}
roots.push({ root, collection });
this.addDocumentTreeToPathMap(
pathMap,
collection.documentStructure,
root,
format
);
}
}
return { pathMap, roots };
}
private addDocumentTreeToPathMap(
map: Map<string, string>,
nodes: NavigationNode[],
root: string,
format: FileOperationFormat
) {
for (const node of nodes) {
const title = serializeFilename(node.title) || "Untitled";
const extension = ExportDocumentTreeTask.extensionForFormat(format);
// Ensure the document is given a unique path in zip, even if it has
// the same title as another document in the same collection. OKF also
// reserves some file names, which a document may not take.
let i = 0;
let filePath = path.join(root, `${title}.${extension}`);
while (
Array.from(map.values()).includes(filePath) ||
(format === FileOperationFormat.OKFZip &&
OKFHelper.isReservedFileName(path.basename(filePath)))
) {
filePath = path.join(root, `${title} (${++i}).${extension}`);
}
map.set(node.url, filePath);
// If this is an imported document, the references to this doc are in the 'doc/{docId}' format.
// Set this format to replace them with relative URLs in the zip.
map.set(`/doc/${node.id}`, filePath);
if (node.children?.length) {
this.addDocumentTreeToPathMap(
map,
node.children,
path.join(root, title),
format
);
}
}
}
}