mirror of
https://github.com/outline/outline.git
synced 2026-11-20 21:23:06 +03:00
475 lines
14 KiB
TypeScript
475 lines
14 KiB
TypeScript
import path from "node:path";
|
|
import { escapeRegExp } from "es-toolkit/compat";
|
|
import type { ZipFile } from "yazl";
|
|
import { errToString } from "@shared/utils/error";
|
|
import type { NavigationNode } from "@shared/types";
|
|
import { FileOperationFormat } from "@shared/types";
|
|
import Logger from "@server/logging/Logger";
|
|
import type { Collection } from "@server/models";
|
|
import Attachment from "@server/models/Attachment";
|
|
import Document from "@server/models/Document";
|
|
import Team from "@server/models/Team";
|
|
import { DocumentHelper } from "@server/models/helpers/DocumentHelper";
|
|
import HTMLHelper from "@server/models/helpers/HTMLHelper";
|
|
import OKFHelper from "@server/models/helpers/OKFHelper";
|
|
import { ProsemirrorHelper } from "@server/models/helpers/ProsemirrorHelper";
|
|
import TextBundleHelper from "@server/models/helpers/TextBundleHelper";
|
|
import { sequelizeReadOnly } from "@server/storage/database";
|
|
import ZipHelper from "@server/utils/ZipHelper";
|
|
import { serializeFilename } from "@server/utils/fs";
|
|
import ExportTask from "./ExportTask";
|
|
|
|
export default abstract class ExportDocumentTreeTask extends ExportTask {
|
|
/**
|
|
* The extension given to each document's entry in the archive. For TextBundle
|
|
* this names a directory rather than a file.
|
|
*
|
|
* @param format The format being exported.
|
|
* @returns The extension, without a leading dot.
|
|
*/
|
|
private static extensionForFormat(format: FileOperationFormat): string {
|
|
switch (format) {
|
|
case FileOperationFormat.HTMLZip:
|
|
return "html";
|
|
case FileOperationFormat.TextBundleZip:
|
|
return TextBundleHelper.bundleExtension;
|
|
default:
|
|
return "md";
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Exports the document tree to the given zip instance.
|
|
*
|
|
* @param zip The yazl ZipFile to add files to
|
|
* @param documentId The document ID to export
|
|
* @param pathInZip The path in the zip to add the document to. For TextBundle
|
|
* this is the bundle directory rather than a file.
|
|
* @param format The format to export in
|
|
*/
|
|
protected async processDocument({
|
|
zip,
|
|
pathInZip,
|
|
documentId,
|
|
format,
|
|
includeAttachments,
|
|
pathMap,
|
|
origin,
|
|
}: {
|
|
zip: ZipFile;
|
|
pathInZip: string;
|
|
documentId: string;
|
|
format: FileOperationFormat;
|
|
includeAttachments: boolean;
|
|
pathMap: Map<string, string>;
|
|
origin?: string;
|
|
}) {
|
|
Logger.debug("task", `Adding document to archive`, { documentId });
|
|
const result = await sequelizeReadOnly.transaction(async (transaction) => {
|
|
const document = await Document.findByPk(documentId, { transaction });
|
|
if (!document) {
|
|
return;
|
|
}
|
|
|
|
const attachmentIds = includeAttachments
|
|
? ProsemirrorHelper.parseAttachmentIds(
|
|
DocumentHelper.toProsemirror(document)
|
|
)
|
|
: [];
|
|
const attachments = attachmentIds.length
|
|
? await Attachment.findAll({
|
|
where: {
|
|
teamId: document.teamId,
|
|
id: attachmentIds,
|
|
},
|
|
transaction,
|
|
})
|
|
: [];
|
|
|
|
return { attachments, document };
|
|
});
|
|
|
|
if (!result) {
|
|
return;
|
|
}
|
|
|
|
const { attachments, document } = result;
|
|
|
|
let text =
|
|
format === FileOperationFormat.HTMLZip
|
|
? await DocumentHelper.toHTML(document, { centered: true })
|
|
: await DocumentHelper.toMarkdown(document, { commonMark: true });
|
|
|
|
const isTextBundle = format === FileOperationFormat.TextBundleZip;
|
|
|
|
// A TextBundle is a directory, so its text and assets sit inside the path
|
|
// reserved for the document rather than beside it.
|
|
const textPathInZip = isTextBundle
|
|
? path.join(pathInZip, TextBundleHelper.textFileName)
|
|
: pathInZip;
|
|
const usedAssetNames = new Set<string>();
|
|
|
|
// Add any referenced attachments to the zip file and replace the
|
|
// reference in the document with the path to the attachment in the zip
|
|
for (const attachment of attachments) {
|
|
Logger.debug("task", `Adding attachment to archive`, {
|
|
documentId,
|
|
key: attachment.key,
|
|
});
|
|
|
|
// Skip attachments with a malformed key that has no filename component,
|
|
// as yazl rejects entries whose path ends with a slash.
|
|
if (!attachment.key || attachment.key.endsWith("/")) {
|
|
Logger.warn(`Skipping attachment with invalid key`, {
|
|
attachmentId: attachment.id,
|
|
teamId: attachment.teamId,
|
|
key: attachment.key,
|
|
});
|
|
continue;
|
|
}
|
|
|
|
const dir = path.dirname(pathInZip);
|
|
|
|
// Inline small images referenced a single time as base64 data URIs
|
|
// rather than writing an external file. PDF export renders from HTML too.
|
|
// Only these are read into memory; the rest are streamed into the archive.
|
|
if (
|
|
(format === FileOperationFormat.HTMLZip ||
|
|
format === FileOperationFormat.PDF) &&
|
|
HTMLHelper.canInlineImage(
|
|
text,
|
|
attachment.redirectUrl,
|
|
attachment.contentType,
|
|
attachment.size
|
|
)
|
|
) {
|
|
let buffer: Buffer | undefined;
|
|
try {
|
|
buffer = await attachment.buffer;
|
|
} catch (err) {
|
|
Logger.warn(`Failed to read attachment from storage`, {
|
|
attachmentId: attachment.id,
|
|
teamId: attachment.teamId,
|
|
error: errToString(err),
|
|
});
|
|
}
|
|
|
|
const inlined = buffer
|
|
? HTMLHelper.inlineImage(
|
|
text,
|
|
attachment.redirectUrl,
|
|
attachment.contentType,
|
|
buffer
|
|
)
|
|
: null;
|
|
if (inlined !== null) {
|
|
text = inlined;
|
|
continue;
|
|
}
|
|
}
|
|
|
|
// TextBundle requires assets to live in the bundle's own assets folder,
|
|
// referenced relative to the text file, rather than at their storage key.
|
|
const reference = isTextBundle
|
|
? TextBundleHelper.assetPath(attachment.name, usedAssetNames)
|
|
: attachment.key;
|
|
|
|
this.addAttachmentToArchive(
|
|
zip,
|
|
attachment,
|
|
isTextBundle
|
|
? path.join(pathInZip, reference)
|
|
: path.join(dir, reference)
|
|
);
|
|
|
|
text = text.replace(
|
|
new RegExp(escapeRegExp(attachment.redirectUrl), "g"),
|
|
encodeURI(reference)
|
|
);
|
|
}
|
|
|
|
// Replace any internal links with relative paths to the document in the zip
|
|
const internalLinks = [
|
|
...text.matchAll(/\/doc\/(?:[0-9a-zA-Z-_~]*-)?([a-zA-Z0-9]{10,15})/g),
|
|
];
|
|
internalLinks.forEach((match) => {
|
|
const matchedLink = match[0];
|
|
const matchedDocPath = pathMap.get(matchedLink);
|
|
|
|
if (matchedDocPath) {
|
|
const relativePath = path.relative(textPathInZip, matchedDocPath);
|
|
if (relativePath.startsWith(".")) {
|
|
text = text.replace(
|
|
matchedLink,
|
|
encodeURI(relativePath.substring(1))
|
|
);
|
|
}
|
|
}
|
|
});
|
|
|
|
if (isTextBundle) {
|
|
zip.addBuffer(
|
|
Buffer.from(TextBundleHelper.info(document)),
|
|
path.join(pathInZip, TextBundleHelper.infoFileName),
|
|
{ mtime: document.updatedAt }
|
|
);
|
|
}
|
|
|
|
// OKF frontmatter carries the document's canonical URL, so it is added
|
|
// only after internal links have been rewritten to relative paths.
|
|
if (format === FileOperationFormat.OKFZip && origin) {
|
|
text = OKFHelper.frontmatter(document, origin) + text;
|
|
}
|
|
|
|
// Finally, add the document to the zip file
|
|
zip.addBuffer(Buffer.from(text), textPathInZip, {
|
|
mtime: document.updatedAt,
|
|
fileComment: JSON.stringify({
|
|
createdAt: document.createdAt,
|
|
updatedAt: document.updatedAt,
|
|
}),
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Exports the documents and attachments in the given collections to a zip file
|
|
* and returns the path to the zip file in tmp.
|
|
*
|
|
* @param zip The yazl ZipFile to add files to
|
|
* @param collections The collections to export
|
|
* @param format The format to export in
|
|
* @param includeAttachments Whether to include attachments in the export
|
|
*
|
|
* @returns The path to the zip file in tmp.
|
|
*/
|
|
protected async addCollectionsToArchive(
|
|
collections: Collection[],
|
|
format: FileOperationFormat,
|
|
includeAttachments = true
|
|
) {
|
|
const { pathMap, roots } = this.createPathMap(collections, format);
|
|
const origin = await this.originForFormat(format, collections[0]?.teamId);
|
|
return await ZipHelper.toTmpFile(async (zip) => {
|
|
await this.addDocumentsToArchive({
|
|
zip,
|
|
pathMap,
|
|
format,
|
|
includeAttachments,
|
|
origin,
|
|
});
|
|
|
|
if (format === FileOperationFormat.OKFZip) {
|
|
zip.addBuffer(
|
|
Buffer.from(
|
|
OKFHelper.rootIndex(
|
|
"Collections",
|
|
roots.map(({ root, collection }) => ({
|
|
title: collection.name,
|
|
path: `${root}/`,
|
|
description: collection.description,
|
|
}))
|
|
)
|
|
),
|
|
OKFHelper.indexFileName
|
|
);
|
|
}
|
|
});
|
|
}
|
|
|
|
protected async addDocumentToArchive({
|
|
document,
|
|
format,
|
|
documentStructure,
|
|
includeAttachments = true,
|
|
}: {
|
|
document: Document;
|
|
format: FileOperationFormat;
|
|
documentStructure: NavigationNode[];
|
|
includeAttachments?: boolean;
|
|
}) {
|
|
const pathMap = new Map<string, string>();
|
|
|
|
const rootFolderName = serializeFilename(document.displayTitle);
|
|
const rootPath = `${rootFolderName}.${ExportDocumentTreeTask.extensionForFormat(format)}`;
|
|
|
|
// entry for root document
|
|
pathMap.set(document.path, rootPath);
|
|
|
|
this.addDocumentTreeToPathMap(
|
|
pathMap,
|
|
documentStructure,
|
|
rootFolderName,
|
|
format
|
|
);
|
|
|
|
const origin = await this.originForFormat(format, document.teamId);
|
|
return await ZipHelper.toTmpFile(async (zip) => {
|
|
await this.addDocumentsToArchive({
|
|
zip,
|
|
pathMap,
|
|
format,
|
|
includeAttachments,
|
|
origin,
|
|
});
|
|
|
|
if (format === FileOperationFormat.OKFZip) {
|
|
zip.addBuffer(
|
|
Buffer.from(
|
|
OKFHelper.rootIndex("Documents", [
|
|
{
|
|
title: document.displayTitle,
|
|
path: rootPath,
|
|
description: document.getSummary(),
|
|
},
|
|
])
|
|
),
|
|
OKFHelper.indexFileName
|
|
);
|
|
}
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Resolves the origin used for canonical document URLs, which only the OKF
|
|
* format writes into its output.
|
|
*
|
|
* @param format The format to export in
|
|
* @param teamId The team that owns the exported documents
|
|
* @returns The team's URL origin, or undefined when the format has no use for it.
|
|
*/
|
|
private async originForFormat(
|
|
format: FileOperationFormat,
|
|
teamId?: string
|
|
): Promise<string | undefined> {
|
|
if (format !== FileOperationFormat.OKFZip || !teamId) {
|
|
return undefined;
|
|
}
|
|
|
|
const team = await Team.findByPk(teamId, { rejectOnEmpty: true });
|
|
return team.url;
|
|
}
|
|
|
|
/**
|
|
* Processes each unique document in the path map and adds it to the zip.
|
|
*
|
|
* @param zip The yazl ZipFile to add files to
|
|
* @param pathMap Map of document urls to their path in the zip
|
|
* @param format The format to export in
|
|
* @param includeAttachments Whether to include attachments in the export
|
|
* @param origin The origin for canonical document URLs, when the format needs one
|
|
*/
|
|
private async addDocumentsToArchive({
|
|
zip,
|
|
pathMap,
|
|
format,
|
|
includeAttachments,
|
|
origin,
|
|
}: {
|
|
zip: ZipFile;
|
|
pathMap: Map<string, string>;
|
|
format: FileOperationFormat;
|
|
includeAttachments: boolean;
|
|
origin?: string;
|
|
}) {
|
|
const processedPaths = new Set<string>();
|
|
|
|
Logger.debug("task", `Start adding documents to archive`);
|
|
|
|
for (const [url, pathInZip] of pathMap) {
|
|
// A document may be keyed by multiple urls in the path map, only
|
|
// process each file in the zip once.
|
|
if (processedPaths.has(pathInZip)) {
|
|
continue;
|
|
}
|
|
processedPaths.add(pathInZip);
|
|
|
|
await this.processDocument({
|
|
zip,
|
|
pathInZip,
|
|
documentId: url.replace("/doc/", ""),
|
|
includeAttachments,
|
|
format,
|
|
pathMap,
|
|
origin,
|
|
});
|
|
}
|
|
|
|
Logger.debug("task", "Completed adding documents to archive");
|
|
}
|
|
|
|
/**
|
|
* Generates a map of document urls to their path in the zip file.
|
|
*
|
|
* @param collections The collections to generate the path map for.
|
|
* @param format The format of the exported documents.
|
|
* @returns The path map and the root directory chosen for each collection.
|
|
*/
|
|
private createPathMap(
|
|
collections: Collection[],
|
|
format: FileOperationFormat
|
|
) {
|
|
const pathMap = new Map<string, string>();
|
|
const roots: { root: string; collection: Collection }[] = [];
|
|
|
|
for (const collection of collections) {
|
|
if (collection.documentStructure) {
|
|
let root = serializeFilename(collection.name);
|
|
let i = 0;
|
|
while (roots.some((entry) => entry.root === root)) {
|
|
root = `${serializeFilename(collection.name)} (${++i})`;
|
|
}
|
|
roots.push({ root, collection });
|
|
|
|
this.addDocumentTreeToPathMap(
|
|
pathMap,
|
|
collection.documentStructure,
|
|
root,
|
|
format
|
|
);
|
|
}
|
|
}
|
|
|
|
return { pathMap, roots };
|
|
}
|
|
|
|
private addDocumentTreeToPathMap(
|
|
map: Map<string, string>,
|
|
nodes: NavigationNode[],
|
|
root: string,
|
|
format: FileOperationFormat
|
|
) {
|
|
for (const node of nodes) {
|
|
const title = serializeFilename(node.title) || "Untitled";
|
|
const extension = ExportDocumentTreeTask.extensionForFormat(format);
|
|
|
|
// Ensure the document is given a unique path in zip, even if it has
|
|
// the same title as another document in the same collection. OKF also
|
|
// reserves some file names, which a document may not take.
|
|
let i = 0;
|
|
let filePath = path.join(root, `${title}.${extension}`);
|
|
while (
|
|
Array.from(map.values()).includes(filePath) ||
|
|
(format === FileOperationFormat.OKFZip &&
|
|
OKFHelper.isReservedFileName(path.basename(filePath)))
|
|
) {
|
|
filePath = path.join(root, `${title} (${++i}).${extension}`);
|
|
}
|
|
|
|
map.set(node.url, filePath);
|
|
|
|
// If this is an imported document, the references to this doc are in the 'doc/{docId}' format.
|
|
// Set this format to replace them with relative URLs in the zip.
|
|
map.set(`/doc/${node.id}`, filePath);
|
|
|
|
if (node.children?.length) {
|
|
this.addDocumentTreeToPathMap(
|
|
map,
|
|
node.children,
|
|
path.join(root, title),
|
|
format
|
|
);
|
|
}
|
|
}
|
|
}
|
|
}
|