import path from "node:path"; import { escapeRegExp } from "es-toolkit/compat"; import type { ZipFile } from "yazl"; import { errToString } from "@shared/utils/error"; import type { NavigationNode } from "@shared/types"; import { FileOperationFormat } from "@shared/types"; import Logger from "@server/logging/Logger"; import type { Collection } from "@server/models"; import Attachment from "@server/models/Attachment"; import Document from "@server/models/Document"; import Team from "@server/models/Team"; import { DocumentHelper } from "@server/models/helpers/DocumentHelper"; import HTMLHelper from "@server/models/helpers/HTMLHelper"; import OKFHelper from "@server/models/helpers/OKFHelper"; import { ProsemirrorHelper } from "@server/models/helpers/ProsemirrorHelper"; import TextBundleHelper from "@server/models/helpers/TextBundleHelper"; import { sequelizeReadOnly } from "@server/storage/database"; import ZipHelper from "@server/utils/ZipHelper"; import { serializeFilename } from "@server/utils/fs"; import ExportTask from "./ExportTask"; export default abstract class ExportDocumentTreeTask extends ExportTask { /** * The extension given to each document's entry in the archive. For TextBundle * this names a directory rather than a file. * * @param format The format being exported. * @returns The extension, without a leading dot. */ private static extensionForFormat(format: FileOperationFormat): string { switch (format) { case FileOperationFormat.HTMLZip: return "html"; case FileOperationFormat.TextBundleZip: return TextBundleHelper.bundleExtension; default: return "md"; } } /** * Exports the document tree to the given zip instance. * * @param zip The yazl ZipFile to add files to * @param documentId The document ID to export * @param pathInZip The path in the zip to add the document to. For TextBundle * this is the bundle directory rather than a file. * @param format The format to export in */ protected async processDocument({ zip, pathInZip, documentId, format, includeAttachments, pathMap, origin, }: { zip: ZipFile; pathInZip: string; documentId: string; format: FileOperationFormat; includeAttachments: boolean; pathMap: Map; origin?: string; }) { Logger.debug("task", `Adding document to archive`, { documentId }); const result = await sequelizeReadOnly.transaction(async (transaction) => { const document = await Document.findByPk(documentId, { transaction }); if (!document) { return; } const attachmentIds = includeAttachments ? ProsemirrorHelper.parseAttachmentIds( DocumentHelper.toProsemirror(document) ) : []; const attachments = attachmentIds.length ? await Attachment.findAll({ where: { teamId: document.teamId, id: attachmentIds, }, transaction, }) : []; return { attachments, document }; }); if (!result) { return; } const { attachments, document } = result; let text = format === FileOperationFormat.HTMLZip ? await DocumentHelper.toHTML(document, { centered: true }) : await DocumentHelper.toMarkdown(document, { commonMark: true }); const isTextBundle = format === FileOperationFormat.TextBundleZip; // A TextBundle is a directory, so its text and assets sit inside the path // reserved for the document rather than beside it. const textPathInZip = isTextBundle ? path.join(pathInZip, TextBundleHelper.textFileName) : pathInZip; const usedAssetNames = new Set(); // Add any referenced attachments to the zip file and replace the // reference in the document with the path to the attachment in the zip for (const attachment of attachments) { Logger.debug("task", `Adding attachment to archive`, { documentId, key: attachment.key, }); // Skip attachments with a malformed key that has no filename component, // as yazl rejects entries whose path ends with a slash. if (!attachment.key || attachment.key.endsWith("/")) { Logger.warn(`Skipping attachment with invalid key`, { attachmentId: attachment.id, teamId: attachment.teamId, key: attachment.key, }); continue; } const dir = path.dirname(pathInZip); // Inline small images referenced a single time as base64 data URIs // rather than writing an external file. PDF export renders from HTML too. // Only these are read into memory; the rest are streamed into the archive. if ( (format === FileOperationFormat.HTMLZip || format === FileOperationFormat.PDF) && HTMLHelper.canInlineImage( text, attachment.redirectUrl, attachment.contentType, attachment.size ) ) { let buffer: Buffer | undefined; try { buffer = await attachment.buffer; } catch (err) { Logger.warn(`Failed to read attachment from storage`, { attachmentId: attachment.id, teamId: attachment.teamId, error: errToString(err), }); } const inlined = buffer ? HTMLHelper.inlineImage( text, attachment.redirectUrl, attachment.contentType, buffer ) : null; if (inlined !== null) { text = inlined; continue; } } // TextBundle requires assets to live in the bundle's own assets folder, // referenced relative to the text file, rather than at their storage key. const reference = isTextBundle ? TextBundleHelper.assetPath(attachment.name, usedAssetNames) : attachment.key; this.addAttachmentToArchive( zip, attachment, isTextBundle ? path.join(pathInZip, reference) : path.join(dir, reference) ); text = text.replace( new RegExp(escapeRegExp(attachment.redirectUrl), "g"), encodeURI(reference) ); } // Replace any internal links with relative paths to the document in the zip const internalLinks = [ ...text.matchAll(/\/doc\/(?:[0-9a-zA-Z-_~]*-)?([a-zA-Z0-9]{10,15})/g), ]; internalLinks.forEach((match) => { const matchedLink = match[0]; const matchedDocPath = pathMap.get(matchedLink); if (matchedDocPath) { const relativePath = path.relative(textPathInZip, matchedDocPath); if (relativePath.startsWith(".")) { text = text.replace( matchedLink, encodeURI(relativePath.substring(1)) ); } } }); if (isTextBundle) { zip.addBuffer( Buffer.from(TextBundleHelper.info(document)), path.join(pathInZip, TextBundleHelper.infoFileName), { mtime: document.updatedAt } ); } // OKF frontmatter carries the document's canonical URL, so it is added // only after internal links have been rewritten to relative paths. if (format === FileOperationFormat.OKFZip && origin) { text = OKFHelper.frontmatter(document, origin) + text; } // Finally, add the document to the zip file zip.addBuffer(Buffer.from(text), textPathInZip, { mtime: document.updatedAt, fileComment: JSON.stringify({ createdAt: document.createdAt, updatedAt: document.updatedAt, }), }); } /** * Exports the documents and attachments in the given collections to a zip file * and returns the path to the zip file in tmp. * * @param zip The yazl ZipFile to add files to * @param collections The collections to export * @param format The format to export in * @param includeAttachments Whether to include attachments in the export * * @returns The path to the zip file in tmp. */ protected async addCollectionsToArchive( collections: Collection[], format: FileOperationFormat, includeAttachments = true ) { const { pathMap, roots } = this.createPathMap(collections, format); const origin = await this.originForFormat(format, collections[0]?.teamId); return await ZipHelper.toTmpFile(async (zip) => { await this.addDocumentsToArchive({ zip, pathMap, format, includeAttachments, origin, }); if (format === FileOperationFormat.OKFZip) { zip.addBuffer( Buffer.from( OKFHelper.rootIndex( "Collections", roots.map(({ root, collection }) => ({ title: collection.name, path: `${root}/`, description: collection.description, })) ) ), OKFHelper.indexFileName ); } }); } protected async addDocumentToArchive({ document, format, documentStructure, includeAttachments = true, }: { document: Document; format: FileOperationFormat; documentStructure: NavigationNode[]; includeAttachments?: boolean; }) { const pathMap = new Map(); const rootFolderName = serializeFilename(document.displayTitle); const rootPath = `${rootFolderName}.${ExportDocumentTreeTask.extensionForFormat(format)}`; // entry for root document pathMap.set(document.path, rootPath); this.addDocumentTreeToPathMap( pathMap, documentStructure, rootFolderName, format ); const origin = await this.originForFormat(format, document.teamId); return await ZipHelper.toTmpFile(async (zip) => { await this.addDocumentsToArchive({ zip, pathMap, format, includeAttachments, origin, }); if (format === FileOperationFormat.OKFZip) { zip.addBuffer( Buffer.from( OKFHelper.rootIndex("Documents", [ { title: document.displayTitle, path: rootPath, description: document.getSummary(), }, ]) ), OKFHelper.indexFileName ); } }); } /** * Resolves the origin used for canonical document URLs, which only the OKF * format writes into its output. * * @param format The format to export in * @param teamId The team that owns the exported documents * @returns The team's URL origin, or undefined when the format has no use for it. */ private async originForFormat( format: FileOperationFormat, teamId?: string ): Promise { if (format !== FileOperationFormat.OKFZip || !teamId) { return undefined; } const team = await Team.findByPk(teamId, { rejectOnEmpty: true }); return team.url; } /** * Processes each unique document in the path map and adds it to the zip. * * @param zip The yazl ZipFile to add files to * @param pathMap Map of document urls to their path in the zip * @param format The format to export in * @param includeAttachments Whether to include attachments in the export * @param origin The origin for canonical document URLs, when the format needs one */ private async addDocumentsToArchive({ zip, pathMap, format, includeAttachments, origin, }: { zip: ZipFile; pathMap: Map; format: FileOperationFormat; includeAttachments: boolean; origin?: string; }) { const processedPaths = new Set(); Logger.debug("task", `Start adding documents to archive`); for (const [url, pathInZip] of pathMap) { // A document may be keyed by multiple urls in the path map, only // process each file in the zip once. if (processedPaths.has(pathInZip)) { continue; } processedPaths.add(pathInZip); await this.processDocument({ zip, pathInZip, documentId: url.replace("/doc/", ""), includeAttachments, format, pathMap, origin, }); } Logger.debug("task", "Completed adding documents to archive"); } /** * Generates a map of document urls to their path in the zip file. * * @param collections The collections to generate the path map for. * @param format The format of the exported documents. * @returns The path map and the root directory chosen for each collection. */ private createPathMap( collections: Collection[], format: FileOperationFormat ) { const pathMap = new Map(); const roots: { root: string; collection: Collection }[] = []; for (const collection of collections) { if (collection.documentStructure) { let root = serializeFilename(collection.name); let i = 0; while (roots.some((entry) => entry.root === root)) { root = `${serializeFilename(collection.name)} (${++i})`; } roots.push({ root, collection }); this.addDocumentTreeToPathMap( pathMap, collection.documentStructure, root, format ); } } return { pathMap, roots }; } private addDocumentTreeToPathMap( map: Map, nodes: NavigationNode[], root: string, format: FileOperationFormat ) { for (const node of nodes) { const title = serializeFilename(node.title) || "Untitled"; const extension = ExportDocumentTreeTask.extensionForFormat(format); // Ensure the document is given a unique path in zip, even if it has // the same title as another document in the same collection. OKF also // reserves some file names, which a document may not take. let i = 0; let filePath = path.join(root, `${title}.${extension}`); while ( Array.from(map.values()).includes(filePath) || (format === FileOperationFormat.OKFZip && OKFHelper.isReservedFileName(path.basename(filePath))) ) { filePath = path.join(root, `${title} (${++i}).${extension}`); } map.set(node.url, filePath); // If this is an imported document, the references to this doc are in the 'doc/{docId}' format. // Set this format to replace them with relative URLs in the zip. map.set(`/doc/${node.id}`, filePath); if (node.children?.length) { this.addDocumentTreeToPathMap( map, node.children, path.join(root, title), format ); } } } }