mirror of
https://github.com/outline/outline.git
synced 2026-11-20 21:23:06 +03:00
864 lines
30 KiB
TypeScript
864 lines
30 KiB
TypeScript
import path from "node:path";
|
|
import { randomUUID } from "node:crypto";
|
|
import { escapeRegExp, truncate } from "es-toolkit/compat";
|
|
import mime from "mime-types";
|
|
import type { Node } from "prosemirror-model";
|
|
import { UniqueConstraintError } from "sequelize";
|
|
import { replaceMarkdownLinks } from "@shared/utils/markdown";
|
|
import {
|
|
CollectionValidation,
|
|
DocumentValidation,
|
|
ImportValidation,
|
|
} from "@shared/validations";
|
|
import type {
|
|
ImportTaskInput,
|
|
ImportTaskOutput,
|
|
MarkdownAttachmentManifestItem,
|
|
MarkdownPageImportTaskInputItem,
|
|
} from "@shared/schema";
|
|
import type { IntegrationService, ProsemirrorDoc } from "@shared/types";
|
|
import { AttachmentPreset } from "@shared/types";
|
|
import attachmentCreator from "@server/commands/attachmentCreator";
|
|
import { createContext } from "@server/context";
|
|
import env from "@server/env";
|
|
import { FileImportError } from "@server/errors";
|
|
import Logger from "@server/logging/Logger";
|
|
import type { ImportTask } from "@server/models";
|
|
import { Attachment } from "@server/models";
|
|
import AttachmentHelper, {
|
|
Buckets,
|
|
} from "@server/models/helpers/AttachmentHelper";
|
|
import { ProsemirrorDataHelper } from "@shared/utils/ProsemirrorDataHelper";
|
|
import { sequelize } from "@server/storage/database";
|
|
import FileStorage from "@server/storage/files";
|
|
import type { ZipTreeNode } from "@server/utils/ZipHelper";
|
|
import ZipHelper from "@server/utils/ZipHelper";
|
|
import type { ProcessOutput } from "./APIImportTask";
|
|
import APIImportTask from "./APIImportTask";
|
|
import { DocumentConverter } from "@server/converters/DocumentConverter";
|
|
|
|
type Markdown = IntegrationService.Markdown;
|
|
|
|
interface DiscoveredDocument {
|
|
id: string;
|
|
title: string;
|
|
pathInZip: string;
|
|
rootId: string;
|
|
parentDocumentId?: string;
|
|
markdownText: string;
|
|
children: DiscoveredDocument[];
|
|
}
|
|
|
|
interface DiscoveredRootNode {
|
|
id: string;
|
|
title: string;
|
|
synthetic?: boolean;
|
|
/** The markdown file that holds the content of this root node, if any. */
|
|
contentFile?: ZipTreeNode;
|
|
children: DiscoveredDocument[];
|
|
}
|
|
|
|
interface AttachmentRef {
|
|
id: string;
|
|
pathInZip: string;
|
|
}
|
|
|
|
/**
|
|
* Rewrites local attachment paths in markdown text into `<<attachmentId>>`
|
|
* placeholders. Supports legacy bucket layouts (`uploads/`, `public/`),
|
|
* arbitrary nested folder names, and `./attachments/...` rooted paths. Both
|
|
* encoded and unencoded path forms are matched.
|
|
*
|
|
* Exported for tests; not part of the module's public surface.
|
|
*
|
|
* @param markdown The raw markdown text from a single document.
|
|
* @param attachments Attachment manifest entries to substitute.
|
|
* @returns Markdown text with local paths replaced by `<<id>>` references.
|
|
*/
|
|
export function rewriteAttachmentPaths(
|
|
markdown: string,
|
|
attachments: AttachmentRef[]
|
|
): string {
|
|
let text = markdown;
|
|
|
|
for (const attachment of attachments) {
|
|
const encodedPath = encodeURI(attachment.pathInZip);
|
|
const attachmentFileName = path.basename(attachment.pathInZip);
|
|
const reference = `<<${attachment.id}>>`;
|
|
|
|
const normalizedAttachmentPath = encodedPath
|
|
.replace(new RegExp(`(.*)/${Buckets.uploads}/`), `${Buckets.uploads}/`)
|
|
.replace(new RegExp(`(.*)/${Buckets.public}/`), `${Buckets.public}/`);
|
|
|
|
const attachmentDir = path.basename(path.dirname(attachment.pathInZip));
|
|
const genericNormalizedPath = `${attachmentDir}/${encodeURI(attachmentFileName)}`;
|
|
|
|
text = text
|
|
.replace(new RegExp(escapeRegExp(encodedPath), "g"), reference)
|
|
.replace(
|
|
new RegExp(`\\.?/?${escapeRegExp(normalizedAttachmentPath)}`, "g"),
|
|
reference
|
|
);
|
|
|
|
const segments = attachment.pathInZip.split(path.sep);
|
|
const attachmentsIdx = segments.findIndex(
|
|
(seg) => seg.toLowerCase() === "attachments"
|
|
);
|
|
if (attachmentsIdx >= 0) {
|
|
const relFromAttachments = segments.slice(attachmentsIdx).join("/");
|
|
text = text.replace(
|
|
new RegExp(`\\.?/?${escapeRegExp(encodeURI(relFromAttachments))}`, "g"),
|
|
reference
|
|
);
|
|
}
|
|
|
|
text = text.replace(
|
|
new RegExp(`\\.?/?${escapeRegExp(genericNormalizedPath)}`, "g"),
|
|
reference
|
|
);
|
|
}
|
|
|
|
return text;
|
|
}
|
|
|
|
/**
|
|
* Rewrites internal markdown links (`[label](./relative.md)`) into
|
|
* `<<documentId>>` placeholders, resolved against a path → id map built from
|
|
* the zip's full document tree.
|
|
*
|
|
* Exported for tests; not part of the module's public surface.
|
|
*
|
|
* @param markdown The raw markdown text from a single document.
|
|
* @param documentPath Zip-relative path of the document being rewritten
|
|
* (e.g. `Collection/parent.md`); used as the base for
|
|
* resolving relative link targets against docMap keys.
|
|
* @param docMap Map of document path (as it appeared in the zip) to its
|
|
* pre-assigned externalId.
|
|
* @param resolve Optional fallback called with the destination and label of
|
|
* links the path lookup cannot resolve; returns the
|
|
* externalId of the linked document, or undefined to leave
|
|
* the link untouched.
|
|
* @returns Markdown text with internal `.md` link targets replaced by
|
|
* `<<id>>` references.
|
|
*/
|
|
export function rewriteInternalLinks(
|
|
markdown: string,
|
|
documentPath: string,
|
|
docMap: Record<string, string>,
|
|
resolve?: (href: string, label: string) => string | undefined
|
|
): string {
|
|
const basePath = path.dirname(documentPath);
|
|
|
|
return replaceMarkdownLinks(markdown, (href, label) => {
|
|
let referredDocId: string | undefined;
|
|
try {
|
|
const normalizedDocPath = decodeURI(
|
|
path.normalize(`${basePath}/${href}`)
|
|
);
|
|
referredDocId = docMap[normalizedDocPath];
|
|
} catch {
|
|
referredDocId = undefined;
|
|
}
|
|
|
|
referredDocId ??= resolve?.(href, label);
|
|
return referredDocId ? `<<${referredDocId}>>` : undefined;
|
|
});
|
|
}
|
|
|
|
export default class MarkdownAPIImportTask extends APIImportTask<Markdown> {
|
|
protected shouldUploadAttachmentsPerPage(): boolean {
|
|
// Per-page upload downloads remote image/attachment URLs referenced in
|
|
// the markdown (e.g. Slab's signed export URLs) and rewrites them to
|
|
// internal redirect URLs. The base step skips URLs that are already
|
|
// internal, so local zip attachments — rewritten to redirect URLs during
|
|
// `rewriteMarkdown` and uploaded from the archive in
|
|
// `onAllTasksCompleted` — pass through untouched.
|
|
return true;
|
|
}
|
|
|
|
/**
|
|
* Whether a document's leading H1 heading should be lifted out as its title
|
|
* (and removed from the body). Outline's own Markdown export writes the
|
|
* title as a leading H1, so this defaults to true. Sources where the
|
|
* filename is authoritative and the first heading is real content (e.g.
|
|
* Slab) override this to keep the heading in the body.
|
|
*
|
|
* @returns true to derive the title from a leading H1 heading.
|
|
*/
|
|
protected shouldExtractTitleFromHeading(): boolean {
|
|
return true;
|
|
}
|
|
|
|
protected async scheduleNextTask(importTask: ImportTask<Markdown>) {
|
|
await new MarkdownAPIImportTask().schedule({ importTaskId: importTask.id });
|
|
}
|
|
|
|
protected async onAllTasksCompleted(
|
|
lastImportTask: ImportTask<Markdown>
|
|
): Promise<void> {
|
|
const scratch = lastImportTask.import.scratch;
|
|
if (!scratch?.storageKey || !scratch.manifest?.length) {
|
|
return;
|
|
}
|
|
|
|
const handle = await FileStorage.getFileHandle(scratch.storageKey);
|
|
|
|
try {
|
|
const createdBy = lastImportTask.import.createdBy;
|
|
const manifestByPath = new Map<string, MarkdownAttachmentManifestItem>(
|
|
scratch.manifest.map((item) => [item.pathInZip, item])
|
|
);
|
|
const maxAttachmentSize = AttachmentHelper.presetToMaxUploadSize(
|
|
AttachmentPreset.DocumentAttachment
|
|
);
|
|
const seen = new Set<string>();
|
|
|
|
await ZipHelper.walk(handle.path, async (entry) => {
|
|
if (entry.isDirectory) {
|
|
return;
|
|
}
|
|
// Normalize to match the bootstrap-phase pathInZip (segments rejoined
|
|
// with `/`, no leading `./` or empty segments).
|
|
const normalized = entry.fileName.split("/").filter(Boolean).join("/");
|
|
const item = manifestByPath.get(normalized);
|
|
if (!item) {
|
|
return;
|
|
}
|
|
seen.add(item.pathInZip);
|
|
const buffer = await entry.readBuffer(maxAttachmentSize);
|
|
|
|
try {
|
|
await sequelize.transaction(async (transaction) =>
|
|
attachmentCreator({
|
|
source: "import",
|
|
preset: AttachmentPreset.DocumentAttachment,
|
|
id: item.id,
|
|
name: item.name,
|
|
type: item.mimeType,
|
|
buffer,
|
|
user: createdBy,
|
|
ctx: createContext({ user: createdBy, transaction }),
|
|
fetchOptions: {
|
|
timeout: env.FILE_STORAGE_IMPORT_TIMEOUT,
|
|
},
|
|
})
|
|
);
|
|
} catch (err) {
|
|
// Each attachment commits in its own transaction, so a retry of
|
|
// this hook can re-encounter ids that already landed. Treat the
|
|
// unique-id collision as a no-op so the import remains resumable.
|
|
if (err instanceof UniqueConstraintError) {
|
|
return;
|
|
}
|
|
throw err;
|
|
}
|
|
});
|
|
|
|
for (const item of scratch.manifest) {
|
|
if (!seen.has(item.pathInZip)) {
|
|
Logger.warn(
|
|
`Markdown import attachment missing in zip, skipping: ${item.pathInZip}`
|
|
);
|
|
}
|
|
}
|
|
} finally {
|
|
await handle.cleanup().catch(() => {});
|
|
}
|
|
}
|
|
|
|
protected async processBootstrap(
|
|
importTask: ImportTask<Markdown>
|
|
): Promise<ProcessOutput<Markdown>> {
|
|
const storageKey = importTask.import.scratch?.storageKey;
|
|
if (!storageKey) {
|
|
throw new Error("Markdown import is missing scratch.storageKey");
|
|
}
|
|
|
|
const handle = await FileStorage.getFileHandle(storageKey);
|
|
|
|
try {
|
|
// Side map of pre-loaded markdown text keyed by tree node identity.
|
|
// ZipHelper's tree carries only metadata; we capture text during the
|
|
// single walk so the downstream pass doesn't have to re-open the zip.
|
|
const markdownByNode = new Map<ZipTreeNode, string>();
|
|
|
|
const tree = await ZipHelper.toFileTree(
|
|
handle.path,
|
|
async (node, entry) => {
|
|
const ext = path.extname(node.name).toLowerCase();
|
|
if (ext === ".md" || ext === ".markdown") {
|
|
const buffer = await entry.readBuffer(
|
|
DocumentValidation.maxStateLength
|
|
);
|
|
markdownByNode.set(node, buffer.toString("utf8"));
|
|
}
|
|
},
|
|
// Text read from the archive may not exceed the upload size limit.
|
|
{
|
|
maxEntries: ImportValidation.maxZipEntries,
|
|
maxTotalSize: AttachmentHelper.presetToMaxUploadSize(
|
|
AttachmentPreset.WorkspaceImport
|
|
),
|
|
}
|
|
);
|
|
|
|
const topLevelNodes = this.resolveCollectionRootNodes(tree.children);
|
|
|
|
if (topLevelNodes.length === 0) {
|
|
throw FileImportError("Could not find valid content in zip file");
|
|
}
|
|
|
|
const rootNodes: DiscoveredRootNode[] = [];
|
|
const manifest: MarkdownAttachmentManifestItem[] = [];
|
|
// Loose markdown files that sit at the archive root rather than inside a
|
|
// root folder are gathered into a synthetic fallback root node below.
|
|
const looseDocuments: ZipTreeNode[] = [];
|
|
// Folders that hold the children of a loose markdown file of the same name.
|
|
const looseDocumentFolders: ZipTreeNode[] = [];
|
|
const hasDestination =
|
|
!!importTask.import.collectionId || !!importTask.import.documentId;
|
|
const looseFilesByTitle = new Map(
|
|
topLevelNodes
|
|
.filter((node) => this.isMarkdownFile(node))
|
|
.map((node) => [node.title, node])
|
|
);
|
|
|
|
for (const node of topLevelNodes) {
|
|
if (node.children.length === 0) {
|
|
if (this.isMarkdownFile(node)) {
|
|
looseDocuments.push(node);
|
|
} else {
|
|
manifest.push({
|
|
id: randomUUID(),
|
|
name: node.name,
|
|
pathInZip: node.pathInZip,
|
|
mimeType: mime.lookup(node.name) || "application/octet-stream",
|
|
});
|
|
}
|
|
continue;
|
|
}
|
|
|
|
if (this.isAttachmentFolder(node)) {
|
|
this.collectAttachments(node, manifest);
|
|
continue;
|
|
}
|
|
|
|
// A document export writes "name.md" beside "name/". In a destination
|
|
// they form one document, otherwise the file is the collection overview.
|
|
const contentFile = looseFilesByTitle.get(node.title);
|
|
if (contentFile && hasDestination) {
|
|
looseDocumentFolders.push(node);
|
|
continue;
|
|
}
|
|
|
|
const rootNode: DiscoveredRootNode = {
|
|
id: randomUUID(),
|
|
title: node.title,
|
|
contentFile,
|
|
children: [],
|
|
};
|
|
rootNodes.push(rootNode);
|
|
|
|
this.collectDocumentsAndAttachments({
|
|
children: node.children,
|
|
rootId: rootNode.id,
|
|
out: rootNode.children,
|
|
manifest,
|
|
markdownByNode,
|
|
});
|
|
}
|
|
|
|
const contentFiles = new Set(rootNodes.map((c) => c.contentFile));
|
|
// Folders follow the files so that merged documents keep the file's path.
|
|
const syntheticChildren = [
|
|
...looseDocuments.filter((node) => !contentFiles.has(node)),
|
|
...looseDocumentFolders,
|
|
];
|
|
|
|
if (syntheticChildren.length > 0) {
|
|
const rootNode: DiscoveredRootNode = {
|
|
id: randomUUID(),
|
|
title: this.fallbackRootNodeTitle(),
|
|
synthetic: true,
|
|
children: [],
|
|
};
|
|
rootNodes.push(rootNode);
|
|
|
|
this.collectDocumentsAndAttachments({
|
|
children: syntheticChildren,
|
|
rootId: rootNode.id,
|
|
out: rootNode.children,
|
|
manifest,
|
|
markdownByNode,
|
|
});
|
|
}
|
|
|
|
// Build docMap (pathInZip -> externalId) for internal-link resolution.
|
|
// Walk the full document tree to collect every doc id, since internal
|
|
// markdown links can target any document regardless of depth.
|
|
const docMap: Record<string, string> = {};
|
|
const collectDocMap = (docs: DiscoveredDocument[]) => {
|
|
for (const d of docs) {
|
|
docMap[d.pathInZip] = d.id;
|
|
collectDocMap(d.children);
|
|
}
|
|
};
|
|
for (const c of rootNodes) {
|
|
if (c.contentFile) {
|
|
docMap[c.contentFile.pathInZip] = c.id;
|
|
}
|
|
collectDocMap(c.children);
|
|
}
|
|
|
|
// Replace (not append) anything past the create-time placeholder with
|
|
// the freshly discovered root nodes so a retried bootstrap doesn't
|
|
// accumulate duplicate entries with fresh UUIDs from a previous
|
|
// partial run. ImportsProcessor's persistence pass materializes these
|
|
// as collections (default) or root documents (destination mode).
|
|
const associatedImport = importTask.import;
|
|
const placeholder = associatedImport.input[0];
|
|
associatedImport.input = [
|
|
placeholder,
|
|
...rootNodes.map((c) => ({
|
|
externalId: c.id,
|
|
permission: placeholder.permission,
|
|
synthetic: c.synthetic,
|
|
})),
|
|
];
|
|
|
|
associatedImport.scratch = { storageKey, manifest, docMap };
|
|
await associatedImport.save();
|
|
|
|
// Append root-node placeholder items so ImportsProcessor iterates
|
|
// them during the bootstrap row (the earliest createdAt) — that
|
|
// guarantees root nodes land in the DB before any per-page document
|
|
// references them.
|
|
const rootInputItems: MarkdownPageImportTaskInputItem[] = rootNodes.map(
|
|
(c) => {
|
|
const markdownText = c.contentFile
|
|
? (markdownByNode.get(c.contentFile) ?? "")
|
|
: "";
|
|
return {
|
|
externalId: c.id,
|
|
title: c.title,
|
|
path: c.contentFile?.pathInZip ?? c.title,
|
|
markdownText,
|
|
attachmentMap: this.attachmentsReferencedBy(markdownText, manifest),
|
|
};
|
|
}
|
|
);
|
|
|
|
importTask.input = [importTask.input[0], ...rootInputItems];
|
|
|
|
const rootOutputs: ImportTaskOutput = await Promise.all(
|
|
rootInputItems.map((item) =>
|
|
this.convertPage(item, docMap, CollectionValidation.maxNameLength)
|
|
)
|
|
);
|
|
|
|
// First wave of document tasks: only top-level docs in each root node.
|
|
// Each carries its descendants in `children` and the per-page handler
|
|
// re-emits them as the next wave of childTasksInput, producing a strict
|
|
// depth-ordered cascade of ImportTask rows so parent FKs are always
|
|
// satisfied at child-doc creation time.
|
|
const childTasksInput: ImportTaskInput<Markdown> = rootNodes.flatMap(
|
|
(c) => c.children.map((d) => this.toPageInput(d, manifest))
|
|
);
|
|
|
|
return { taskOutput: rootOutputs, childTasksInput };
|
|
} finally {
|
|
await handle.cleanup().catch(() => {});
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Converts a discovered document subtree into a per-page task input,
|
|
* recursively packing the doc's descendants into the `children` field so
|
|
* each tree-depth runs as its own task wave.
|
|
*
|
|
* @param doc The discovered document, including its descendants.
|
|
* @param manifest The full attachment manifest (used for per-page refs).
|
|
* @returns A self-contained per-page task input.
|
|
*/
|
|
private toPageInput(
|
|
doc: DiscoveredDocument,
|
|
manifest: MarkdownAttachmentManifestItem[]
|
|
): MarkdownPageImportTaskInputItem {
|
|
return {
|
|
externalId: doc.id,
|
|
parentExternalId: doc.parentDocumentId,
|
|
rootExternalId: doc.rootId,
|
|
title: doc.title,
|
|
path: doc.pathInZip,
|
|
markdownText: doc.markdownText,
|
|
attachmentMap: this.attachmentsReferencedBy(doc.markdownText, manifest),
|
|
children: doc.children.length
|
|
? doc.children.map((c) => this.toPageInput(c, manifest))
|
|
: undefined,
|
|
};
|
|
}
|
|
|
|
protected async processPage(
|
|
importTask: ImportTask<Markdown>
|
|
): Promise<ProcessOutput<Markdown>> {
|
|
const taskOutput: ImportTaskOutput = [];
|
|
const childTasksInput: MarkdownPageImportTaskInputItem[] = [];
|
|
|
|
// docMap is stored once on scratch rather than embedded in each page's
|
|
// input (see processBootstrap) — read it back here for link rewriting.
|
|
const docMap = importTask.import.scratch?.docMap ?? {};
|
|
|
|
const items = importTask.input as MarkdownPageImportTaskInputItem[];
|
|
for (const item of items) {
|
|
taskOutput.push(
|
|
await this.convertPage(item, docMap, DocumentValidation.maxTitleLength)
|
|
);
|
|
|
|
// Cascade this doc's direct descendants as the next task wave. Their
|
|
// ImportTask rows will be created after the current one returns, so
|
|
// their createdAt is strictly later — guaranteeing parent-before-child
|
|
// FK ordering during ImportsProcessor's persistence pass.
|
|
if (item.children?.length) {
|
|
childTasksInput.push(...item.children);
|
|
}
|
|
}
|
|
|
|
return { taskOutput, childTasksInput };
|
|
}
|
|
|
|
/**
|
|
* Converts a page's markdown text into task output.
|
|
*
|
|
* @param item The per-page task input.
|
|
* @param docMap Path → externalId map for internal link rewriting.
|
|
* @param maxTitleLength The maximum length of the title in the output.
|
|
* @returns The task output for the page.
|
|
*/
|
|
private async convertPage(
|
|
item: MarkdownPageImportTaskInputItem,
|
|
docMap: Record<string, string>,
|
|
maxTitleLength: number
|
|
): Promise<ImportTaskOutput[number]> {
|
|
// Empty markdown short-circuits, as a root node without a content file has
|
|
// no text to convert.
|
|
if (!item.markdownText) {
|
|
return {
|
|
externalId: item.externalId,
|
|
title: truncate(item.title, { length: maxTitleLength }),
|
|
content: ProsemirrorDataHelper.getEmpty() as ProsemirrorDoc,
|
|
};
|
|
}
|
|
|
|
const converted = await DocumentConverter.convert(
|
|
this.rewriteMarkdown(item, docMap),
|
|
path.basename(item.path),
|
|
"text/markdown",
|
|
{ extractTitle: this.shouldExtractTitleFromHeading() }
|
|
);
|
|
|
|
return {
|
|
externalId: item.externalId,
|
|
title: truncate(converted.title || item.title, {
|
|
length: maxTitleLength,
|
|
}),
|
|
icon: converted.icon,
|
|
content: this.transformDocument(converted.doc).toJSON() as ProsemirrorDoc,
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Pre-rewrites a page's markdown text. Internal `.md` links become mention
|
|
* markdown so the editor parses them as Document mentions. Attachment paths
|
|
* are first reduced to `<<id>>` placeholders by the shared rewriter, then
|
|
* — distinct from the prosemirror-tree walk we used to do — substituted
|
|
* with their final attachment redirect URLs in the markdown text. Doing
|
|
* the resolution at the text layer avoids markdown-it parsing `<<id>>` as
|
|
* an angle-bracket-wrapped URL (which produced broken image src attrs).
|
|
*
|
|
* @param page The per-page task input.
|
|
* @param docMap Path → externalId map for internal link rewriting, shared
|
|
* across all pages via `Import.scratch`.
|
|
* @returns Rewritten markdown text ready for DocumentConverter.
|
|
*/
|
|
private rewriteMarkdown(
|
|
page: MarkdownPageImportTaskInputItem,
|
|
docMap: Record<string, string>
|
|
): string {
|
|
let text = rewriteInternalLinks(
|
|
page.markdownText,
|
|
page.path,
|
|
docMap,
|
|
(href, label) => this.resolveLinkedDocument(href, label, docMap)
|
|
);
|
|
|
|
// Convert `[label](<<id>>)` links from rewriteInternalLinks into mention
|
|
// markdown the editor recognises: `@[label](mention://<uuid>/document/<id>)`.
|
|
text = text.replace(
|
|
/\[([^\]]+)\]\(<<([^>]+)>>\)/g,
|
|
(_full, label: string, externalId: string) =>
|
|
`@[${label}](mention://${randomUUID()}/document/${externalId})`
|
|
);
|
|
|
|
text = rewriteAttachmentPaths(
|
|
text,
|
|
page.attachmentMap.map((m) => ({ id: m.id, pathInZip: m.pathInZip }))
|
|
);
|
|
|
|
// Resolve remaining `<<id>>` placeholders to attachment redirect URLs.
|
|
text = text.replace(/<<([^>]+)>>/g, (_full, id: string) =>
|
|
Attachment.getRedirectUrl(id)
|
|
);
|
|
|
|
return text;
|
|
}
|
|
|
|
/**
|
|
* Returns the subset of the full manifest that is referenced anywhere in
|
|
* the given markdown text. Used to bound the per-page task input size.
|
|
*
|
|
* @param markdown Raw markdown text for a single document.
|
|
* @param manifest The full attachment manifest from the bootstrap phase.
|
|
* @returns Manifest entries that appear (by filename) in the markdown.
|
|
*/
|
|
private attachmentsReferencedBy(
|
|
markdown: string,
|
|
manifest: MarkdownAttachmentManifestItem[]
|
|
): MarkdownAttachmentManifestItem[] {
|
|
return manifest.filter((item) => {
|
|
const fileName = path.basename(item.pathInZip);
|
|
return (
|
|
markdown.includes(fileName) || markdown.includes(encodeURI(fileName))
|
|
);
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Resolves the archive's top-level entries into the nodes that should be
|
|
* treated as collections. The base implementation uses the entries as-is;
|
|
* subclasses can override to unwrap a known wrapper directory before the
|
|
* bootstrap phase maps each node to a collection.
|
|
*
|
|
* @param nodes The archive's top-level tree nodes.
|
|
* @returns The nodes to import as collections.
|
|
*/
|
|
protected resolveCollectionRootNodes(nodes: ZipTreeNode[]): ZipTreeNode[] {
|
|
return nodes;
|
|
}
|
|
|
|
/**
|
|
* Resolves a link destination that does not point at a markdown file in the
|
|
* archive to a document in the import. The base implementation resolves
|
|
* nothing; subclasses can override to map a source's own document URLs
|
|
* onto the documents discovered in the archive.
|
|
*
|
|
* @param href The link destination as written in the markdown.
|
|
* @param label The link label as written in the markdown.
|
|
* @param docMap Map of document path in the zip to its externalId.
|
|
* @returns The externalId of the linked document, or undefined.
|
|
*/
|
|
protected resolveLinkedDocument(
|
|
// oxlint-disable-next-line @typescript-eslint/no-unused-vars
|
|
href: string,
|
|
// oxlint-disable-next-line @typescript-eslint/no-unused-vars
|
|
label: string,
|
|
// oxlint-disable-next-line @typescript-eslint/no-unused-vars
|
|
docMap: Record<string, string>
|
|
): string | undefined {
|
|
return undefined;
|
|
}
|
|
|
|
/**
|
|
* Post-processes a converted document before it is handed to the attachment
|
|
* upload step. The base implementation returns the document unchanged;
|
|
* subclasses can override to reshape source-specific content.
|
|
*
|
|
* @param doc The converted ProseMirror document.
|
|
* @returns The document to persist.
|
|
*/
|
|
protected transformDocument(doc: Node): Node {
|
|
return doc;
|
|
}
|
|
|
|
/**
|
|
* Title for the fallback root node that holds markdown files found loose at
|
|
* the archive root (outside any folder). Subclasses can override to give it
|
|
* a source-appropriate name.
|
|
*
|
|
* @returns The fallback root node title.
|
|
*/
|
|
protected fallbackRootNodeTitle(): string {
|
|
return "Documents";
|
|
}
|
|
|
|
/**
|
|
* Detects files in the archive that hold a markdown document.
|
|
*
|
|
* @param node ZipTreeNode to inspect.
|
|
* @returns true when the node is a markdown file.
|
|
*/
|
|
private isMarkdownFile(node: ZipTreeNode): boolean {
|
|
const ext = path.extname(node.name).toLowerCase();
|
|
return node.children.length === 0 && (ext === ".md" || ext === ".markdown");
|
|
}
|
|
|
|
/**
|
|
* Detects folders containing only attachments (no markdown documents).
|
|
* Recursively considers nested folders; mirrors the legacy heuristic.
|
|
*
|
|
* @param node ZipTreeNode to inspect.
|
|
* @returns true when the folder appears to hold only attachments.
|
|
*/
|
|
private isAttachmentFolder(node: ZipTreeNode): boolean {
|
|
if (node.children.length === 0) {
|
|
return false;
|
|
}
|
|
if (node.title.toLowerCase() === "attachments") {
|
|
return true;
|
|
}
|
|
return node.children.every((child) => {
|
|
if (child.children.length > 0) {
|
|
return this.isAttachmentFolder(child);
|
|
}
|
|
const ext = path.extname(child.name).toLowerCase();
|
|
if (!ext) {
|
|
return false;
|
|
}
|
|
return ext !== ".md" && ext !== ".markdown";
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Recursively collects all files under an attachment-only folder into the
|
|
* manifest. `pathInZip` is stored verbatim so the completion phase can find
|
|
* the same entry when re-walking the archive.
|
|
*
|
|
* @param node Attachment-folder ZipTreeNode.
|
|
* @param manifest Manifest array to push entries into.
|
|
*/
|
|
private collectAttachments(
|
|
node: ZipTreeNode,
|
|
manifest: MarkdownAttachmentManifestItem[]
|
|
): void {
|
|
for (const child of node.children) {
|
|
if (child.children.length > 0) {
|
|
this.collectAttachments(child, manifest);
|
|
continue;
|
|
}
|
|
manifest.push({
|
|
id: randomUUID(),
|
|
name: child.name,
|
|
pathInZip: child.pathInZip,
|
|
mimeType: mime.lookup(child.name) || "application/octet-stream",
|
|
});
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Walks a collection subtree and gathers documents (markdown files) and
|
|
* loose attachments. Documents are appended to `out` as a tree — each
|
|
* entry's `children` holds its direct descendants. This is the shape the
|
|
* per-page task cascade consumes.
|
|
*
|
|
* @param children ZipTreeNode children of the current folder.
|
|
* @param rootId Pre-assigned id of the enclosing root node.
|
|
* @param parentDocumentId Optional parent document id when nested.
|
|
* @param out Sibling accumulator to push discovered documents into.
|
|
* @param manifest Attachment manifest accumulator.
|
|
* @param markdownByNode Pre-loaded markdown text keyed by tree node.
|
|
*/
|
|
private collectDocumentsAndAttachments({
|
|
children,
|
|
rootId,
|
|
parentDocumentId,
|
|
out,
|
|
manifest,
|
|
markdownByNode,
|
|
}: {
|
|
children: ZipTreeNode[];
|
|
rootId: string;
|
|
parentDocumentId?: string;
|
|
out: DiscoveredDocument[];
|
|
manifest: MarkdownAttachmentManifestItem[];
|
|
markdownByNode: Map<ZipTreeNode, string>;
|
|
}): void {
|
|
// Index existing docs by title so the folder/file merge lookup below is O(1)
|
|
// instead of an `out.find` scan per child (O(n²) over the number of siblings
|
|
// in a single directory). First occurrence wins, mirroring `find`.
|
|
const byTitle = new Map<string, DiscoveredDocument>();
|
|
for (const doc of out) {
|
|
if (!byTitle.has(doc.title)) {
|
|
byTitle.set(doc.title, doc);
|
|
}
|
|
}
|
|
|
|
for (const child of children) {
|
|
if (child.children.length > 0 && this.isAttachmentFolder(child)) {
|
|
this.collectAttachments(child, manifest);
|
|
continue;
|
|
}
|
|
|
|
const ext = path.extname(child.name).toLowerCase();
|
|
const isMarkdown = ext === ".md" || ext === ".markdown";
|
|
const isFolder = child.children.length > 0;
|
|
|
|
if (!isMarkdown && !isFolder) {
|
|
manifest.push({
|
|
id: randomUUID(),
|
|
name: child.name,
|
|
pathInZip: child.pathInZip,
|
|
mimeType: mime.lookup(child.name) || "application/octet-stream",
|
|
});
|
|
continue;
|
|
}
|
|
|
|
const id = randomUUID();
|
|
const markdownText = isFolder ? "" : (markdownByNode.get(child) ?? "");
|
|
|
|
// Folder-and-file with the same title (a "name.md" alongside a "name/"
|
|
// directory) is merged onto a single document: the folder body picks up
|
|
// the file's markdown text, and the folder's contents become children.
|
|
const sibling = byTitle.get(child.title);
|
|
|
|
if (sibling) {
|
|
if (sibling.markdownText === "" && markdownText) {
|
|
sibling.markdownText = markdownText;
|
|
}
|
|
if (isFolder) {
|
|
this.collectDocumentsAndAttachments({
|
|
children: child.children,
|
|
rootId,
|
|
parentDocumentId: sibling.id,
|
|
out: sibling.children,
|
|
manifest,
|
|
markdownByNode,
|
|
});
|
|
}
|
|
continue;
|
|
}
|
|
|
|
const node: DiscoveredDocument = {
|
|
id,
|
|
title: child.title,
|
|
pathInZip: child.pathInZip,
|
|
rootId,
|
|
parentDocumentId,
|
|
markdownText,
|
|
children: [],
|
|
};
|
|
out.push(node);
|
|
byTitle.set(node.title, node);
|
|
|
|
if (isFolder) {
|
|
this.collectDocumentsAndAttachments({
|
|
children: child.children,
|
|
rootId,
|
|
parentDocumentId: id,
|
|
out: node.children,
|
|
manifest,
|
|
markdownByNode,
|
|
});
|
|
}
|
|
}
|
|
}
|
|
}
|