Files
Tom MoorandClaude Opus 5 b032f1f819 feat: PDF import support (#13302)
* feat: Import PDF documents

Adds a PdfConverter that parses PDF files to markdown with pdf-inspector,
recovering headings, lists and tables from the layout of the text on the
page. A PDF with no embedded text is rejected, as reading it would need OCR.

Markdown has no syntax for underline, so the parser writes underlined text
as an HTML <u> element. Inline HTML is not enabled when the markdown is read
back, so the tags are rewritten to the underline syntax the editor knows.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

* Accept PDF files in the import dialog, and give it a two column layout

The supported formats move alongside the drop target rather than below it,
so the dialog reads in one glance. Their badge icons are dropped with the
grid they were laid out in, leaving FileFormatIcon unused.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

* feedback

---------

Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-08-05 18:36:40 -04:00

110 lines
2.8 KiB
TypeScript

import mime from "mime-types";
import { truncate } from "es-toolkit/compat";
import type { ProsemirrorData } from "@shared/types";
import { DocumentValidation } from "@shared/validations";
import { serializer } from "@server/editor";
import { traceFunction } from "@server/logging/tracing";
import type { User } from "@server/models";
import { ProsemirrorHelper } from "@server/models/helpers/ProsemirrorHelper";
import type { APIContext } from "@server/types";
import { DocumentConverter } from "@server/converters/DocumentConverter";
import { InvalidRequestError } from "../errors";
type Props = {
user: User;
mimeType: string;
fileName: string;
content: Buffer | string;
ctx: APIContext;
};
type ImportResult = {
icon?: string;
text: string;
title: string;
state: Buffer;
};
/**
* Converts document content to state and validates size constraints.
*
* @param content The document content as Prosemirror JSON.
* @param title The document title (used in error messages).
* @returns The Y.Doc state buffer.
*/
function convertToState(content: ProsemirrorData, title: string): Buffer {
const ydoc = ProsemirrorHelper.toYDoc(content);
const state = ProsemirrorHelper.toState(ydoc);
if (state.length > DocumentValidation.maxStateLength) {
throw InvalidRequestError(
`The document "${title}" is too large to import, please reduce the length and try again`
);
}
return state;
}
async function documentImporter({
mimeType,
fileName,
content,
user,
ctx,
}: Props): Promise<ImportResult> {
// Find valid extensions and remove them from the title
const extensions = [
"docx",
"md",
"markdown",
"htm",
"html",
"csv",
"tsv",
"mhtml",
"mht",
"eml",
"textpack",
"pdf",
...(mime.extensions[mimeType] ?? []),
];
const fileTitle = fileName.replace(
new RegExp(`\\.(${extensions.join("|")})$`, "i"),
""
);
// Convert document using unified converter
const {
doc,
title: extractedTitle,
icon,
} = await DocumentConverter.convert(content, fileName, mimeType);
// Use extracted title or fall back to filename
let title = extractedTitle || fileTitle;
// Replace external images with attachments
const processedDoc = await ProsemirrorHelper.replaceImagesWithAttachments(
ctx,
doc,
user
);
// Serialize final text and handle empty documents
let text = serializer.serialize(processedDoc).trim();
// Empty paragraphs serialize to escaped newlines/backslashes, treat as empty
if (/^[\\\s]*$/.test(text)) {
text = "";
}
// Truncate title and validate size
title = truncate(title, { length: DocumentValidation.maxTitleLength });
const state = convertToState(processedDoc.toJSON() as ProsemirrorData, title);
return { text, state, title, icon };
}
export default traceFunction({
spanName: "documentImporter",
})(documentImporter);