import type { JobOptions } from "bull"; import { chunk, truncate, uniqBy } from "es-toolkit/compat"; import httpErrors from "http-errors"; import { Fragment, Node } from "prosemirror-model"; import type { WhereOptions } from "sequelize"; import { Transaction } from "sequelize"; import { randomUUID } from "node:crypto"; import type { BaseImportTaskInput, ImportTaskInput, ImportTaskOutput, } from "@shared/schema"; import type { ImportableIntegrationService, ProsemirrorData, ProsemirrorDoc, } from "@shared/types"; import { AttachmentPreset, ImportState, ImportTaskPhase, ImportTaskState, } from "@shared/types"; import { toError } from "@shared/utils/error"; import { isBase64Url, isExternalUrl, isUrl } from "@shared/utils/urls"; import { createContext } from "@server/context"; import { schema } from "@server/editor"; import Logger from "@server/logging/Logger"; import type { User } from "@server/models"; import { Attachment, Import, ImportTask } from "@server/models"; import AttachmentHelper from "@server/models/helpers/AttachmentHelper"; import { ProsemirrorHelper } from "@server/models/helpers/ProsemirrorHelper"; import { sequelize } from "@server/storage/database"; import { PagePerImportTask } from "../processors/ImportsProcessor"; import { BaseTask, TaskPriority } from "./base/BaseTask"; import UploadAttachmentsForImportTask from "./UploadAttachmentsForImportTask"; export type ProcessOutput = { taskOutput: ImportTaskOutput; childTasksInput: ImportTaskInput; }; type Props = { /** id of the import_task */ importTaskId: string; }; export default abstract class APIImportTask< T extends ImportableIntegrationService, > extends BaseTask { /** * Run the import task. * * @param importTaskId id of the import_task model. * @returns Promise that resolves once the task has completed. */ public async perform({ importTaskId }: Props) { let importTask = await ImportTask.findByPk>(importTaskId, { include: [ { model: Import, as: "import", required: true, }, ], }); // The import_task row may have been deleted (e.g. its Import was removed) // between the job being enqueued and the worker picking it up. Nothing to do. if (!importTask) { return; } // Stop when the import has ended, or its tasks are all processed already. if (this.isFinished(importTask.import.state)) { if (importTask.state !== ImportTaskState.Completed) { importTask.state = ImportTaskState.Canceled; await importTask.save(); } return; } try { switch (importTask.state) { case ImportTaskState.Created: { importTask.state = ImportTaskState.InProgress; importTask = await importTask.save(); return await this.onProcess(importTask); } case ImportTaskState.InProgress: return await this.onProcess(importTask); case ImportTaskState.Completed: return await this.onCompletion(importTask); default: } } catch (err) { if (err instanceof Error) { importTask.error = truncate(err.message, { length: 255 }); await importTask.save(); } if (this.isNonRetryable(err)) { await this.onFailed({ importTaskId }); } throw err; // throw error for retry. } } /** * Whether a failure is deterministic and therefore not worth retrying. A * 4xx-class error raised by our own error factories signals invalid source * data (a missing file in an export, an unparseable document) rather than a * transient problem such as a network or storage blip. * * @param err The error thrown while performing the task. * @returns true when retrying the task cannot succeed. */ private isNonRetryable(err: unknown): boolean { return httpErrors.isHttpError(err) && err.status >= 400 && err.status < 500; } /** * Handle failure when all attempts of APIImportTask has failed. * * @param importTaskId id of the import_task model. * @returns Promise that resolves once failure has been handled. */ public async onFailed({ importTaskId }: Props) { await sequelize.transaction(async (transaction) => { const importTask = await ImportTask.findByPk>( importTaskId, { transaction, lock: { level: Transaction.LOCK.NO_KEY_UPDATE, of: ImportTask }, } ); if (!importTask) { return; } const associatedImport = await this.lockImport( importTask.importId, transaction ); if (!associatedImport) { return; } // Keep the outcome of a cancel or of a finished import. if (this.isFinished(associatedImport.state)) { return; } if (importTask.state !== ImportTaskState.Completed) { importTask.state = ImportTaskState.Errored; await importTask.save({ transaction }); } associatedImport.error = importTask.error; // copy error from ImportTask that caused the failure. associatedImport.state = ImportState.Errored; await associatedImport.saveWithCtx( createContext({ user: associatedImport.createdBy, transaction, }) ); }); } /** * Creation flow for the task. * This fetches data from external source, stores the task output and creates subsequent import_task models. * * @param importTask import_task model to process. * @returns Promise that resolves once processing has completed. */ private async onProcess(importTask: ImportTask) { const { taskOutput, childTasksInput } = importTask.phase === ImportTaskPhase.Bootstrap ? await this.processBootstrap(importTask) : await this.processPage(importTask); const taskOutputWithReplacements = this.shouldUploadAttachmentsPerPage() ? await Promise.all( taskOutput.map(async (item) => ({ ...item, content: await this.uploadAttachments({ doc: item.content, externalId: item.externalId, createdBy: importTask.import.createdBy, }), })) ) : taskOutput; await sequelize.transaction(async (transaction) => { await Promise.all( chunk(childTasksInput as BaseImportTaskInput, PagePerImportTask).map( async (input) => { await ImportTask.create( { state: ImportTaskState.Created, phase: ImportTaskPhase.Page, input: input as ImportTaskInput, importId: importTask.importId, }, { transaction } ); } ) ); importTask.output = taskOutputWithReplacements; importTask.state = ImportTaskState.Completed; importTask.error = null; // unset any error from previous attempts. await importTask.save({ transaction }); const associatedImport = importTask.import; associatedImport.documentCount += taskOutputWithReplacements.length; await associatedImport.saveWithCtx( createContext({ user: associatedImport.createdBy, transaction, }), undefined, { persist: false, } ); }); await this.scheduleNextTask(importTask); } /** * Completion flow for the task. * This determines if there are any more import_tasks to process (or) all tasks for the import have been processed, and schedules the next step. * * @param importTask import_task model to process. * @returns Promise that resolves once processing has completed. */ private async onCompletion(importTask: ImportTask) { const where: WhereOptions> = { state: ImportTaskState.Created, importId: importTask.importId, }; const nextImportTask = await ImportTask.findOne>({ where, order: [["createdAt", "ASC"]], }); // Tasks available to process for this import. if (nextImportTask) { return await this.scheduleNextTask(nextImportTask); } // All tasks for this import have been processed. Run the post-completion // hook before flipping state so subclasses can perform work that must // happen before "imports.processed" downstream handlers fire. await this.onAllTasksCompleted(importTask); await sequelize.transaction(async (transaction) => { const associatedImport = await this.lockImport( importTask.importId, transaction ); // The import may have been canceled while this task was running. if (!associatedImport || this.isFinished(associatedImport.state)) { return; } associatedImport.state = ImportState.Processed; await associatedImport.saveWithCtx( createContext({ user: associatedImport.createdBy, transaction, }), undefined, { name: "processed" } ); }); } /** * Whether the import has ended, or its tasks are all processed already. * * @param state state of the import. * @returns true if no more import tasks should change the import. */ private isFinished(state: ImportState) { return ( state === ImportState.Processed || Import.terminalStates.includes(state) ); } /** * Load the import with a row lock so that concurrent state changes are not lost. * * @param importId id of the import. * @param transaction transaction to hold the lock in. * @returns the locked import, or null if it does not exist. */ private lockImport(importId: string, transaction: Transaction) { return Import.findByPk>(importId, { transaction, lock: { level: transaction.LOCK.NO_KEY_UPDATE, of: Import }, }); } /** * Whether the base class should create Attachment rows and upload S3 blobs * per page during `onProcess`. Defaults to `true` for sources whose * attachments are addressable by per-task URLs (e.g. Notion). Sources where * attachments are shared across pages or live in a single archive may * override this and handle attachment persistence in `onAllTasksCompleted`. * * @returns true to enable the per-page attachment upload step. */ protected shouldUploadAttachmentsPerPage(): boolean { return true; } /** * Hook invoked after the final import task has been processed but before the * associated `Import` state transitions to `Processed`. Subclasses can * override to perform cross-task finalization (e.g. uploading shared * attachments) that must happen before the persistence pass. * * @param lastImportTask The most recently completed ImportTask for the import. * @returns Promise that resolves when finalization is complete. */ protected async onAllTasksCompleted( // oxlint-disable-next-line @typescript-eslint/no-unused-vars lastImportTask: ImportTask ): Promise { return; } /** * Bootstrap phase. Runs once per import on a worker that owns the source * artifact (e.g. extracts a zip, walks the file tree, schedules child page * tasks). Subclasses without a bootstrap step leave this unimplemented; the * base only invokes it when an `ImportTask` is created with * `phase === ImportTaskPhase.Bootstrap`. * * @param importTask ImportTask model to process. * @returns Promise with output that resolves once processing has completed. */ protected processBootstrap( // oxlint-disable-next-line @typescript-eslint/no-unused-vars importTask: ImportTask ): Promise> { throw new Error( `${this.constructor.name} does not implement processBootstrap()` ); } /** * Page phase. Runs for every `ImportTask` row with * `phase === ImportTaskPhase.Page`, transforming a batch of source pages * into ProseMirror output and optionally cascading descendants as the next * wave of child tasks. * * @param importTask ImportTask model to process. * @returns Promise with output that resolves once processing has completed. */ protected abstract processPage( importTask: ImportTask ): Promise>; /** * Schedule the next `APIImportTask`. * * @param importTask ImportTask model associated with the `APIImportTask`. * @returns Promise that resolves when the task is scheduled. */ protected abstract scheduleNextTask(importTask: ImportTask): Promise; /** * Upload attachments found in the external document. * * @param doc ProseMirrorDoc that represents collection (or) document content. * @param externalId id of the document in the external service. * @param createdBy user who created the import. * @returns Updated ProseMirrorDoc. */ private async uploadAttachments({ doc, externalId, createdBy, }: { doc: ProsemirrorDoc; externalId: string; createdBy: User; }): Promise { const docNode = ProsemirrorHelper.toProsemirror(doc); const nodes = [ ...ProsemirrorHelper.getImages(docNode), ...ProsemirrorHelper.getVideos(docNode), ...ProsemirrorHelper.getAttachments(docNode), ]; if (!nodes.length) { return doc; } const urlToAttachment: Record = {}; // perf: dedup url. const attachmentsData = uniqBy( nodes.map((node) => { const url = String( node.type.name === "attachment" ? node.attrs.href : node.attrs.src ); const name = String( (node.type.name === "image" ? node.attrs.alt : node.attrs.title) || node.type.name ).trim(); return { url, name: name.length !== 0 ? name : node.type.name }; }), "url" ).filter((item) => isDownloadableUrl(item.url)); // Nothing remote to download — content already points at internal // attachments (e.g. a Markdown zip's local files resolved to redirect // URLs), so leave the doc untouched. if (!attachmentsData.length) { return doc; } await sequelize.transaction(async (transaction) => { const dbPromises = attachmentsData.map(async (item) => { const modelId = randomUUID(); const acl = AttachmentHelper.presetToAcl( AttachmentPreset.DocumentAttachment ); const key = AttachmentHelper.getKey({ id: modelId, name: item.name, userId: createdBy.id, }); const attachment = await Attachment.create( { id: modelId, key, acl, size: 0, expiresAt: AttachmentHelper.presetToExpiry( AttachmentPreset.DocumentAttachment ), contentType: "application/octet-stream", documentId: externalId, teamId: createdBy.teamId, userId: createdBy.id, }, { transaction } ); urlToAttachment[item.url] = attachment; }); return await Promise.all(dbPromises); }); try { const uploadItems = Object.entries(urlToAttachment).map( ([url, attachment]) => ({ attachmentId: attachment.id, url }) ); await new UploadAttachmentsForImportTask().schedule(uploadItems); } catch (err) { // upload attachments failure is not critical enough to fail the whole import. Logger.error( `upload attachment task failed for externalId ${externalId}`, toError(err) ); } return this.replaceAttachmentUrls(docNode, urlToAttachment).toJSON(); } /** * Replace remote url to internal redirect url for attachments. * * @param doc ProseMirror node that represents collection (or) document content. * @param urlToAttachment Map of remote url to attachment model. * @returns Updated Prosemirror node. */ private replaceAttachmentUrls( doc: Node, urlToAttachment: Record ): Node { const attachmentTypes = ["attachment", "image", "video"]; const transformAttachmentNode = (node: Node): Node => { const json = node.toJSON() as ProsemirrorData; const attrs = json.attrs ?? {}; if (node.type.name === "attachment") { // attachment node uses 'href' attribute. const attachmentModel = urlToAttachment[attrs.href as string]; // Nodes already pointing at internal attachments aren't in the map; // leave them untouched. if (!attachmentModel) { return node; } attrs.href = attachmentModel.redirectUrl; // attachment node can have id. attrs.id = attachmentModel.id; } else if (node.type.name === "image" || node.type.name === "video") { // image & video nodes use 'src' attribute. const attachmentModel = urlToAttachment[attrs.src as string]; if (!attachmentModel) { return node; } attrs.src = attachmentModel.redirectUrl; } json.attrs = attrs; return Node.fromJSON(schema, json); }; const transformFragment = (fragment: Fragment): Fragment => { const nodes: Node[] = []; fragment.forEach((node) => { nodes.push( attachmentTypes.includes(node.type.name) ? transformAttachmentNode(node) : node.copy(transformFragment(node.content)) ); }); return Fragment.fromArray(nodes); }; return doc.copy(transformFragment(doc.content)); } /** * Job options such as priority and retry strategy, as defined by Bull. */ public get options(): JobOptions { return { priority: TaskPriority.Normal, attempts: 3, backoff: { type: "exponential", delay: 60 * 1000, }, }; } } /** * Whether the url points to content that can be downloaded into an attachment. * * @param url the url of an image, video, or attachment node. * @returns true if the url is a remote absolute url or base64 data. */ function isDownloadableUrl(url: string) { return ( isExternalUrl(url) && (!!isBase64Url(url) || isUrl(url, { requireHostname: true })) ); }