Files
Tom Moor 6ece062b8c perf: Fetch Notion pages in an import task in parallel (#14612)
* perf: Fetch Notion pages in an import task in parallel

* fix: Settle all Notion page fetches before failing an import task
2026-11-16 23:46:43 +00:00

240 lines
8.5 KiB
TypeScript

import { APIResponseError, APIErrorCode } from "@notionhq/client";
import { uniqBy } from "es-toolkit/compat";
import type { ImportTaskInput, ImportTaskOutput } from "@shared/schema";
import type { IntegrationService, ProsemirrorDoc } from "@shared/types";
import { ProsemirrorDataHelper } from "@shared/utils/ProsemirrorDataHelper";
import { CollectionValidation, DocumentValidation } from "@shared/validations";
import Logger from "@server/logging/Logger";
import { Integration } from "@server/models";
import type ImportTask from "@server/models/ImportTask";
import type { ProcessOutput } from "@server/queues/tasks/APIImportTask";
import APIImportTask from "@server/queues/tasks/APIImportTask";
import type { Block } from "../../shared/types";
import { PageType } from "../../shared/types";
import { NotionClient } from "../notion";
import type { NotionPage } from "../utils/NotionConverter";
import { NotionConverter } from "../utils/NotionConverter";
type ChildPage = { type: PageType; externalId: string };
type ParsePageOutput = ImportTaskOutput[number] & {
rootExternalId?: string;
children: ChildPage[];
};
export default class NotionAPIImportTask extends APIImportTask<IntegrationService.Notion> {
private skippableErrorMessages = [
"Database retrievals do not support linked databases",
"does not contain any data sources accessible by this API bot", // error msg for linked database views,
"Databases with multiple data sources are not supported in this API version", // https://github.com/outline/outline/issues/11573#issuecomment-3993691460
];
/**
* Process a Notion page-phase import task.
* This fetches data from Notion and converts it to task output.
*
* @param importTask ImportTask model to process.
* @returns Promise with output that resolves once processing has completed.
*/
protected async processPage(
importTask: ImportTask<IntegrationService.Notion>
): Promise<ProcessOutput<IntegrationService.Notion>> {
if (!importTask.import.integrationId) {
throw new Error("Notion import is missing integrationId");
}
const integration = await Integration.scope("withAuthentication").findByPk(
importTask.import.integrationId,
{ rejectOnEmpty: true }
);
const client = new NotionClient(integration.authentication.token);
// Settle every page before failing so no requests outlive this attempt.
const results = await Promise.allSettled(
importTask.input.map((item) => this.parsePage({ item, client }))
);
const parsedPages: (ParsePageOutput & { ancestorExternalIds: string[] })[] =
[];
results.forEach((result, index) => {
if (result.status === "rejected") {
throw result.reason;
}
const parsedPage = result.value;
const item = importTask.input[index];
// Skip pages/databases that couldn't be accessed
if (parsedPage) {
parsedPages.push({
...parsedPage,
ancestorExternalIds: [
...(item.ancestorExternalIds ?? []),
item.externalId,
],
});
}
});
const taskOutput: ImportTaskOutput = parsedPages.map((parsedPage) => ({
externalId: parsedPage.externalId,
title: parsedPage.title,
icon: parsedPage.icon,
content: parsedPage.content,
author: parsedPage.author,
createdAt: parsedPage.createdAt,
updatedAt: parsedPage.updatedAt,
}));
const childTasksInput: ImportTaskInput<IntegrationService.Notion> =
parsedPages.flatMap((parsedPage) =>
uniqBy(parsedPage.children, "externalId")
.filter((childPage) => {
// A page that contains one of its ancestors would be queued forever.
if (parsedPage.ancestorExternalIds.includes(childPage.externalId)) {
Logger.warn(
`Skipping Notion page ${childPage.externalId} - it is an ancestor of ${parsedPage.externalId}`,
{ importId: importTask.importId }
);
return false;
}
return true;
})
.map((childPage) => ({
type: childPage.type,
externalId: childPage.externalId,
parentExternalId: parsedPage.externalId,
rootExternalId: parsedPage.rootExternalId,
ancestorExternalIds: parsedPage.ancestorExternalIds,
}))
);
return { taskOutput, childTasksInput };
}
/**
* Schedule the next `NotionAPIImportTask`.
*
* @param importTask ImportTask model associated with the `NotionAPIImportTask`.
* @returns Promise that resolves when the task is scheduled.
*/
protected async scheduleNextTask(
importTask: ImportTask<IntegrationService.Notion>
) {
await new NotionAPIImportTask().schedule({ importTaskId: importTask.id });
return;
}
/**
* Fetch page data from Notion and convert it to expected output.
*
* @param item Object containing data about a notion page (or) database.
* @param client Notion client.
* @returns Promise of parsed page output that resolves when the task is scheduled.
*/
private async parsePage({
item,
client,
}: {
item: ImportTaskInput<IntegrationService.Notion>[number];
client: NotionClient;
}): Promise<ParsePageOutput | null> {
const rootExternalId =
item.rootExternalId ?? item.collectionExternalId ?? item.externalId;
const titleMaxLength =
item.externalId === rootExternalId // This means it's a root page (materialized as a collection or root document)
? CollectionValidation.maxNameLength
: DocumentValidation.maxTitleLength;
try {
// Convert Notion database to an empty page with "pages in database" as its children.
if (item.type === PageType.Database) {
const { pages, emoji, ...databaseInfo } = await client.fetchDatabase(
item.externalId,
{ titleMaxLength }
);
return {
...databaseInfo,
icon: emoji,
externalId: item.externalId,
content: ProsemirrorDataHelper.getEmpty() as ProsemirrorDoc,
rootExternalId,
children: pages.map((page) => ({
type: page.type,
externalId: page.id,
})),
};
}
const { blocks, emoji, ...pageInfo } = await client.fetchPage(
item.externalId,
{ titleMaxLength }
);
return {
...pageInfo,
icon: emoji,
externalId: item.externalId,
content: NotionConverter.page({ children: blocks } as NotionPage),
rootExternalId,
children: this.parseChildPages(blocks),
};
} catch (error) {
if (error instanceof APIResponseError) {
// Skip this page/database if it's not found or not accessible
if (
error.code === APIErrorCode.ObjectNotFound ||
error.code === APIErrorCode.Unauthorized ||
this.skippableErrorMessages.some((errorMsg) =>
error.message.includes(errorMsg)
)
) {
Logger.warn(
`Skipping Notion ${
item.type === PageType.Database ? "database" : "page"
} ${item.externalId} - Error code: ${error.code} - ${error.message}`
);
return null;
}
// Rate limit errors should be handled by the fetchWithRetry method in NotionClient
// If we still get here, it means the maximum retries were exceeded
if (error.code === APIErrorCode.RateLimited) {
Logger.error(
`Rate limit exceeded for Notion API when processing ${
item.type === PageType.Database ? "database" : "page"
} ${item.externalId}. Maximum retries reached.`,
error
);
}
}
// Re-throw other errors to be handled by the parent try/catch
throw error;
}
}
/**
* Parse Notion page blocks to get its child pages and databases.
*
* @param pageBlocks Array of blocks representing the page's content.
* @returns Array containing child page and child database info.
*/
private parseChildPages(pageBlocks: Block[]): ChildPage[] {
const childPages: ChildPage[] = [];
pageBlocks.forEach((block) => {
if (block.type === "child_page") {
childPages.push({ type: PageType.Page, externalId: block.id });
} else if (block.type === "child_database") {
childPages.push({ type: PageType.Database, externalId: block.id });
} else if (block.children?.length) {
childPages.push(...this.parseChildPages(block.children));
}
});
return childPages;
}
}