mirror of
https://github.com/outline/outline.git
synced 2026-11-20 21:23:06 +03:00
* perf: Fetch Notion pages in an import task in parallel * fix: Settle all Notion page fetches before failing an import task
240 lines
8.5 KiB
TypeScript
240 lines
8.5 KiB
TypeScript
import { APIResponseError, APIErrorCode } from "@notionhq/client";
|
|
import { uniqBy } from "es-toolkit/compat";
|
|
import type { ImportTaskInput, ImportTaskOutput } from "@shared/schema";
|
|
import type { IntegrationService, ProsemirrorDoc } from "@shared/types";
|
|
import { ProsemirrorDataHelper } from "@shared/utils/ProsemirrorDataHelper";
|
|
import { CollectionValidation, DocumentValidation } from "@shared/validations";
|
|
import Logger from "@server/logging/Logger";
|
|
import { Integration } from "@server/models";
|
|
import type ImportTask from "@server/models/ImportTask";
|
|
import type { ProcessOutput } from "@server/queues/tasks/APIImportTask";
|
|
import APIImportTask from "@server/queues/tasks/APIImportTask";
|
|
import type { Block } from "../../shared/types";
|
|
import { PageType } from "../../shared/types";
|
|
import { NotionClient } from "../notion";
|
|
import type { NotionPage } from "../utils/NotionConverter";
|
|
import { NotionConverter } from "../utils/NotionConverter";
|
|
|
|
type ChildPage = { type: PageType; externalId: string };
|
|
|
|
type ParsePageOutput = ImportTaskOutput[number] & {
|
|
rootExternalId?: string;
|
|
children: ChildPage[];
|
|
};
|
|
|
|
export default class NotionAPIImportTask extends APIImportTask<IntegrationService.Notion> {
|
|
private skippableErrorMessages = [
|
|
"Database retrievals do not support linked databases",
|
|
"does not contain any data sources accessible by this API bot", // error msg for linked database views,
|
|
"Databases with multiple data sources are not supported in this API version", // https://github.com/outline/outline/issues/11573#issuecomment-3993691460
|
|
];
|
|
|
|
/**
|
|
* Process a Notion page-phase import task.
|
|
* This fetches data from Notion and converts it to task output.
|
|
*
|
|
* @param importTask ImportTask model to process.
|
|
* @returns Promise with output that resolves once processing has completed.
|
|
*/
|
|
protected async processPage(
|
|
importTask: ImportTask<IntegrationService.Notion>
|
|
): Promise<ProcessOutput<IntegrationService.Notion>> {
|
|
if (!importTask.import.integrationId) {
|
|
throw new Error("Notion import is missing integrationId");
|
|
}
|
|
|
|
const integration = await Integration.scope("withAuthentication").findByPk(
|
|
importTask.import.integrationId,
|
|
{ rejectOnEmpty: true }
|
|
);
|
|
|
|
const client = new NotionClient(integration.authentication.token);
|
|
|
|
// Settle every page before failing so no requests outlive this attempt.
|
|
const results = await Promise.allSettled(
|
|
importTask.input.map((item) => this.parsePage({ item, client }))
|
|
);
|
|
|
|
const parsedPages: (ParsePageOutput & { ancestorExternalIds: string[] })[] =
|
|
[];
|
|
results.forEach((result, index) => {
|
|
if (result.status === "rejected") {
|
|
throw result.reason;
|
|
}
|
|
|
|
const parsedPage = result.value;
|
|
const item = importTask.input[index];
|
|
|
|
// Skip pages/databases that couldn't be accessed
|
|
if (parsedPage) {
|
|
parsedPages.push({
|
|
...parsedPage,
|
|
ancestorExternalIds: [
|
|
...(item.ancestorExternalIds ?? []),
|
|
item.externalId,
|
|
],
|
|
});
|
|
}
|
|
});
|
|
|
|
const taskOutput: ImportTaskOutput = parsedPages.map((parsedPage) => ({
|
|
externalId: parsedPage.externalId,
|
|
title: parsedPage.title,
|
|
icon: parsedPage.icon,
|
|
content: parsedPage.content,
|
|
author: parsedPage.author,
|
|
createdAt: parsedPage.createdAt,
|
|
updatedAt: parsedPage.updatedAt,
|
|
}));
|
|
|
|
const childTasksInput: ImportTaskInput<IntegrationService.Notion> =
|
|
parsedPages.flatMap((parsedPage) =>
|
|
uniqBy(parsedPage.children, "externalId")
|
|
.filter((childPage) => {
|
|
// A page that contains one of its ancestors would be queued forever.
|
|
if (parsedPage.ancestorExternalIds.includes(childPage.externalId)) {
|
|
Logger.warn(
|
|
`Skipping Notion page ${childPage.externalId} - it is an ancestor of ${parsedPage.externalId}`,
|
|
{ importId: importTask.importId }
|
|
);
|
|
return false;
|
|
}
|
|
return true;
|
|
})
|
|
.map((childPage) => ({
|
|
type: childPage.type,
|
|
externalId: childPage.externalId,
|
|
parentExternalId: parsedPage.externalId,
|
|
rootExternalId: parsedPage.rootExternalId,
|
|
ancestorExternalIds: parsedPage.ancestorExternalIds,
|
|
}))
|
|
);
|
|
|
|
return { taskOutput, childTasksInput };
|
|
}
|
|
|
|
/**
|
|
* Schedule the next `NotionAPIImportTask`.
|
|
*
|
|
* @param importTask ImportTask model associated with the `NotionAPIImportTask`.
|
|
* @returns Promise that resolves when the task is scheduled.
|
|
*/
|
|
protected async scheduleNextTask(
|
|
importTask: ImportTask<IntegrationService.Notion>
|
|
) {
|
|
await new NotionAPIImportTask().schedule({ importTaskId: importTask.id });
|
|
return;
|
|
}
|
|
|
|
/**
|
|
* Fetch page data from Notion and convert it to expected output.
|
|
*
|
|
* @param item Object containing data about a notion page (or) database.
|
|
* @param client Notion client.
|
|
* @returns Promise of parsed page output that resolves when the task is scheduled.
|
|
*/
|
|
private async parsePage({
|
|
item,
|
|
client,
|
|
}: {
|
|
item: ImportTaskInput<IntegrationService.Notion>[number];
|
|
client: NotionClient;
|
|
}): Promise<ParsePageOutput | null> {
|
|
const rootExternalId =
|
|
item.rootExternalId ?? item.collectionExternalId ?? item.externalId;
|
|
const titleMaxLength =
|
|
item.externalId === rootExternalId // This means it's a root page (materialized as a collection or root document)
|
|
? CollectionValidation.maxNameLength
|
|
: DocumentValidation.maxTitleLength;
|
|
|
|
try {
|
|
// Convert Notion database to an empty page with "pages in database" as its children.
|
|
if (item.type === PageType.Database) {
|
|
const { pages, emoji, ...databaseInfo } = await client.fetchDatabase(
|
|
item.externalId,
|
|
{ titleMaxLength }
|
|
);
|
|
|
|
return {
|
|
...databaseInfo,
|
|
icon: emoji,
|
|
externalId: item.externalId,
|
|
content: ProsemirrorDataHelper.getEmpty() as ProsemirrorDoc,
|
|
rootExternalId,
|
|
children: pages.map((page) => ({
|
|
type: page.type,
|
|
externalId: page.id,
|
|
})),
|
|
};
|
|
}
|
|
|
|
const { blocks, emoji, ...pageInfo } = await client.fetchPage(
|
|
item.externalId,
|
|
{ titleMaxLength }
|
|
);
|
|
|
|
return {
|
|
...pageInfo,
|
|
icon: emoji,
|
|
externalId: item.externalId,
|
|
content: NotionConverter.page({ children: blocks } as NotionPage),
|
|
rootExternalId,
|
|
children: this.parseChildPages(blocks),
|
|
};
|
|
} catch (error) {
|
|
if (error instanceof APIResponseError) {
|
|
// Skip this page/database if it's not found or not accessible
|
|
if (
|
|
error.code === APIErrorCode.ObjectNotFound ||
|
|
error.code === APIErrorCode.Unauthorized ||
|
|
this.skippableErrorMessages.some((errorMsg) =>
|
|
error.message.includes(errorMsg)
|
|
)
|
|
) {
|
|
Logger.warn(
|
|
`Skipping Notion ${
|
|
item.type === PageType.Database ? "database" : "page"
|
|
} ${item.externalId} - Error code: ${error.code} - ${error.message}`
|
|
);
|
|
return null;
|
|
}
|
|
|
|
// Rate limit errors should be handled by the fetchWithRetry method in NotionClient
|
|
// If we still get here, it means the maximum retries were exceeded
|
|
if (error.code === APIErrorCode.RateLimited) {
|
|
Logger.error(
|
|
`Rate limit exceeded for Notion API when processing ${
|
|
item.type === PageType.Database ? "database" : "page"
|
|
} ${item.externalId}. Maximum retries reached.`,
|
|
error
|
|
);
|
|
}
|
|
}
|
|
// Re-throw other errors to be handled by the parent try/catch
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Parse Notion page blocks to get its child pages and databases.
|
|
*
|
|
* @param pageBlocks Array of blocks representing the page's content.
|
|
* @returns Array containing child page and child database info.
|
|
*/
|
|
private parseChildPages(pageBlocks: Block[]): ChildPage[] {
|
|
const childPages: ChildPage[] = [];
|
|
|
|
pageBlocks.forEach((block) => {
|
|
if (block.type === "child_page") {
|
|
childPages.push({ type: PageType.Page, externalId: block.id });
|
|
} else if (block.type === "child_database") {
|
|
childPages.push({ type: PageType.Database, externalId: block.id });
|
|
} else if (block.children?.length) {
|
|
childPages.push(...this.parseChildPages(block.children));
|
|
}
|
|
});
|
|
|
|
return childPages;
|
|
}
|
|
}
|