mirror of
https://github.com/outline/outline.git
synced 2026-11-21 21:44:30 +03:00
* feat: Map creator/updater IDs to existing users during JSON import When importing documents from JSON, resolve the original document author to an internal user by matching on user ID first, then email, falling back to the importing user. Results are cached to avoid redundant queries. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * fix: Add negative caching for user resolution during import Cache misses (not just hits) in resolveUserId so that repeated lookups for users that don't exist in the target team are served from cache instead of hitting the database for every document. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * docs: Fix resolveUserId JSDoc to match actual behavior Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
623 lines
18 KiB
TypeScript
623 lines
18 KiB
TypeScript
import path from "node:path";
|
|
import fs from "fs-extra";
|
|
import chunk from "lodash/chunk";
|
|
import truncate from "lodash/truncate";
|
|
import type { InferCreationAttributes } from "sequelize";
|
|
import tmp from "tmp";
|
|
import type { CollectionSort, ProsemirrorData } from "@shared/types";
|
|
import {
|
|
AttachmentPreset,
|
|
CollectionPermission,
|
|
FileOperationState,
|
|
} from "@shared/types";
|
|
import { CollectionValidation } from "@shared/validations";
|
|
import attachmentCreator from "@server/commands/attachmentCreator";
|
|
import documentCreator from "@server/commands/documentCreator";
|
|
import { createContext } from "@server/context";
|
|
import { InternalError, ValidationError } from "@server/errors";
|
|
import Logger from "@server/logging/Logger";
|
|
import {
|
|
User,
|
|
Event,
|
|
Document,
|
|
Collection,
|
|
FileOperation,
|
|
Attachment,
|
|
} from "@server/models";
|
|
import { sequelize } from "@server/storage/database";
|
|
import ZipHelper from "@server/utils/ZipHelper";
|
|
import { generateUrlId } from "@server/utils/url";
|
|
import { BaseTask, TaskPriority } from "./base/BaseTask";
|
|
import env from "@server/env";
|
|
|
|
type Props = {
|
|
fileOperationId: string;
|
|
};
|
|
|
|
/**
|
|
* Standardized format for data importing, to be used by all import tasks.
|
|
*/
|
|
export type StructuredImportData = {
|
|
collections: {
|
|
id: string;
|
|
urlId?: string;
|
|
color?: string | null;
|
|
icon?: string | null;
|
|
sort?: CollectionSort;
|
|
permission?: CollectionPermission | null;
|
|
name: string;
|
|
/**
|
|
* The collection description. To reference an attachment or image use the
|
|
* special formatting <<attachmentId>>. It will be replaced with a reference
|
|
* to the actual attachment as part of persistData.
|
|
*
|
|
* To reference a document use <<documentId>>, it will be replaced with a
|
|
* link to the document as part of persistData once the document url is
|
|
* generated.
|
|
*/
|
|
description?: string | null;
|
|
data?: ProsemirrorData | null;
|
|
/** Optional id from import source, useful for mapping */
|
|
externalId?: string;
|
|
}[];
|
|
documents: {
|
|
id: string;
|
|
urlId?: string;
|
|
title: string;
|
|
emoji?: string | null;
|
|
icon?: string | null;
|
|
color?: string | null;
|
|
/**
|
|
* The document text. To reference an attachment or image use the special
|
|
* formatting <<attachmentId>>. It will be replaced with a reference to the
|
|
* actual attachment as part of persistData.
|
|
*
|
|
* To reference another document use <<documentId>>, it will be replaced
|
|
* with a link to the document as part of persistData once the document url
|
|
* is generated.
|
|
*/
|
|
text: string;
|
|
data?: ProsemirrorData;
|
|
collectionId: string;
|
|
updatedAt?: Date;
|
|
createdAt?: Date;
|
|
publishedAt?: Date | null;
|
|
parentDocumentId?: string | null;
|
|
createdById?: string;
|
|
createdByName?: string;
|
|
createdByEmail?: string | null;
|
|
path: string;
|
|
mimeType: string;
|
|
/** Optional id from import source, useful for mapping */
|
|
externalId?: string;
|
|
}[];
|
|
attachments: {
|
|
id: string;
|
|
name: string;
|
|
path: string;
|
|
mimeType: string;
|
|
buffer: () => Promise<Buffer>;
|
|
/** Optional id from import source, useful for mapping */
|
|
externalId?: string;
|
|
}[];
|
|
};
|
|
|
|
export default abstract class ImportTask extends BaseTask<Props> {
|
|
/**
|
|
* Runs the import task.
|
|
*
|
|
* @param props The props
|
|
*/
|
|
public async perform({ fileOperationId }: Props) {
|
|
let dirPath;
|
|
const fileOperation = await FileOperation.unscoped().findByPk(
|
|
fileOperationId,
|
|
{
|
|
rejectOnEmpty: true,
|
|
}
|
|
);
|
|
|
|
try {
|
|
Logger.info("task", `ImportTask fetching data for ${fileOperationId}`);
|
|
dirPath = await this.fetchAndExtractData(fileOperation);
|
|
if (!dirPath) {
|
|
throw InternalError("Failed to fetch data for import from storage.");
|
|
}
|
|
|
|
Logger.info("task", `ImportTask parsing data for ${fileOperationId}`, {
|
|
dirPath,
|
|
});
|
|
const parsed = await this.parseData(dirPath, fileOperation);
|
|
|
|
if (parsed.collections.length === 0) {
|
|
throw ValidationError(
|
|
"Uploaded file does not contain any valid collections. It may be corrupt, the wrong type, or version."
|
|
);
|
|
}
|
|
|
|
if (parsed.documents.length === 0) {
|
|
throw ValidationError(
|
|
"Uploaded file does not contain any valid documents"
|
|
);
|
|
}
|
|
|
|
let result;
|
|
try {
|
|
Logger.info(
|
|
"task",
|
|
`ImportTask persisting data for ${fileOperationId}`
|
|
);
|
|
result = await this.persistData(parsed, fileOperation);
|
|
} catch (error) {
|
|
Logger.error(
|
|
`ImportTask failed to persist data for ${fileOperationId}`,
|
|
error
|
|
);
|
|
throw new Error("Sorry, an internal error occurred during import");
|
|
}
|
|
|
|
await this.updateFileOperation(
|
|
fileOperation,
|
|
FileOperationState.Complete
|
|
);
|
|
|
|
return result;
|
|
} catch (error) {
|
|
Logger.error(`ImportTask failed for ${fileOperationId}`, error);
|
|
|
|
await this.updateFileOperation(
|
|
fileOperation,
|
|
FileOperationState.Error,
|
|
error
|
|
);
|
|
throw error;
|
|
} finally {
|
|
if (dirPath) {
|
|
await this.cleanupExtractedData(dirPath, fileOperation);
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Update the state of the underlying FileOperation in the database and send
|
|
* an event to the client.
|
|
*
|
|
* @param fileOperation The FileOperation to update
|
|
*/
|
|
private async updateFileOperation(
|
|
fileOperation: FileOperation,
|
|
state: FileOperationState,
|
|
error?: Error
|
|
) {
|
|
await fileOperation.update(
|
|
{
|
|
state,
|
|
error: error ? truncate(error.message, { length: 255 }) : undefined,
|
|
},
|
|
{
|
|
hooks: false,
|
|
}
|
|
);
|
|
await Event.schedule({
|
|
name: "fileOperations.update",
|
|
modelId: fileOperation.id,
|
|
teamId: fileOperation.teamId,
|
|
actorId: fileOperation.userId,
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Fetch the remote data associated with the file operation into a temporary disk location.
|
|
*
|
|
* @param fileOperation The FileOperation to fetch data for
|
|
* @returns A promise that resolves to the temporary file path.
|
|
*/
|
|
protected async fetchAndExtractData(
|
|
fileOperation: FileOperation
|
|
): Promise<string> {
|
|
let cleanup;
|
|
let filePath: string;
|
|
|
|
try {
|
|
const res = await fileOperation.handle;
|
|
filePath = res.path;
|
|
cleanup = res.cleanup;
|
|
|
|
const tmpPath = await new Promise<string>((resolve, reject) => {
|
|
tmp.dir((err, tmpDir) => {
|
|
if (err) {
|
|
Logger.error("Could not create temporary directory", err);
|
|
return reject(err);
|
|
}
|
|
|
|
Logger.debug(
|
|
"task",
|
|
`ImportTask extracting data for ${fileOperation.id}`
|
|
);
|
|
|
|
void ZipHelper.extract(filePath, tmpDir)
|
|
.then(() => resolve(tmpDir))
|
|
.catch((zErr) => {
|
|
Logger.error("Could not extract zip file", zErr);
|
|
reject(zErr);
|
|
});
|
|
});
|
|
});
|
|
|
|
return tmpPath;
|
|
} finally {
|
|
Logger.debug(
|
|
"task",
|
|
`ImportTask cleaning up temporary data for ${fileOperation.id}`
|
|
);
|
|
|
|
await cleanup?.();
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Cleanup the temporary directory where the data was fetched and extracted.
|
|
*
|
|
* @param dirPath The temporary directory path where the data was fetched
|
|
* @param fileOperation The associated FileOperation
|
|
*/
|
|
protected async cleanupExtractedData(
|
|
dirPath: string,
|
|
fileOperation: FileOperation
|
|
) {
|
|
try {
|
|
await fs.rm(dirPath, { recursive: true, force: true });
|
|
} catch (error) {
|
|
Logger.error(
|
|
`ImportTask failed to cleanup extracted data for ${fileOperation.id}`,
|
|
error
|
|
);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Parse the data loaded from fetchAndExtractData into a consistent structured format
|
|
* that represents collections, documents, and the relationships between them.
|
|
*
|
|
* @param dirPath The temporary directory path where the data was fetched
|
|
* @param fileOperation The FileOperation to parse data for
|
|
* @returns A promise that resolves to the structured data
|
|
*/
|
|
protected abstract parseData(
|
|
dirPath: string,
|
|
fileOperation: FileOperation
|
|
): Promise<StructuredImportData>;
|
|
|
|
/**
|
|
* Persist the data that was already fetched and parsed into the consistent
|
|
* structured data.
|
|
*
|
|
* @param props The props
|
|
*/
|
|
protected async persistData(
|
|
data: StructuredImportData,
|
|
fileOperation: FileOperation
|
|
): Promise<{
|
|
collections: Map<string, Collection>;
|
|
documents: Map<string, Document>;
|
|
attachments: Map<string, Attachment>;
|
|
}> {
|
|
const collections = new Map<string, Collection>();
|
|
const documents = new Map<string, Document>();
|
|
const attachments = new Map<string, Attachment>();
|
|
const userIdCache = new Map<string, string | undefined>();
|
|
|
|
const user = await User.findByPk(fileOperation.userId, {
|
|
rejectOnEmpty: true,
|
|
});
|
|
|
|
try {
|
|
await this.preprocessDocUrlIds(data);
|
|
|
|
// Collections
|
|
for (const item of data.collections) {
|
|
await sequelize.transaction(async (transaction) => {
|
|
Logger.debug(
|
|
"task",
|
|
`ImportTask persisting collection ${item.name} (${item.id})`
|
|
);
|
|
let description = item.description;
|
|
|
|
if (description) {
|
|
// Check all of the attachments we've created against urls in the text
|
|
// and replace them out with attachment redirect urls before saving.
|
|
for (const aitem of data.attachments) {
|
|
description = description.replace(
|
|
new RegExp(`<<${aitem.id}>>`, "g"),
|
|
Attachment.getRedirectUrl(aitem.id)
|
|
);
|
|
}
|
|
|
|
// Check all of the document we've created against urls in the text
|
|
// and replace them out with a valid internal link.
|
|
for (const ditem of data.documents) {
|
|
description = description.replace(
|
|
new RegExp(`<<${ditem.id}>>`, "g"),
|
|
Document.getPath({ title: ditem.title, urlId: ditem.urlId! })
|
|
);
|
|
}
|
|
}
|
|
|
|
const options: { urlId?: string } = {};
|
|
if (item.urlId) {
|
|
const existing = await Collection.unscoped().findOne({
|
|
attributes: ["id"],
|
|
paranoid: false,
|
|
transaction,
|
|
where: {
|
|
urlId: item.urlId,
|
|
},
|
|
});
|
|
|
|
if (!existing) {
|
|
options.urlId = item.urlId;
|
|
}
|
|
}
|
|
|
|
const truncatedDescription = description
|
|
? truncate(description, {
|
|
length: CollectionValidation.maxDescriptionLength,
|
|
})
|
|
: null;
|
|
|
|
const sharedDefaults: Partial<InferCreationAttributes<Collection>> = {
|
|
...options,
|
|
id: item.id,
|
|
description: truncatedDescription,
|
|
content: item.data,
|
|
color: item.color,
|
|
icon: item.icon,
|
|
sort: item.sort,
|
|
createdById: fileOperation.userId,
|
|
permission:
|
|
(item.permission ??
|
|
fileOperation.options?.permission !== undefined)
|
|
? fileOperation.options?.permission
|
|
: CollectionPermission.ReadWrite,
|
|
importId: fileOperation.id,
|
|
};
|
|
|
|
const ctx = createContext({ user, transaction });
|
|
|
|
// check if collection with name exists
|
|
const response = await Collection.findOrCreateWithCtx(ctx, {
|
|
where: {
|
|
teamId: fileOperation.teamId,
|
|
name: item.name,
|
|
},
|
|
defaults: sharedDefaults,
|
|
});
|
|
|
|
let collection = response[0];
|
|
const isCreated = response[1];
|
|
|
|
// create new collection if name already exists, yes it's possible that
|
|
// there is also a "Name (Imported)" but this is a case not worth dealing
|
|
// with right now
|
|
if (!isCreated) {
|
|
const name = `${item.name} (Imported)`;
|
|
collection = await Collection.createWithCtx(ctx, {
|
|
...sharedDefaults,
|
|
name,
|
|
teamId: fileOperation.teamId,
|
|
});
|
|
}
|
|
|
|
collections.set(item.id, collection);
|
|
|
|
// Documents
|
|
for (const item of data.documents.filter(
|
|
(d) => d.collectionId === collection.id
|
|
)) {
|
|
Logger.debug(
|
|
"task",
|
|
`ImportTask persisting document ${item.title} (${item.id})`
|
|
);
|
|
let text = item.text;
|
|
|
|
// Check all of the attachments we've created against urls in the text
|
|
// and replace them out with attachment redirect urls before saving.
|
|
for (const aitem of data.attachments) {
|
|
text = text.replace(
|
|
new RegExp(`<<${aitem.id}>>`, "g"),
|
|
Attachment.getRedirectUrl(aitem.id)
|
|
);
|
|
}
|
|
|
|
// Check all of the document we've created against urls in the text
|
|
// and replace them out with a valid internal link.
|
|
for (const ditem of data.documents) {
|
|
text = text.replace(
|
|
new RegExp(`<<${ditem.id}>>`, "g"),
|
|
Document.getPath({ title: ditem.title, urlId: ditem.urlId! })
|
|
);
|
|
}
|
|
|
|
const resolvedUserId =
|
|
(await this.resolveUserId(
|
|
item,
|
|
fileOperation.teamId,
|
|
userIdCache
|
|
)) ?? fileOperation.userId;
|
|
|
|
const document = await documentCreator(ctx, {
|
|
sourceMetadata: {
|
|
fileName: path.basename(item.path),
|
|
mimeType: item.mimeType,
|
|
externalId: item.externalId,
|
|
createdByName: item.createdByName,
|
|
},
|
|
id: item.id,
|
|
title: item.title,
|
|
urlId: item.urlId,
|
|
text,
|
|
content: item.data,
|
|
icon: item.icon,
|
|
color: item.color,
|
|
collectionId: item.collectionId,
|
|
createdAt: item.createdAt,
|
|
updatedAt: item.updatedAt ?? item.createdAt,
|
|
publishedAt: item.updatedAt ?? item.createdAt ?? new Date(),
|
|
parentDocumentId: item.parentDocumentId,
|
|
importId: fileOperation.id,
|
|
createdById: resolvedUserId,
|
|
lastModifiedById: resolvedUserId,
|
|
});
|
|
documents.set(item.id, document);
|
|
|
|
await collection.addDocumentToStructure(document, undefined, {
|
|
transaction,
|
|
save: false,
|
|
insertOrder: "append",
|
|
});
|
|
}
|
|
|
|
await collection.save({ transaction });
|
|
});
|
|
}
|
|
|
|
// Attachments
|
|
await sequelize.transaction(async (transaction) => {
|
|
const chunks = chunk(data.attachments, 10);
|
|
|
|
for (const attChunk of chunks) {
|
|
// Parallelize 10 uploads at a time
|
|
await Promise.all(
|
|
attChunk.map(async (item) => {
|
|
Logger.debug(
|
|
"task",
|
|
`ImportTask persisting attachment ${item.name} (${item.id})`
|
|
);
|
|
const attachment = await attachmentCreator({
|
|
source: "import",
|
|
preset: AttachmentPreset.DocumentAttachment,
|
|
id: item.id,
|
|
name: item.name,
|
|
type: item.mimeType,
|
|
buffer: await item.buffer(),
|
|
user,
|
|
ctx: createContext({ user, transaction }),
|
|
fetchOptions: {
|
|
timeout: env.FILE_STORAGE_IMPORT_TIMEOUT,
|
|
},
|
|
});
|
|
if (attachment) {
|
|
attachments.set(item.id, attachment);
|
|
}
|
|
})
|
|
);
|
|
}
|
|
});
|
|
} catch (err) {
|
|
Logger.info(
|
|
"task",
|
|
`Removing ${attachments.size} attachments on failure`
|
|
);
|
|
|
|
await Promise.all(
|
|
Array.from(attachments.values()).map((model) =>
|
|
Attachment.deleteAttachmentFromS3(model)
|
|
)
|
|
);
|
|
throw err;
|
|
}
|
|
|
|
// Return value is only used for testing
|
|
return {
|
|
collections,
|
|
documents,
|
|
attachments,
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Job options such as priority and retry strategy, as defined by Bull.
|
|
*/
|
|
public get options() {
|
|
return {
|
|
priority: TaskPriority.Low,
|
|
attempts: 1,
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Resolves the original document author to an internal user, using a cache
|
|
* to avoid redundant database queries. Attempts to match by user ID first,
|
|
* then by email. Both hits and misses are cached.
|
|
*
|
|
* @param item the document import item containing createdById and createdByEmail.
|
|
* @param teamId the team ID to scope the lookup to.
|
|
* @param cache a map used to cache resolved user IDs across calls.
|
|
* @returns the resolved user ID, or undefined if no match was found.
|
|
*/
|
|
private async resolveUserId(
|
|
item: { createdById?: string; createdByEmail?: string | null },
|
|
teamId: string,
|
|
cache: Map<string, string | undefined>
|
|
): Promise<string | undefined> {
|
|
if (item.createdById) {
|
|
const cacheKey = `id:${item.createdById}`;
|
|
if (cache.has(cacheKey)) {
|
|
return cache.get(cacheKey);
|
|
}
|
|
|
|
const user = await User.findOne({
|
|
where: { id: item.createdById, teamId },
|
|
});
|
|
if (user) {
|
|
cache.set(cacheKey, user.id);
|
|
return user.id;
|
|
}
|
|
cache.set(cacheKey, undefined);
|
|
}
|
|
|
|
if (item.createdByEmail) {
|
|
const email = item.createdByEmail.toLowerCase().trim();
|
|
const cacheKey = `email:${email}`;
|
|
if (cache.has(cacheKey)) {
|
|
return cache.get(cacheKey);
|
|
}
|
|
|
|
const user = await User.findOne({
|
|
where: { email, teamId },
|
|
});
|
|
if (user) {
|
|
cache.set(cacheKey, user.id);
|
|
if (item.createdById) {
|
|
cache.set(`id:${item.createdById}`, user.id);
|
|
}
|
|
return user.id;
|
|
}
|
|
cache.set(cacheKey, undefined);
|
|
}
|
|
|
|
return undefined;
|
|
}
|
|
|
|
private async preprocessDocUrlIds(data: StructuredImportData) {
|
|
for (const doc of data.documents) {
|
|
// check DB only if urlId is present in the input.
|
|
if (doc.urlId) {
|
|
const existing = await Document.unscoped().findOne({
|
|
attributes: ["id"],
|
|
paranoid: false,
|
|
where: {
|
|
urlId: doc.urlId,
|
|
},
|
|
});
|
|
|
|
if (!existing) {
|
|
continue;
|
|
}
|
|
}
|
|
|
|
doc.urlId = generateUrlId();
|
|
}
|
|
}
|
|
}
|