mirror of
https://github.com/outline/outline.git
synced 2026-11-22 05:51:43 +03:00
Add import validation behavior options
- Add ImportValidationBehavior enum with Skip, Truncate, and Abort options - Update FileOperation options to include validationBehavior - Add validation behavior selection to DropToImport UI component - Update collections import API to accept validationBehavior parameter - Implement validation logic in ImportTask with document processing - Set Truncate as the new default behavior (was Abort) Addresses GitHub issue #9285
This commit is contained in:
1 parent
6efcf1beee
commit
933dde935d
7 files changed
+173
-22
No files matched your search
@@ -6,10 +6,15 @@ import { useTranslation } from "react-i18next";
|
||||
import { toast } from "sonner";
|
||||
import styled from "styled-components";
|
||||
import { s } from "@shared/styles";
|
||||
import { AttachmentPreset, CollectionPermission } from "@shared/types";
|
||||
import {
|
||||
AttachmentPreset,
|
||||
CollectionPermission,
|
||||
ImportValidationBehavior,
|
||||
} from "@shared/types";
|
||||
import { bytesToHumanReadable } from "@shared/utils/files";
|
||||
import Button from "~/components/Button";
|
||||
import Flex from "~/components/Flex";
|
||||
import InputSelect from "~/components/InputSelect";
|
||||
import InputSelectPermission from "~/components/InputSelectPermission";
|
||||
import LoadingIndicator from "~/components/LoadingIndicator";
|
||||
import Text from "~/components/Text";
|
||||
@@ -33,6 +38,8 @@ function DropToImport({ disabled, onSubmit, children, format }: Props) {
|
||||
const [permission, setPermission] = useState<CollectionPermission | null>(
|
||||
CollectionPermission.ReadWrite
|
||||
);
|
||||
const [validationBehavior, setValidationBehavior] =
|
||||
useState<ImportValidationBehavior>(ImportValidationBehavior.Truncate);
|
||||
|
||||
const handleFiles = (files: File[]) => {
|
||||
if (files.length > 1) {
|
||||
@@ -53,7 +60,11 @@ function DropToImport({ disabled, onSubmit, children, format }: Props) {
|
||||
name: file.name,
|
||||
preset: AttachmentPreset.WorkspaceImport,
|
||||
});
|
||||
await collections.import(attachment.id, { format, permission });
|
||||
await collections.import(attachment.id, {
|
||||
format,
|
||||
permission,
|
||||
validationBehavior,
|
||||
});
|
||||
onSubmit();
|
||||
toast.message(file.name, {
|
||||
description: t(
|
||||
@@ -117,6 +128,32 @@ function DropToImport({ disabled, onSubmit, children, format }: Props) {
|
||||
.
|
||||
</Text>
|
||||
</div>
|
||||
<div>
|
||||
<InputSelect
|
||||
value={validationBehavior}
|
||||
options={[
|
||||
{
|
||||
label: t("Skip invalid documents"),
|
||||
value: ImportValidationBehavior.Skip,
|
||||
},
|
||||
{
|
||||
label: t("Truncate oversized documents"),
|
||||
value: ImportValidationBehavior.Truncate,
|
||||
},
|
||||
{
|
||||
label: t("Abort import on validation errors"),
|
||||
value: ImportValidationBehavior.Abort,
|
||||
},
|
||||
]}
|
||||
onChange={setValidationBehavior}
|
||||
/>
|
||||
<Text as="span" type="secondary">
|
||||
{t(
|
||||
"Choose how to handle documents that fail validation during import"
|
||||
)}
|
||||
.
|
||||
</Text>
|
||||
</div>
|
||||
<Flex justify="flex-end">
|
||||
<Button disabled={!file} onClick={handleStartImport}>
|
||||
{t("Start import")}
|
||||
|
||||
@@ -8,6 +8,7 @@ import {
|
||||
CollectionPermission,
|
||||
CollectionStatusFilter,
|
||||
FileOperationFormat,
|
||||
ImportValidationBehavior,
|
||||
SubscriptionType,
|
||||
} from "@shared/types";
|
||||
import Collection from "~/models/Collection";
|
||||
@@ -88,7 +89,11 @@ export default class CollectionsStore extends Store<Collection> {
|
||||
@action
|
||||
import = async (
|
||||
attachmentId: string,
|
||||
options: { format?: string; permission?: CollectionPermission | null }
|
||||
options: {
|
||||
format?: string;
|
||||
permission?: CollectionPermission | null;
|
||||
validationBehavior?: ImportValidationBehavior;
|
||||
}
|
||||
) => {
|
||||
await client.post("/collections.import", {
|
||||
attachmentId,
|
||||
|
||||
@@ -18,6 +18,7 @@ import {
|
||||
FileOperationFormat,
|
||||
FileOperationState,
|
||||
FileOperationType,
|
||||
ImportValidationBehavior,
|
||||
} from "@shared/types";
|
||||
import FileStorage from "@server/storage/files";
|
||||
import Collection from "./Collection";
|
||||
@@ -29,6 +30,7 @@ import Fix from "./decorators/Fix";
|
||||
export type FileOperationOptions = {
|
||||
includeAttachments?: boolean;
|
||||
permission?: CollectionPermission | null;
|
||||
validationBehavior?: ImportValidationBehavior;
|
||||
};
|
||||
|
||||
@DefaultScope(() => ({
|
||||
|
||||
@@ -9,9 +9,10 @@ import {
|
||||
CollectionPermission,
|
||||
CollectionSort,
|
||||
FileOperationState,
|
||||
ImportValidationBehavior,
|
||||
ProsemirrorData,
|
||||
} from "@shared/types";
|
||||
import { CollectionValidation } from "@shared/validations";
|
||||
import { CollectionValidation, DocumentValidation } from "@shared/validations";
|
||||
import attachmentCreator from "@server/commands/attachmentCreator";
|
||||
import documentCreator from "@server/commands/documentCreator";
|
||||
import { createContext } from "@server/context";
|
||||
@@ -423,14 +424,29 @@ export default abstract class ImportTask extends BaseTask<Props> {
|
||||
collections.set(item.id, collection);
|
||||
|
||||
// Documents
|
||||
for (const item of data.documents.filter(
|
||||
for (const docItem of data.documents.filter(
|
||||
(d) => d.collectionId === collection.id
|
||||
)) {
|
||||
Logger.debug(
|
||||
"task",
|
||||
`ImportTask persisting document ${item.title} (${item.id})`
|
||||
`ImportTask persisting document ${docItem.title} (${docItem.id})`
|
||||
);
|
||||
let text = item.text;
|
||||
|
||||
// Apply validation behavior
|
||||
const validationBehavior =
|
||||
fileOperation.options?.validationBehavior ||
|
||||
ImportValidationBehavior.Truncate;
|
||||
const processedDocument = this.validateAndProcessDocument(
|
||||
docItem,
|
||||
validationBehavior
|
||||
);
|
||||
|
||||
// Skip this document if validation behavior is Skip and document failed validation
|
||||
if (!processedDocument) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let text = processedDocument.text;
|
||||
|
||||
// Check all of the attachments we've created against urls in the text
|
||||
// and replace them out with attachment redirect urls before saving.
|
||||
@@ -452,26 +468,30 @@ export default abstract class ImportTask extends BaseTask<Props> {
|
||||
|
||||
const document = await documentCreator({
|
||||
sourceMetadata: {
|
||||
fileName: path.basename(item.path),
|
||||
mimeType: item.mimeType,
|
||||
externalId: item.externalId,
|
||||
createdByName: item.createdByName,
|
||||
fileName: path.basename(processedDocument.path),
|
||||
mimeType: processedDocument.mimeType,
|
||||
externalId: processedDocument.externalId,
|
||||
createdByName: processedDocument.createdByName,
|
||||
},
|
||||
id: item.id,
|
||||
title: item.title,
|
||||
urlId: item.urlId,
|
||||
id: processedDocument.id,
|
||||
title: processedDocument.title,
|
||||
urlId: processedDocument.urlId,
|
||||
text,
|
||||
content: item.data,
|
||||
collectionId: item.collectionId,
|
||||
createdAt: item.createdAt,
|
||||
updatedAt: item.updatedAt ?? item.createdAt,
|
||||
publishedAt: item.updatedAt ?? item.createdAt ?? new Date(),
|
||||
parentDocumentId: item.parentDocumentId,
|
||||
content: processedDocument.data,
|
||||
collectionId: processedDocument.collectionId,
|
||||
createdAt: processedDocument.createdAt,
|
||||
updatedAt:
|
||||
processedDocument.updatedAt ?? processedDocument.createdAt,
|
||||
publishedAt:
|
||||
processedDocument.updatedAt ??
|
||||
processedDocument.createdAt ??
|
||||
new Date(),
|
||||
parentDocumentId: processedDocument.parentDocumentId,
|
||||
importId: fileOperation.id,
|
||||
user,
|
||||
ctx: createContext({ user, transaction }),
|
||||
});
|
||||
documents.set(item.id, document);
|
||||
documents.set(processedDocument.id, document);
|
||||
|
||||
await collection.addDocumentToStructure(document, undefined, {
|
||||
transaction,
|
||||
@@ -535,6 +555,81 @@ export default abstract class ImportTask extends BaseTask<Props> {
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Validates a document and applies the appropriate behavior based on validation settings.
|
||||
*
|
||||
* @param document The document to validate
|
||||
* @param validationBehavior The behavior to apply when validation fails
|
||||
* @returns The processed document or null if it should be skipped
|
||||
*/
|
||||
private validateAndProcessDocument(
|
||||
document: StructuredImportData["documents"][number],
|
||||
validationBehavior: ImportValidationBehavior
|
||||
): StructuredImportData["documents"][number] | null {
|
||||
const titleTooLong =
|
||||
document.title.length > DocumentValidation.maxTitleLength;
|
||||
const textTooLong =
|
||||
document.text.length > DocumentValidation.maxRecommendedLength;
|
||||
const stateTooLarge =
|
||||
JSON.stringify(document.data || {}).length >
|
||||
DocumentValidation.maxStateLength;
|
||||
|
||||
const hasValidationIssues = titleTooLong || textTooLong || stateTooLarge;
|
||||
|
||||
if (!hasValidationIssues) {
|
||||
return document;
|
||||
}
|
||||
|
||||
switch (validationBehavior) {
|
||||
case ImportValidationBehavior.Skip: {
|
||||
Logger.info(
|
||||
"task",
|
||||
`Skipping document "${document.title}" due to validation issues`
|
||||
);
|
||||
return null;
|
||||
}
|
||||
|
||||
case ImportValidationBehavior.Truncate: {
|
||||
const processedDocument = { ...document };
|
||||
|
||||
if (titleTooLong) {
|
||||
processedDocument.title = truncate(document.title, {
|
||||
length: DocumentValidation.maxTitleLength,
|
||||
omission: "...",
|
||||
});
|
||||
}
|
||||
|
||||
if (textTooLong) {
|
||||
processedDocument.text = truncate(document.text, {
|
||||
length: DocumentValidation.maxRecommendedLength,
|
||||
omission: "\n\n[Content truncated due to size limits]",
|
||||
});
|
||||
}
|
||||
|
||||
if (stateTooLarge && document.data) {
|
||||
// For ProseMirror data, we need to be more careful about truncation
|
||||
// For now, we'll just remove the data and rely on the text
|
||||
processedDocument.data = undefined;
|
||||
}
|
||||
|
||||
Logger.info(
|
||||
"task",
|
||||
`Truncated document "${document.title}" due to validation issues`
|
||||
);
|
||||
return processedDocument;
|
||||
}
|
||||
|
||||
case ImportValidationBehavior.Abort:
|
||||
default: {
|
||||
throw new ValidationError(
|
||||
`Document "${document.title}" exceeds validation limits. ` +
|
||||
`Title: ${document.title.length}/${DocumentValidation.maxTitleLength}, ` +
|
||||
`Text: ${document.text.length}/${DocumentValidation.maxRecommendedLength} characters`
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Job options such as priority and retry strategy, as defined by Bull.
|
||||
*/
|
||||
|
||||
@@ -177,7 +177,8 @@ router.post(
|
||||
transaction(),
|
||||
async (ctx: APIContext<T.CollectionsImportReq>) => {
|
||||
const { transaction } = ctx.state;
|
||||
const { attachmentId, permission, format } = ctx.input.body;
|
||||
const { attachmentId, permission, format, validationBehavior } =
|
||||
ctx.input.body;
|
||||
const { user } = ctx.state.auth;
|
||||
authorize(user, "importCollection", user.team);
|
||||
|
||||
@@ -196,6 +197,7 @@ router.post(
|
||||
teamId: user.teamId,
|
||||
options: {
|
||||
permission,
|
||||
validationBehavior,
|
||||
},
|
||||
});
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ import {
|
||||
CollectionPermission,
|
||||
CollectionStatusFilter,
|
||||
FileOperationFormat,
|
||||
ImportValidationBehavior,
|
||||
} from "@shared/types";
|
||||
import { Collection } from "@server/models";
|
||||
import { zodIconType, zodIdType } from "@server/utils/zod";
|
||||
@@ -73,6 +74,9 @@ export const CollectionsImportSchema = BaseSchema.extend({
|
||||
format: z
|
||||
.nativeEnum(FileOperationFormat)
|
||||
.default(FileOperationFormat.MarkdownZip),
|
||||
validationBehavior: z
|
||||
.nativeEnum(ImportValidationBehavior)
|
||||
.default(ImportValidationBehavior.Truncate),
|
||||
}),
|
||||
});
|
||||
|
||||
|
||||
@@ -552,3 +552,9 @@ export type ReactionSummary = {
|
||||
emoji: string;
|
||||
userIds: string[];
|
||||
};
|
||||
|
||||
export enum ImportValidationBehavior {
|
||||
Skip = "skip",
|
||||
Truncate = "truncate",
|
||||
Abort = "abort",
|
||||
}
|
||||
Reference in new issue
Block a user