Add import validation behavior options

- Add ImportValidationBehavior enum with Skip, Truncate, and Abort options
- Update FileOperation options to include validationBehavior
- Add validation behavior selection to DropToImport UI component
- Update collections import API to accept validationBehavior parameter
- Implement validation logic in ImportTask with document processing
- Set Truncate as the new default behavior (was Abort)

Addresses GitHub issue #9285
This commit is contained in:
codegen-sh[bot] committed 2025-05-31 14:49:54 +00:00
1 parent 6efcf1beee
commit 933dde935d
7 files changed
+173 -22

No files matched your search

@@ -6,10 +6,15 @@ import { useTranslation } from "react-i18next";
import { toast } from "sonner";
import styled from "styled-components";
import { s } from "@shared/styles";
import { AttachmentPreset, CollectionPermission } from "@shared/types";
import {
AttachmentPreset,
CollectionPermission,
ImportValidationBehavior,
} from "@shared/types";
import { bytesToHumanReadable } from "@shared/utils/files";
import Button from "~/components/Button";
import Flex from "~/components/Flex";
import InputSelect from "~/components/InputSelect";
import InputSelectPermission from "~/components/InputSelectPermission";
import LoadingIndicator from "~/components/LoadingIndicator";
import Text from "~/components/Text";
@@ -33,6 +38,8 @@ function DropToImport({ disabled, onSubmit, children, format }: Props) {
const [permission, setPermission] = useState<CollectionPermission | null>(
CollectionPermission.ReadWrite
);
const [validationBehavior, setValidationBehavior] =
useState<ImportValidationBehavior>(ImportValidationBehavior.Truncate);
const handleFiles = (files: File[]) => {
if (files.length > 1) {
@@ -53,7 +60,11 @@ function DropToImport({ disabled, onSubmit, children, format }: Props) {
name: file.name,
preset: AttachmentPreset.WorkspaceImport,
});
await collections.import(attachment.id, { format, permission });
await collections.import(attachment.id, {
format,
permission,
validationBehavior,
});
onSubmit();
toast.message(file.name, {
description: t(
@@ -117,6 +128,32 @@ function DropToImport({ disabled, onSubmit, children, format }: Props) {
.
</Text>
</div>
<div>
<InputSelect
value={validationBehavior}
options={[
{
label: t("Skip invalid documents"),
value: ImportValidationBehavior.Skip,
},
{
label: t("Truncate oversized documents"),
value: ImportValidationBehavior.Truncate,
},
{
label: t("Abort import on validation errors"),
value: ImportValidationBehavior.Abort,
},
]}
onChange={setValidationBehavior}
/>
<Text as="span" type="secondary">
{t(
"Choose how to handle documents that fail validation during import"
)}
.
</Text>
</div>
<Flex justify="flex-end">
<Button disabled={!file} onClick={handleStartImport}>
{t("Start import")}
+6 -1
View File
@@ -8,6 +8,7 @@ import {
CollectionPermission,
CollectionStatusFilter,
FileOperationFormat,
ImportValidationBehavior,
SubscriptionType,
} from "@shared/types";
import Collection from "~/models/Collection";
@@ -88,7 +89,11 @@ export default class CollectionsStore extends Store<Collection> {
@action
import = async (
attachmentId: string,
options: { format?: string; permission?: CollectionPermission | null }
options: {
format?: string;
permission?: CollectionPermission | null;
validationBehavior?: ImportValidationBehavior;
}
) => {
await client.post("/collections.import", {
attachmentId,
+2
View File
@@ -18,6 +18,7 @@ import {
FileOperationFormat,
FileOperationState,
FileOperationType,
ImportValidationBehavior,
} from "@shared/types";
import FileStorage from "@server/storage/files";
import Collection from "./Collection";
@@ -29,6 +30,7 @@ import Fix from "./decorators/Fix";
export type FileOperationOptions = {
includeAttachments?: boolean;
permission?: CollectionPermission | null;
validationBehavior?: ImportValidationBehavior;
};
@DefaultScope(() => ({
+113 -18
View File
@@ -9,9 +9,10 @@ import {
CollectionPermission,
CollectionSort,
FileOperationState,
ImportValidationBehavior,
ProsemirrorData,
} from "@shared/types";
import { CollectionValidation } from "@shared/validations";
import { CollectionValidation, DocumentValidation } from "@shared/validations";
import attachmentCreator from "@server/commands/attachmentCreator";
import documentCreator from "@server/commands/documentCreator";
import { createContext } from "@server/context";
@@ -423,14 +424,29 @@ export default abstract class ImportTask extends BaseTask<Props> {
collections.set(item.id, collection);
// Documents
for (const item of data.documents.filter(
for (const docItem of data.documents.filter(
(d) => d.collectionId === collection.id
)) {
Logger.debug(
"task",
`ImportTask persisting document ${item.title} (${item.id})`
`ImportTask persisting document ${docItem.title} (${docItem.id})`
);
let text = item.text;
// Apply validation behavior
const validationBehavior =
fileOperation.options?.validationBehavior ||
ImportValidationBehavior.Truncate;
const processedDocument = this.validateAndProcessDocument(
docItem,
validationBehavior
);
// Skip this document if validation behavior is Skip and document failed validation
if (!processedDocument) {
continue;
}
let text = processedDocument.text;
// Check all of the attachments we've created against urls in the text
// and replace them out with attachment redirect urls before saving.
@@ -452,26 +468,30 @@ export default abstract class ImportTask extends BaseTask<Props> {
const document = await documentCreator({
sourceMetadata: {
fileName: path.basename(item.path),
mimeType: item.mimeType,
externalId: item.externalId,
createdByName: item.createdByName,
fileName: path.basename(processedDocument.path),
mimeType: processedDocument.mimeType,
externalId: processedDocument.externalId,
createdByName: processedDocument.createdByName,
},
id: item.id,
title: item.title,
urlId: item.urlId,
id: processedDocument.id,
title: processedDocument.title,
urlId: processedDocument.urlId,
text,
content: item.data,
collectionId: item.collectionId,
createdAt: item.createdAt,
updatedAt: item.updatedAt ?? item.createdAt,
publishedAt: item.updatedAt ?? item.createdAt ?? new Date(),
parentDocumentId: item.parentDocumentId,
content: processedDocument.data,
collectionId: processedDocument.collectionId,
createdAt: processedDocument.createdAt,
updatedAt:
processedDocument.updatedAt ?? processedDocument.createdAt,
publishedAt:
processedDocument.updatedAt ??
processedDocument.createdAt ??
new Date(),
parentDocumentId: processedDocument.parentDocumentId,
importId: fileOperation.id,
user,
ctx: createContext({ user, transaction }),
});
documents.set(item.id, document);
documents.set(processedDocument.id, document);
await collection.addDocumentToStructure(document, undefined, {
transaction,
@@ -535,6 +555,81 @@ export default abstract class ImportTask extends BaseTask<Props> {
};
}
/**
* Validates a document and applies the appropriate behavior based on validation settings.
*
* @param document The document to validate
* @param validationBehavior The behavior to apply when validation fails
* @returns The processed document or null if it should be skipped
*/
private validateAndProcessDocument(
document: StructuredImportData["documents"][number],
validationBehavior: ImportValidationBehavior
): StructuredImportData["documents"][number] | null {
const titleTooLong =
document.title.length > DocumentValidation.maxTitleLength;
const textTooLong =
document.text.length > DocumentValidation.maxRecommendedLength;
const stateTooLarge =
JSON.stringify(document.data || {}).length >
DocumentValidation.maxStateLength;
const hasValidationIssues = titleTooLong || textTooLong || stateTooLarge;
if (!hasValidationIssues) {
return document;
}
switch (validationBehavior) {
case ImportValidationBehavior.Skip: {
Logger.info(
"task",
`Skipping document "${document.title}" due to validation issues`
);
return null;
}
case ImportValidationBehavior.Truncate: {
const processedDocument = { ...document };
if (titleTooLong) {
processedDocument.title = truncate(document.title, {
length: DocumentValidation.maxTitleLength,
omission: "...",
});
}
if (textTooLong) {
processedDocument.text = truncate(document.text, {
length: DocumentValidation.maxRecommendedLength,
omission: "\n\n[Content truncated due to size limits]",
});
}
if (stateTooLarge && document.data) {
// For ProseMirror data, we need to be more careful about truncation
// For now, we'll just remove the data and rely on the text
processedDocument.data = undefined;
}
Logger.info(
"task",
`Truncated document "${document.title}" due to validation issues`
);
return processedDocument;
}
case ImportValidationBehavior.Abort:
default: {
throw new ValidationError(
`Document "${document.title}" exceeds validation limits. ` +
`Title: ${document.title.length}/${DocumentValidation.maxTitleLength}, ` +
`Text: ${document.text.length}/${DocumentValidation.maxRecommendedLength} characters`
);
}
}
}
/**
* Job options such as priority and retry strategy, as defined by Bull.
*/
+3 -1
View File
@@ -177,7 +177,8 @@ router.post(
transaction(),
async (ctx: APIContext<T.CollectionsImportReq>) => {
const { transaction } = ctx.state;
const { attachmentId, permission, format } = ctx.input.body;
const { attachmentId, permission, format, validationBehavior } =
ctx.input.body;
const { user } = ctx.state.auth;
authorize(user, "importCollection", user.team);
@@ -196,6 +197,7 @@ router.post(
teamId: user.teamId,
options: {
permission,
validationBehavior,
},
});
+4
View File
@@ -4,6 +4,7 @@ import {
CollectionPermission,
CollectionStatusFilter,
FileOperationFormat,
ImportValidationBehavior,
} from "@shared/types";
import { Collection } from "@server/models";
import { zodIconType, zodIdType } from "@server/utils/zod";
@@ -73,6 +74,9 @@ export const CollectionsImportSchema = BaseSchema.extend({
format: z
.nativeEnum(FileOperationFormat)
.default(FileOperationFormat.MarkdownZip),
validationBehavior: z
.nativeEnum(ImportValidationBehavior)
.default(ImportValidationBehavior.Truncate),
}),
});
+6
View File
@@ -552,3 +552,9 @@ export type ReactionSummary = {
emoji: string;
userIds: string[];
};
export enum ImportValidationBehavior {
Skip = "skip",
Truncate = "truncate",
Abort = "abort",
}