diff --git a/server/queues/tasks/UploadAttachmentFromUrlTask.ts b/server/queues/tasks/UploadAttachmentFromUrlTask.ts index dcd4fe2ecc..bccc8db584 100644 --- a/server/queues/tasks/UploadAttachmentFromUrlTask.ts +++ b/server/queues/tasks/UploadAttachmentFromUrlTask.ts @@ -49,7 +49,14 @@ export default class UploadAttachmentFromUrlTask extends BaseTask { public get options() { return { attempts: 3, - priority: TaskPriority.Normal, + + // A request waits for this task, so it runs ahead of background work. + priority: TaskPriority.High, + + // Keep the result for a short time so that Job.finished can read it if + // the completed event is missed. + removeOnComplete: { age: 60 }, + removeOnFail: { age: 60 }, }; } } diff --git a/server/routes/api/attachments/attachments.test.ts b/server/routes/api/attachments/attachments.test.ts index ab479383e3..7e41d724e7 100644 --- a/server/routes/api/attachments/attachments.test.ts +++ b/server/routes/api/attachments/attachments.test.ts @@ -1,9 +1,11 @@ // @vitest-isolate true import { randomUUID } from "node:crypto"; +import type { Job } from "bull"; import { AttachmentPreset, CollectionPermission } from "@shared/types"; import env from "@server/env"; import { UserMembership } from "@server/models"; import Attachment from "@server/models/Attachment"; +import UploadAttachmentFromUrlTask from "@server/queues/tasks/UploadAttachmentFromUrlTask"; import { buildUser, buildAdmin, @@ -515,6 +517,65 @@ describe("#attachments.create", () => { }); }); +describe("#attachments.createFromUrl", () => { + const defaultRequestTimeout = env.REQUEST_TIMEOUT; + + afterEach(() => { + env.REQUEST_TIMEOUT = defaultRequestTimeout; + }); + + /** + * Replaces the upload task with a job that settles with the given promise. + * + * @param result The promise returned by Job.finished. + */ + const mockUploadJob = (result: Promise) => + vi + .spyOn(UploadAttachmentFromUrlTask.prototype, "schedule") + .mockResolvedValue({ finished: () => result } as Job); + + it("should return the attachment when the upload finishes", async () => { + mockUploadJob(Promise.resolve({})); + const user = await buildUser(); + const document = await buildDocument({ + teamId: user.teamId, + userId: user.id, + }); + + const res = await server.post("/api/attachments.createFromUrl", user, { + body: { + url: "https://files.example.com/file.pdf", + documentId: document.id, + preset: AttachmentPreset.DocumentAttachment, + }, + }); + + expect(res.status).toEqual(200); + const body = await res.json(); + expect(body.data.documentId).toEqual(document.id); + }); + + it("should time out before the request when the upload does not finish", async () => { + env.REQUEST_TIMEOUT = 1500; + mockUploadJob(new Promise(() => undefined)); + const user = await buildUser(); + const document = await buildDocument({ + teamId: user.teamId, + userId: user.id, + }); + + const res = await server.post("/api/attachments.createFromUrl", user, { + body: { + url: "https://files.example.com/file.pdf", + documentId: document.id, + preset: AttachmentPreset.DocumentAttachment, + }, + }); + + expect(res.status).toEqual(503); + }); +}); + describe("#attachments.delete", () => { it("should require authentication", async () => { const res = await server.post("/api/attachments.delete"); diff --git a/server/routes/api/attachments/attachments.ts b/server/routes/api/attachments/attachments.ts index 18f203f77e..05e0e3c811 100644 --- a/server/routes/api/attachments/attachments.ts +++ b/server/routes/api/attachments/attachments.ts @@ -3,12 +3,14 @@ import type { Transaction, WhereOptions } from "sequelize"; import { randomUUID } from "node:crypto"; import { AttachmentPreset } from "@shared/types"; import { bytesToHumanReadable, getFileNameFromUrl } from "@shared/utils/files"; +import { Second } from "@shared/utils/time"; import env from "@server/env"; import { AttachmentValidation } from "@shared/validations"; import { createContext } from "@server/context"; import { AuthorizationError, InvalidRequestError, + RequestTimeoutError, ValidationError, } from "@server/errors"; import auth from "@server/middlewares/authentication"; @@ -278,7 +280,19 @@ router.post( url, }); - const response = await job.finished(); + let timer: NodeJS.Timeout | undefined; + const timeout = new Promise((_resolve, reject) => { + timer = setTimeout( + () => reject(RequestTimeoutError("Timed out uploading file from URL")), + Math.max(env.REQUEST_TIMEOUT - Second.ms, Second.ms) + ); + }); + + // Respond before the socket times out, as the job may never finish. + const response = await Promise.race([job.finished(), timeout]).finally(() => + clearTimeout(timer) + ); + if ("error" in response) { throw InvalidRequestError(response.error); } diff --git a/server/storage/files/BaseStorage.test.ts b/server/storage/files/BaseStorage.test.ts index 47202fa9f5..68e508c78d 100644 --- a/server/storage/files/BaseStorage.test.ts +++ b/server/storage/files/BaseStorage.test.ts @@ -1,6 +1,10 @@ +import type { Readable } from "node:stream"; +import { buffer } from "node:stream/consumers"; +import { http, HttpResponse } from "msw"; import { Hour, Minute, Week } from "@shared/utils/time"; import BaseStorage from "./BaseStorage"; import env from "@server/env"; +import { server } from "@server/test/msw"; /** * Mock implementation of BaseStorage for testing purposes. @@ -39,14 +43,14 @@ class MockStorage extends BaseStorage { key, acl, }: { - body: Buffer; + body: Buffer | Readable; contentType?: string; key: string; acl?: string; }) { this.storedFiles.push({ key, - body: body as Buffer, + body: Buffer.isBuffer(body) ? body : await buffer(body), contentType, acl, }); @@ -63,7 +67,11 @@ class MockStorage extends BaseStorage { async moveFile() {} - async deleteFile() {} + public deletedKeys: string[] = []; + + async deleteFile(key: string) { + this.deletedKeys.push(key); + } static signingDateFor(expiresIn: number) { return MockStorage.getSigningDate(expiresIn); @@ -326,6 +334,139 @@ describe("BaseStorage", () => { }); }); + describe("remote URLs", () => { + const remoteUrl = "https://files.example.com/file.bin"; + + /** + * Responds to the remote URL with a body that is streamed in chunks and + * has no Content-Length header. + * + * @param chunks The chunks of the body. + * @param contentType The content type of the response. + */ + const respondWithChunks = (chunks: Buffer[], contentType: string) => + server.use( + http.get( + remoteUrl, + () => + new HttpResponse( + new ReadableStream({ + start(controller) { + chunks.forEach((chunk) => controller.enqueue(chunk)); + controller.close(); + }, + }), + { headers: { "Content-Type": contentType } } + ) + ) + ); + + it("should stream the file to storage", async () => { + respondWithChunks( + [Buffer.alloc(100, "a"), Buffer.alloc(150, "b")], + "application/pdf" + ); + + const result = await storage.storeFromUrl( + remoteUrl, + "test-key", + "private" + ); + + expect(result?.contentLength).toBe(250); + expect(result?.contentType).toBe("application/pdf"); + expect(storage.storedFiles).toHaveLength(1); + expect(storage.storedFiles[0].body.byteLength).toBe(250); + }); + + it("should not store a file that exceeds the size limit while streaming", async () => { + respondWithChunks( + [Buffer.alloc(400, "a"), Buffer.alloc(400, "b")], + "application/pdf" + ); + + const result = await storage.storeFromUrl( + remoteUrl, + "test-key", + "private" + ); + + expect(result).toBeUndefined(); + expect(storage.deletedKeys).toEqual(["test-key"]); + }); + + it("should not download a file with a Content-Length over the size limit", async () => { + server.use( + http.get( + remoteUrl, + () => + new HttpResponse(Buffer.alloc(800, "a"), { + headers: { "Content-Length": "800" }, + }) + ) + ); + + const result = await storage.storeFromUrl( + remoteUrl, + "test-key", + "private" + ); + + expect(result).toBeUndefined(); + expect(storage.storedFiles).toHaveLength(0); + }); + + it("should not store an empty file", async () => { + respondWithChunks([], "application/pdf"); + + const result = await storage.storeFromUrl( + remoteUrl, + "test-key", + "private" + ); + + expect(result).toBeUndefined(); + expect(storage.storedFiles).toHaveLength(0); + }); + + it("should detect an image from leading bytes split across chunks", async () => { + const png = Buffer.from([ + 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, + ]); + respondWithChunks( + [png.subarray(0, 3), png.subarray(3), Buffer.alloc(20)], + "text/html" + ); + + const result = await storage.storeFromUrl( + remoteUrl, + "test-key", + "public-read", + undefined, + { imagesOnly: true } + ); + + expect(result?.contentType).toBe("image/png"); + expect(result?.contentLength).toBe(30); + expect(storage.storedFiles[0].body.byteLength).toBe(30); + }); + + it("should not store a remote file that is not an image", async () => { + respondWithChunks([Buffer.from("")], "image/png"); + + const result = await storage.storeFromUrl( + remoteUrl, + "test-key", + "public-read", + undefined, + { imagesOnly: true } + ); + + expect(result).toBeUndefined(); + expect(storage.storedFiles).toHaveLength(0); + }); + }); + describe("internal URL handling", () => { it("should return undefined for URLs already on the storage provider", async () => { const result = await storage.storeFromUrl( diff --git a/server/storage/files/BaseStorage.ts b/server/storage/files/BaseStorage.ts index 6b0dc74b49..29c7144bfa 100644 --- a/server/storage/files/BaseStorage.ts +++ b/server/storage/files/BaseStorage.ts @@ -1,5 +1,5 @@ import type { Blob } from "node:buffer"; -import type { Readable } from "node:stream"; +import { pipeline, Readable, Transform } from "node:stream"; import { buffer } from "node:stream/consumers"; import type { PresignedPost } from "@aws-sdk/s3-presigned-post"; import contentDisposition from "content-disposition"; @@ -228,19 +228,19 @@ export default abstract class BaseStorage { return; } - let buffer, contentType; + const maxSize = Math.min( + options?.maxUploadSize ?? Infinity, + env.FILE_STORAGE_UPLOAD_MAX_SIZE + ); + let source: NodeJS.ReadableStream; + let contentType: string; const match = isBase64Url(url); if (match) { contentType = match[1]; - buffer = Buffer.from(match[2], "base64"); + const buffer = Buffer.from(match[2], "base64"); // Validate size for base64 URLs, same as for remote URLs - const maxSize = Math.min( - options?.maxUploadSize ?? Infinity, - env.FILE_STORAGE_UPLOAD_MAX_SIZE - ); - if (buffer.byteLength > maxSize) { Logger.warn("Base64 URL exceeds size limit", { size: buffer.byteLength, @@ -249,6 +249,7 @@ export default abstract class BaseStorage { }); return; } + source = Readable.from([buffer]); } else { try { const headers = new Headers(init?.headers); @@ -260,10 +261,6 @@ export default abstract class BaseStorage { const res = await fetch(url, { follow: 3, redirect: "follow", - size: Math.min( - options?.maxUploadSize ?? Infinity, - env.FILE_STORAGE_UPLOAD_MAX_SIZE - ), headers, timeout: 10000, ...initWithoutHeaders, @@ -273,8 +270,11 @@ export default abstract class BaseStorage { throw new Error(`Error fetching URL to upload: ${res.status}`); } - buffer = await res.buffer(); + if (Number(res.headers.get("content-length")) > maxSize) { + throw new UploadSizeExceededError(maxSize); + } + source = res.body; contentType = res.headers.get("content-type") ?? "application/octet-stream"; } catch (err) { @@ -288,18 +288,37 @@ export default abstract class BaseStorage { } } - const contentLength = buffer.byteLength; - if (contentLength === 0) { + // The file is streamed to storage so that it is never held in memory. + // Only the leading bytes are read first, to detect an empty file or the + // image type. + const limiter = new SizeLimitStream(maxSize); + pipeline(source, limiter, () => undefined); + + let head: { data: Buffer; ended: boolean }; + try { + head = await readHead(limiter, ImageSignatureLength); + } catch (err) { + Logger.warn("Error fetching URL to upload", { + error: errToString(err), + url, + key, + acl, + }); + return; + } + + if (head.data.byteLength === 0) { return; } if (options?.imagesOnly) { - const imageType = ImageHelper.detectType(buffer); + const imageType = ImageHelper.detectType(head.data); if (!imageType) { Logger.info("utils", "URL to upload is not a supported image", { url, key, }); + limiter.destroy(); return; } contentType = imageType.contentType; @@ -307,7 +326,7 @@ export default abstract class BaseStorage { try { const result = await this.store({ - body: buffer, + body: head.ended ? Readable.from([head.data]) : limiter, contentType, key, acl, @@ -316,11 +335,24 @@ export default abstract class BaseStorage { return result ? { url: result, - contentLength, + contentLength: limiter.bytes, contentType, } : undefined; } catch (err) { + const sourceFailed = limiter.errored !== null; + limiter.destroy(); + + // Remove any part of the file that was written before the failure. + if (sourceFailed) { + await this.deleteFile(key).catch(() => undefined); + } + + if (err instanceof UploadSizeExceededError) { + Logger.warn("URL to upload exceeds size limit", { maxSize, url, key }); + return; + } + Logger.error("Error uploading to file storage from URL", toError(err), { url, key, @@ -410,3 +442,98 @@ export default abstract class BaseStorage { return Math.min(BaseStorage.maxSigningWindow, Math.floor(expiresIn / 2)); } } + +/** The number of leading bytes that are needed to detect an image type. */ +const ImageSignatureLength = 12; + +class UploadSizeExceededError extends Error { + constructor(maxSize: number) { + super(`File exceeds the maximum upload size of ${maxSize} bytes`); + } +} + +/** + * A stream that passes data through unchanged, counts the bytes, and fails + * when the total exceeds the maximum size. + */ +class SizeLimitStream extends Transform { + /** The number of bytes that have passed through the stream. */ + public bytes = 0; + + constructor(private readonly maxSize: number) { + super(); + } + + _transform( + chunk: Buffer, + _encoding: BufferEncoding, + callback: (error?: Error | null, data?: Buffer) => void + ) { + this.bytes += chunk.byteLength; + if (this.bytes > this.maxSize) { + callback(new UploadSizeExceededError(this.maxSize)); + return; + } + callback(null, chunk); + } +} + +/** + * Reads the leading bytes of a stream. If the stream has more data, it is + * paused and the bytes are put back so that the full content can be read. + * + * @param stream The stream to read from. + * @param length The minimum number of bytes to read. + * @returns the bytes that were read, and whether the stream has ended. + * @throws if the stream fails or closes before the bytes are read. + */ +function readHead( + stream: Readable, + length: number +): Promise<{ data: Buffer; ended: boolean }> { + return new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + let size = 0; + + const cleanup = () => { + stream.off("data", handleData); + stream.off("end", handleEnd); + stream.off("error", handleError); + stream.off("close", handleClose); + }; + + const handleData = (chunk: Buffer) => { + chunks.push(chunk); + size += chunk.byteLength; + if (size >= length) { + stream.pause(); + cleanup(); + const data = Buffer.concat(chunks); + stream.unshift(data); + resolve({ data, ended: false }); + } + }; + + const handleEnd = () => { + cleanup(); + resolve({ data: Buffer.concat(chunks), ended: true }); + }; + + const handleError = (err: Error) => { + cleanup(); + reject(err); + }; + + // A stream that is destroyed without an error emits neither "end" nor + // "error", only "close". + const handleClose = () => { + cleanup(); + reject(new Error("Stream closed before it ended")); + }; + + stream.on("data", handleData); + stream.once("end", handleEnd); + stream.once("error", handleError); + stream.once("close", handleClose); + }); +}