From cd63c4d219619105cd75acfddd0ce8bd2a742bb0 Mon Sep 17 00:00:00 2001 From: Tom Moor Date: Tue, 27 Oct 2026 19:19:10 -0400 Subject: [PATCH] fix: Stream remote uploads and stop createFromUrl from hanging Uploads from a URL are now streamed to storage instead of being buffered in memory. attachments.createFromUrl responds before the socket times out, keeps the job result long enough for Job.finished to read it, and runs at high priority. --- .../tasks/UploadAttachmentFromUrlTask.ts | 9 +- .../api/attachments/attachments.test.ts | 61 +++++++ server/routes/api/attachments/attachments.ts | 16 +- server/storage/files/BaseStorage.test.ts | 147 +++++++++++++++- server/storage/files/BaseStorage.ts | 163 ++++++++++++++++-- 5 files changed, 373 insertions(+), 23 deletions(-) diff --git a/server/queues/tasks/UploadAttachmentFromUrlTask.ts b/server/queues/tasks/UploadAttachmentFromUrlTask.ts index dcd4fe2ecc..bccc8db584 100644 --- a/server/queues/tasks/UploadAttachmentFromUrlTask.ts +++ b/server/queues/tasks/UploadAttachmentFromUrlTask.ts @@ -49,7 +49,14 @@ export default class UploadAttachmentFromUrlTask extends BaseTask { public get options() { return { attempts: 3, - priority: TaskPriority.Normal, + + // A request waits for this task, so it runs ahead of background work. + priority: TaskPriority.High, + + // Keep the result for a short time so that Job.finished can read it if + // the completed event is missed. + removeOnComplete: { age: 60 }, + removeOnFail: { age: 60 }, }; } } diff --git a/server/routes/api/attachments/attachments.test.ts b/server/routes/api/attachments/attachments.test.ts index ab479383e3..7e41d724e7 100644 --- a/server/routes/api/attachments/attachments.test.ts +++ b/server/routes/api/attachments/attachments.test.ts @@ -1,9 +1,11 @@ // @vitest-isolate true import { randomUUID } from "node:crypto"; +import type { Job } from "bull"; import { AttachmentPreset, CollectionPermission } from "@shared/types"; import env from "@server/env"; import { UserMembership } from "@server/models"; import Attachment from "@server/models/Attachment"; +import UploadAttachmentFromUrlTask from "@server/queues/tasks/UploadAttachmentFromUrlTask"; import { buildUser, buildAdmin, @@ -515,6 +517,65 @@ describe("#attachments.create", () => { }); }); +describe("#attachments.createFromUrl", () => { + const defaultRequestTimeout = env.REQUEST_TIMEOUT; + + afterEach(() => { + env.REQUEST_TIMEOUT = defaultRequestTimeout; + }); + + /** + * Replaces the upload task with a job that settles with the given promise. + * + * @param result The promise returned by Job.finished. + */ + const mockUploadJob = (result: Promise) => + vi + .spyOn(UploadAttachmentFromUrlTask.prototype, "schedule") + .mockResolvedValue({ finished: () => result } as Job); + + it("should return the attachment when the upload finishes", async () => { + mockUploadJob(Promise.resolve({})); + const user = await buildUser(); + const document = await buildDocument({ + teamId: user.teamId, + userId: user.id, + }); + + const res = await server.post("/api/attachments.createFromUrl", user, { + body: { + url: "https://files.example.com/file.pdf", + documentId: document.id, + preset: AttachmentPreset.DocumentAttachment, + }, + }); + + expect(res.status).toEqual(200); + const body = await res.json(); + expect(body.data.documentId).toEqual(document.id); + }); + + it("should time out before the request when the upload does not finish", async () => { + env.REQUEST_TIMEOUT = 1500; + mockUploadJob(new Promise(() => undefined)); + const user = await buildUser(); + const document = await buildDocument({ + teamId: user.teamId, + userId: user.id, + }); + + const res = await server.post("/api/attachments.createFromUrl", user, { + body: { + url: "https://files.example.com/file.pdf", + documentId: document.id, + preset: AttachmentPreset.DocumentAttachment, + }, + }); + + expect(res.status).toEqual(503); + }); +}); + describe("#attachments.delete", () => { it("should require authentication", async () => { const res = await server.post("/api/attachments.delete"); diff --git a/server/routes/api/attachments/attachments.ts b/server/routes/api/attachments/attachments.ts index 18f203f77e..05e0e3c811 100644 --- a/server/routes/api/attachments/attachments.ts +++ b/server/routes/api/attachments/attachments.ts @@ -3,12 +3,14 @@ import type { Transaction, WhereOptions } from "sequelize"; import { randomUUID } from "node:crypto"; import { AttachmentPreset } from "@shared/types"; import { bytesToHumanReadable, getFileNameFromUrl } from "@shared/utils/files"; +import { Second } from "@shared/utils/time"; import env from "@server/env"; import { AttachmentValidation } from "@shared/validations"; import { createContext } from "@server/context"; import { AuthorizationError, InvalidRequestError, + RequestTimeoutError, ValidationError, } from "@server/errors"; import auth from "@server/middlewares/authentication"; @@ -278,7 +280,19 @@ router.post( url, }); - const response = await job.finished(); + let timer: NodeJS.Timeout | undefined; + const timeout = new Promise((_resolve, reject) => { + timer = setTimeout( + () => reject(RequestTimeoutError("Timed out uploading file from URL")), + Math.max(env.REQUEST_TIMEOUT - Second.ms, Second.ms) + ); + }); + + // Respond before the socket times out, as the job may never finish. + const response = await Promise.race([job.finished(), timeout]).finally(() => + clearTimeout(timer) + ); + if ("error" in response) { throw InvalidRequestError(response.error); } diff --git a/server/storage/files/BaseStorage.test.ts b/server/storage/files/BaseStorage.test.ts index 47202fa9f5..68e508c78d 100644 --- a/server/storage/files/BaseStorage.test.ts +++ b/server/storage/files/BaseStorage.test.ts @@ -1,6 +1,10 @@ +import type { Readable } from "node:stream"; +import { buffer } from "node:stream/consumers"; +import { http, HttpResponse } from "msw"; import { Hour, Minute, Week } from "@shared/utils/time"; import BaseStorage from "./BaseStorage"; import env from "@server/env"; +import { server } from "@server/test/msw"; /** * Mock implementation of BaseStorage for testing purposes. @@ -39,14 +43,14 @@ class MockStorage extends BaseStorage { key, acl, }: { - body: Buffer; + body: Buffer | Readable; contentType?: string; key: string; acl?: string; }) { this.storedFiles.push({ key, - body: body as Buffer, + body: Buffer.isBuffer(body) ? body : await buffer(body), contentType, acl, }); @@ -63,7 +67,11 @@ class MockStorage extends BaseStorage { async moveFile() {} - async deleteFile() {} + public deletedKeys: string[] = []; + + async deleteFile(key: string) { + this.deletedKeys.push(key); + } static signingDateFor(expiresIn: number) { return MockStorage.getSigningDate(expiresIn); @@ -326,6 +334,139 @@ describe("BaseStorage", () => { }); }); + describe("remote URLs", () => { + const remoteUrl = "https://files.example.com/file.bin"; + + /** + * Responds to the remote URL with a body that is streamed in chunks and + * has no Content-Length header. + * + * @param chunks The chunks of the body. + * @param contentType The content type of the response. + */ + const respondWithChunks = (chunks: Buffer[], contentType: string) => + server.use( + http.get( + remoteUrl, + () => + new HttpResponse( + new ReadableStream({ + start(controller) { + chunks.forEach((chunk) => controller.enqueue(chunk)); + controller.close(); + }, + }), + { headers: { "Content-Type": contentType } } + ) + ) + ); + + it("should stream the file to storage", async () => { + respondWithChunks( + [Buffer.alloc(100, "a"), Buffer.alloc(150, "b")], + "application/pdf" + ); + + const result = await storage.storeFromUrl( + remoteUrl, + "test-key", + "private" + ); + + expect(result?.contentLength).toBe(250); + expect(result?.contentType).toBe("application/pdf"); + expect(storage.storedFiles).toHaveLength(1); + expect(storage.storedFiles[0].body.byteLength).toBe(250); + }); + + it("should not store a file that exceeds the size limit while streaming", async () => { + respondWithChunks( + [Buffer.alloc(400, "a"), Buffer.alloc(400, "b")], + "application/pdf" + ); + + const result = await storage.storeFromUrl( + remoteUrl, + "test-key", + "private" + ); + + expect(result).toBeUndefined(); + expect(storage.deletedKeys).toEqual(["test-key"]); + }); + + it("should not download a file with a Content-Length over the size limit", async () => { + server.use( + http.get( + remoteUrl, + () => + new HttpResponse(Buffer.alloc(800, "a"), { + headers: { "Content-Length": "800" }, + }) + ) + ); + + const result = await storage.storeFromUrl( + remoteUrl, + "test-key", + "private" + ); + + expect(result).toBeUndefined(); + expect(storage.storedFiles).toHaveLength(0); + }); + + it("should not store an empty file", async () => { + respondWithChunks([], "application/pdf"); + + const result = await storage.storeFromUrl( + remoteUrl, + "test-key", + "private" + ); + + expect(result).toBeUndefined(); + expect(storage.storedFiles).toHaveLength(0); + }); + + it("should detect an image from leading bytes split across chunks", async () => { + const png = Buffer.from([ + 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, + ]); + respondWithChunks( + [png.subarray(0, 3), png.subarray(3), Buffer.alloc(20)], + "text/html" + ); + + const result = await storage.storeFromUrl( + remoteUrl, + "test-key", + "public-read", + undefined, + { imagesOnly: true } + ); + + expect(result?.contentType).toBe("image/png"); + expect(result?.contentLength).toBe(30); + expect(storage.storedFiles[0].body.byteLength).toBe(30); + }); + + it("should not store a remote file that is not an image", async () => { + respondWithChunks([Buffer.from("")], "image/png"); + + const result = await storage.storeFromUrl( + remoteUrl, + "test-key", + "public-read", + undefined, + { imagesOnly: true } + ); + + expect(result).toBeUndefined(); + expect(storage.storedFiles).toHaveLength(0); + }); + }); + describe("internal URL handling", () => { it("should return undefined for URLs already on the storage provider", async () => { const result = await storage.storeFromUrl( diff --git a/server/storage/files/BaseStorage.ts b/server/storage/files/BaseStorage.ts index 6b0dc74b49..29c7144bfa 100644 --- a/server/storage/files/BaseStorage.ts +++ b/server/storage/files/BaseStorage.ts @@ -1,5 +1,5 @@ import type { Blob } from "node:buffer"; -import type { Readable } from "node:stream"; +import { pipeline, Readable, Transform } from "node:stream"; import { buffer } from "node:stream/consumers"; import type { PresignedPost } from "@aws-sdk/s3-presigned-post"; import contentDisposition from "content-disposition"; @@ -228,19 +228,19 @@ export default abstract class BaseStorage { return; } - let buffer, contentType; + const maxSize = Math.min( + options?.maxUploadSize ?? Infinity, + env.FILE_STORAGE_UPLOAD_MAX_SIZE + ); + let source: NodeJS.ReadableStream; + let contentType: string; const match = isBase64Url(url); if (match) { contentType = match[1]; - buffer = Buffer.from(match[2], "base64"); + const buffer = Buffer.from(match[2], "base64"); // Validate size for base64 URLs, same as for remote URLs - const maxSize = Math.min( - options?.maxUploadSize ?? Infinity, - env.FILE_STORAGE_UPLOAD_MAX_SIZE - ); - if (buffer.byteLength > maxSize) { Logger.warn("Base64 URL exceeds size limit", { size: buffer.byteLength, @@ -249,6 +249,7 @@ export default abstract class BaseStorage { }); return; } + source = Readable.from([buffer]); } else { try { const headers = new Headers(init?.headers); @@ -260,10 +261,6 @@ export default abstract class BaseStorage { const res = await fetch(url, { follow: 3, redirect: "follow", - size: Math.min( - options?.maxUploadSize ?? Infinity, - env.FILE_STORAGE_UPLOAD_MAX_SIZE - ), headers, timeout: 10000, ...initWithoutHeaders, @@ -273,8 +270,11 @@ export default abstract class BaseStorage { throw new Error(`Error fetching URL to upload: ${res.status}`); } - buffer = await res.buffer(); + if (Number(res.headers.get("content-length")) > maxSize) { + throw new UploadSizeExceededError(maxSize); + } + source = res.body; contentType = res.headers.get("content-type") ?? "application/octet-stream"; } catch (err) { @@ -288,18 +288,37 @@ export default abstract class BaseStorage { } } - const contentLength = buffer.byteLength; - if (contentLength === 0) { + // The file is streamed to storage so that it is never held in memory. + // Only the leading bytes are read first, to detect an empty file or the + // image type. + const limiter = new SizeLimitStream(maxSize); + pipeline(source, limiter, () => undefined); + + let head: { data: Buffer; ended: boolean }; + try { + head = await readHead(limiter, ImageSignatureLength); + } catch (err) { + Logger.warn("Error fetching URL to upload", { + error: errToString(err), + url, + key, + acl, + }); + return; + } + + if (head.data.byteLength === 0) { return; } if (options?.imagesOnly) { - const imageType = ImageHelper.detectType(buffer); + const imageType = ImageHelper.detectType(head.data); if (!imageType) { Logger.info("utils", "URL to upload is not a supported image", { url, key, }); + limiter.destroy(); return; } contentType = imageType.contentType; @@ -307,7 +326,7 @@ export default abstract class BaseStorage { try { const result = await this.store({ - body: buffer, + body: head.ended ? Readable.from([head.data]) : limiter, contentType, key, acl, @@ -316,11 +335,24 @@ export default abstract class BaseStorage { return result ? { url: result, - contentLength, + contentLength: limiter.bytes, contentType, } : undefined; } catch (err) { + const sourceFailed = limiter.errored !== null; + limiter.destroy(); + + // Remove any part of the file that was written before the failure. + if (sourceFailed) { + await this.deleteFile(key).catch(() => undefined); + } + + if (err instanceof UploadSizeExceededError) { + Logger.warn("URL to upload exceeds size limit", { maxSize, url, key }); + return; + } + Logger.error("Error uploading to file storage from URL", toError(err), { url, key, @@ -410,3 +442,98 @@ export default abstract class BaseStorage { return Math.min(BaseStorage.maxSigningWindow, Math.floor(expiresIn / 2)); } } + +/** The number of leading bytes that are needed to detect an image type. */ +const ImageSignatureLength = 12; + +class UploadSizeExceededError extends Error { + constructor(maxSize: number) { + super(`File exceeds the maximum upload size of ${maxSize} bytes`); + } +} + +/** + * A stream that passes data through unchanged, counts the bytes, and fails + * when the total exceeds the maximum size. + */ +class SizeLimitStream extends Transform { + /** The number of bytes that have passed through the stream. */ + public bytes = 0; + + constructor(private readonly maxSize: number) { + super(); + } + + _transform( + chunk: Buffer, + _encoding: BufferEncoding, + callback: (error?: Error | null, data?: Buffer) => void + ) { + this.bytes += chunk.byteLength; + if (this.bytes > this.maxSize) { + callback(new UploadSizeExceededError(this.maxSize)); + return; + } + callback(null, chunk); + } +} + +/** + * Reads the leading bytes of a stream. If the stream has more data, it is + * paused and the bytes are put back so that the full content can be read. + * + * @param stream The stream to read from. + * @param length The minimum number of bytes to read. + * @returns the bytes that were read, and whether the stream has ended. + * @throws if the stream fails or closes before the bytes are read. + */ +function readHead( + stream: Readable, + length: number +): Promise<{ data: Buffer; ended: boolean }> { + return new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + let size = 0; + + const cleanup = () => { + stream.off("data", handleData); + stream.off("end", handleEnd); + stream.off("error", handleError); + stream.off("close", handleClose); + }; + + const handleData = (chunk: Buffer) => { + chunks.push(chunk); + size += chunk.byteLength; + if (size >= length) { + stream.pause(); + cleanup(); + const data = Buffer.concat(chunks); + stream.unshift(data); + resolve({ data, ended: false }); + } + }; + + const handleEnd = () => { + cleanup(); + resolve({ data: Buffer.concat(chunks), ended: true }); + }; + + const handleError = (err: Error) => { + cleanup(); + reject(err); + }; + + // A stream that is destroyed without an error emits neither "end" nor + // "error", only "close". + const handleClose = () => { + cleanup(); + reject(new Error("Stream closed before it ended")); + }; + + stream.on("data", handleData); + stream.once("end", handleEnd); + stream.once("error", handleError); + stream.once("close", handleClose); + }); +}