Files
outline/server/utils/ZipHelper.ts
T
Tom Moor 7b751ab04b perf: Attach output stream before adding zip entries (#13167)
* perf: Attach output stream before adding zip entries

* fix: Handle failure
2026-07-27 19:13:18 -04:00

341 lines
11 KiB
TypeScript

import path from "node:path";
import { pipeline } from "node:stream/promises";
import fs from "fs-extra";
import tmp from "tmp";
import type { Entry } from "yauzl";
import yauzl, { validateFileName } from "yauzl";
import { ZipFile } from "yazl";
import { bytesToHumanReadable } from "@shared/utils/files";
import { ValidationError } from "@server/errors";
import Logger from "@server/logging/Logger";
import { trace } from "@server/logging/tracing";
import { deserializeFilename } from "./fs";
export interface ZipEntryHandle {
/** UTF-8 filename as recorded in the zip; directory entries end with `/`. */
fileName: string;
/** Size of the uncompressed entry in bytes. */
uncompressedSize: number;
/** True when this entry is a directory marker rather than a file. */
isDirectory: boolean;
/**
* Read the entry's contents into memory. Safe to skip — entries the caller
* does not read are simply advanced past.
*
* @param maxSize Maximum uncompressed size to read into memory, in bytes.
*/
readBuffer(maxSize: number): Promise<Buffer>;
}
export interface ZipTreeNode {
/** The file name (last path segment) including extension. */
name: string;
/** Title derived from the file name (extension stripped, deserialized). */
title: string;
/** Path within the zip (no leading slash, segments joined with `/`). */
pathInZip: string;
/** Nested children — populated for directory entries. */
children: ZipTreeNode[];
}
/**
* Helper for creating and reading zip files.
*/
@trace()
export default class ZipHelper {
/**
* Write a zip file to a temporary disk location.
*
* Entries are added by the `addEntries` callback, which receives an archive
* that is already draining to disk. Adding entries only after a reader is
* attached keeps memory proportional to a single entry rather than to the
* size of the whole archive.
*
* @param addEntries Callback that populates the archive.
* @returns pathname of the temporary file where the zip was written to disk.
* @throws if the archive could not be built or written to disk.
*/
public static async toTmpFile(
addEntries: (zip: ZipFile) => Promise<void>
): Promise<string> {
Logger.debug("utils", "Creating tmp file…");
const filePath = await createTmpFile({
prefix: "export-",
postfix: ".zip",
});
const zip = new ZipFile();
const writeStream = fs.createWriteStream(filePath);
// yazl reports failures on the archive rather than on its output stream,
// so route them into the pipeline to get a single failure path.
zip.on("error", (error: Error) => writeStream.destroy(error));
const writing = pipeline(zip.outputStream, writeStream);
// Resolve rather than reject, so that a write failure while entries are
// still being added is never an unhandled rejection.
const written = writing.then(
() => undefined,
(error: Error) => error
);
try {
await addEntries(zip);
zip.end();
} catch (error) {
// Unblock the pipeline, which is still waiting on entries that will now
// never arrive.
writeStream.destroy();
await written;
await removeTmpFile(filePath);
throw error;
}
const writeError = await written;
if (writeError) {
await removeTmpFile(filePath);
throw writeError;
}
Logger.debug("utils", "Writing zip complete", { path: filePath });
return filePath;
}
/**
* Iterate through entries in a zip file without extracting it to disk.
* Entries are visited serially in archive order. `onEntry` may be async; the
* next entry is only read once the previous handler resolves.
*
* @param filePath The file path where the zip is located.
* @param onEntry Handler invoked for each entry. Skip an entry by returning
* without calling `entry.readBuffer(maxSize)`.
* @returns Promise that resolves once the archive has been fully walked.
*/
public static walk(
filePath: string,
onEntry: (entry: ZipEntryHandle) => Promise<void> | void
): Promise<void> {
return new Promise((resolve, reject) => {
yauzl.open(
filePath,
{
lazyEntries: true,
autoClose: true,
decodeStrings: false,
},
function (err, zipfile) {
if (err) {
return reject(err);
}
let settled = false;
const fail = (error: Error) => {
if (settled) {
return;
}
settled = true;
zipfile.close();
reject(error);
};
zipfile.on("entry", (entry: Entry) => {
const fileName = Buffer.from(entry.fileName).toString("utf8");
if (validateFileName(fileName)) {
Logger.warn("Invalid zip entry", { fileName });
zipfile.readEntry();
return;
}
const handle: ZipEntryHandle = {
fileName,
uncompressedSize: entry.uncompressedSize,
isDirectory: fileName.endsWith("/"),
readBuffer: (maxSize) =>
new Promise<Buffer>((res, rej) => {
if (entry.uncompressedSize > maxSize) {
return rej(ZipHelper.entryTooLargeError(fileName, maxSize));
}
zipfile.openReadStream(entry, (rErr, readStream) => {
if (rErr) {
return rej(rErr);
}
const chunks: Buffer[] = [];
let bytesRead = 0;
let settled = false;
readStream.on("data", (chunk: Buffer) => {
bytesRead += chunk.length;
if (bytesRead > maxSize) {
readStream.destroy(
ZipHelper.entryTooLargeError(fileName, maxSize)
);
return;
}
chunks.push(chunk);
});
readStream.on("end", () => {
if (!settled) {
settled = true;
res(Buffer.concat(chunks));
}
});
readStream.on("error", (err) => {
if (!settled) {
settled = true;
rej(err);
}
});
readStream.on("close", () => {
if (!settled) {
settled = true;
rej(
new Error(
`Stream closed before completing read of ${fileName}`
)
);
}
});
});
}),
};
Promise.resolve()
.then(() => onEntry(handle))
.then(() => {
if (!settled) {
zipfile.readEntry();
}
})
.catch(fail);
});
zipfile.on("close", () => {
if (!settled) {
settled = true;
resolve();
}
});
zipfile.on("error", (error) => fail(error));
zipfile.readEntry();
}
);
});
}
/**
* Walk a zip file once and build a tree of its entries without extracting
* to disk. macOS metadata directories (`__MACOSX`) and dotfiles are
* filtered out at any path segment.
*
* The optional `onFile` callback fires once per file entry as it is
* encountered, with both the materialized tree node and a handle to the
* raw entry. Callers that need to pre-load contents (e.g. small text
* files) can call `entry.readBuffer(maxSize)`; callers that only need the tree
* structure can omit the callback entirely.
*
* @param filePath Local filesystem path to the zip.
* @param onFile Optional per-file hook; not called for directory entries.
* @returns A synthetic root node whose `children` are the zip's top-level
* entries.
*/
public static async toFileTree(
filePath: string,
onFile?: (node: ZipTreeNode, entry: ZipEntryHandle) => Promise<void> | void
): Promise<ZipTreeNode> {
const root: ZipTreeNode = {
name: "",
title: "",
pathInZip: "",
children: [],
};
const isFiltered = (segment: string) =>
segment === "__MACOSX" || segment.startsWith(".");
const nodesByPath = new Map<string, ZipTreeNode>();
const resolveNode = (entryName: string): ZipTreeNode | null => {
// Drop empty segments and the path-no-op `.` (e.g. entries written as
// `./Collection/page.md`). `..` is preserved so the dotfile filter
// below rejects it — we never resolve path traversal in zip entries.
const segments = entryName
.split("/")
.filter((s) => s !== "" && s !== ".");
if (segments.length === 0) {
return null;
}
let current = root;
let pathSoFar = "";
for (let i = 0; i < segments.length; i++) {
const segment = segments[i];
if (isFiltered(segment)) {
return null;
}
pathSoFar = pathSoFar ? `${pathSoFar}/${segment}` : segment;
let next = nodesByPath.get(pathSoFar);
if (!next) {
next = {
name: segment,
title: deserializeFilename(path.parse(segment).name),
pathInZip: pathSoFar,
children: [],
};
current.children.push(next);
nodesByPath.set(pathSoFar, next);
}
current = next;
}
return current;
};
await this.walk(filePath, async (entry) => {
const node = resolveNode(entry.fileName);
if (!node || entry.isDirectory) {
return;
}
if (onFile) {
await onFile(node, entry);
}
});
return root;
}
private static entryTooLargeError(fileName: string, maxSize: number): Error {
return ValidationError(
`${fileName} is too large - the maximum size is ${bytesToHumanReadable(
maxSize
)}`
);
}
}
/**
* Promisified wrapper around tmp.file.
*
* The descriptor tmp opens is discarded, as the file is written through a
* separate stream — retaining it would leak a descriptor per call.
*
* @param options options passed through to tmp.
* @returns the path of the created temporary file.
*/
const createTmpFile = (options: tmp.FileOptions) =>
new Promise<string>((resolve, reject) => {
tmp.file({ ...options, discardDescriptor: true }, (err, filePath) =>
err ? reject(err) : resolve(filePath)
);
});
/**
* Delete a temporary file, logging rather than throwing on failure.
*
* @param filePath the path of the file to remove.
*/
const removeTmpFile = (filePath: string) =>
fs.remove(filePath).catch((error) => {
Logger.error("Failed to remove tmp file", error);
});