Files

416 lines
13 KiB
TypeScript

import path from "node:path";
import { pipeline } from "node:stream/promises";
import fs from "fs-extra";
import httpErrors from "http-errors";
import tmp from "tmp";
import type { Entry } from "yauzl";
import yauzl, { validateFileName } from "yauzl";
import { ZipFile } from "yazl";
import { bytesToHumanReadable } from "@shared/utils/files";
import { FileImportError, ValidationError } from "@server/errors";
import Logger from "@server/logging/Logger";
import { trace } from "@server/logging/tracing";
import { deserializeFilename } from "./fs";
export interface ZipEntryHandle {
/** UTF-8 filename as recorded in the zip; directory entries end with `/`. */
fileName: string;
/** Size of the uncompressed entry in bytes. */
uncompressedSize: number;
/** True when this entry is a directory marker rather than a file. */
isDirectory: boolean;
/**
* Read the entry's contents into memory. Safe to skip — entries the caller
* does not read are simply advanced past.
*
* @param maxSize Maximum uncompressed size to read into memory, in bytes.
*/
readBuffer(maxSize: number): Promise<Buffer>;
}
export interface ZipWalkOptions {
/** Maximum number of entries the archive may contain. */
maxEntries?: number;
/** Maximum total bytes that may be read into memory across all entries. */
maxTotalSize?: number;
}
export interface ZipTreeNode {
/** The file name (last path segment) including extension. */
name: string;
/** Title derived from the file name (extension stripped, deserialized). */
title: string;
/** Path within the zip (no leading slash, segments joined with `/`). */
pathInZip: string;
/** Nested children — populated for directory entries. */
children: ZipTreeNode[];
}
/**
* Helper for creating and reading zip files.
*/
@trace()
export default class ZipHelper {
/**
* Write a zip file to a temporary disk location.
*
* Entries are added by the `addEntries` callback, which receives an archive
* that is already draining to disk. Adding entries only after a reader is
* attached keeps memory proportional to a single entry rather than to the
* size of the whole archive.
*
* @param addEntries Callback that populates the archive.
* @returns pathname of the temporary file where the zip was written to disk.
* @throws if the archive could not be built or written to disk.
*/
public static async toTmpFile(
addEntries: (zip: ZipFile) => Promise<void>
): Promise<string> {
Logger.debug("utils", "Creating tmp file…");
const filePath = await createTmpFile({
prefix: "export-",
postfix: ".zip",
});
const zip = new ZipFile();
const writeStream = fs.createWriteStream(filePath);
// yazl reports failures on the archive rather than on its output stream,
// so route them into the pipeline to get a single failure path.
zip.on("error", (error: Error) => writeStream.destroy(error));
const writing = pipeline(zip.outputStream, writeStream);
// Resolve rather than reject, so that a write failure while entries are
// still being added is never an unhandled rejection.
const written = writing.then(
() => undefined,
(error: Error) => error
);
try {
await addEntries(zip);
zip.end();
} catch (error) {
// Unblock the pipeline, which is still waiting on entries that will now
// never arrive.
writeStream.destroy();
await written;
await removeTmpFile(filePath);
throw error;
}
const writeError = await written;
if (writeError) {
await removeTmpFile(filePath);
throw writeError;
}
Logger.debug("utils", "Writing zip complete", { path: filePath });
return filePath;
}
/**
* Iterate through entries in a zip file without extracting it to disk.
* Entries are visited serially in archive order. `onEntry` may be async; the
* next entry is only read once the previous handler resolves.
*
* @param source The file path where the zip is located, or a Buffer holding
* the zip contents already in memory.
* @param onEntry Handler invoked for each entry. Skip an entry by returning
* without calling `entry.readBuffer(maxSize)`.
* @param options Limits on the entry count and total bytes read.
* @returns Promise that resolves once the archive has been fully walked.
* @throws {FileImportError} if the source is not a readable zip archive, or
* exceeds the given limits.
*/
public static walk(
source: string | Buffer,
onEntry: (entry: ZipEntryHandle) => Promise<void> | void,
options: ZipWalkOptions = {}
): Promise<void> {
const { maxEntries = Infinity, maxTotalSize = Infinity } = options;
return new Promise((resolve, reject) => {
const openOptions = {
lazyEntries: true,
autoClose: true,
decodeStrings: false,
};
const onOpen = function (
err: Error | null,
zipfile: yauzl.ZipFile
): void {
if (err) {
return reject(ZipHelper.toArchiveError(err));
}
// The count comes from the central directory, so it is known before
// any entry is read.
if (zipfile.entryCount > maxEntries) {
zipfile.close();
return reject(
FileImportError(
`The archive contains too many files - the maximum is ${maxEntries.toLocaleString()}`
)
);
}
let settled = false;
let totalRead = 0;
const fail = (error: Error) => {
if (settled) {
return;
}
settled = true;
zipfile.close();
reject(error);
};
zipfile.on("entry", (entry: Entry) => {
const fileName = Buffer.from(entry.fileName).toString("utf8");
if (validateFileName(fileName)) {
Logger.warn("Invalid zip entry", { fileName });
zipfile.readEntry();
return;
}
const handle: ZipEntryHandle = {
fileName,
uncompressedSize: entry.uncompressedSize,
isDirectory: fileName.endsWith("/"),
readBuffer: (maxSize) =>
new Promise<Buffer>((res, rej) => {
const remaining = maxTotalSize - totalRead;
const limit = Math.min(maxSize, remaining);
const tooLargeError = () =>
maxSize <= remaining
? ZipHelper.entryTooLargeError(fileName, maxSize)
: ZipHelper.archiveTooLargeError(maxTotalSize);
if (entry.uncompressedSize > limit) {
return rej(tooLargeError());
}
zipfile.openReadStream(entry, (rErr, readStream) => {
if (rErr) {
return rej(ZipHelper.toArchiveError(rErr));
}
const chunks: Buffer[] = [];
let bytesRead = 0;
let settled = false;
readStream.on("data", (chunk: Buffer) => {
bytesRead += chunk.length;
if (bytesRead > limit) {
readStream.destroy(tooLargeError());
return;
}
chunks.push(chunk);
});
readStream.on("end", () => {
if (!settled) {
settled = true;
totalRead += bytesRead;
res(Buffer.concat(chunks));
}
});
readStream.on("error", (err) => {
if (!settled) {
settled = true;
rej(ZipHelper.toArchiveError(err));
}
});
readStream.on("close", () => {
if (!settled) {
settled = true;
rej(
new Error(
`Stream closed before completing read of ${fileName}`
)
);
}
});
});
}),
};
Promise.resolve()
.then(() => onEntry(handle))
.then(() => {
if (!settled) {
zipfile.readEntry();
}
})
.catch(fail);
});
const done = () => {
if (!settled) {
settled = true;
resolve();
}
};
// A file-backed archive resolves on "close", so that its descriptor is
// released before the caller gets control back and can delete the file.
// A buffer-backed one has no descriptor and yauzl's BufferSlicer never
// emits "close", so "end" — every entry read — is the only signal.
if (typeof source === "string") {
zipfile.on("close", done);
} else {
zipfile.on("end", done);
}
zipfile.on("error", (error) => fail(ZipHelper.toArchiveError(error)));
zipfile.readEntry();
};
if (typeof source === "string") {
yauzl.open(source, openOptions, onOpen);
} else {
yauzl.fromBuffer(source, openOptions, onOpen);
}
});
}
/**
* Walk a zip file once and build a tree of its entries without extracting
* to disk. macOS metadata directories (`__MACOSX`) and dotfiles are
* filtered out at any path segment.
*
* The optional `onFile` callback fires once per file entry as it is
* encountered, with both the materialized tree node and a handle to the
* raw entry. Callers that need to pre-load contents (e.g. small text
* files) can call `entry.readBuffer(maxSize)`; callers that only need the tree
* structure can omit the callback entirely.
*
* @param filePath Local filesystem path to the zip.
* @param onFile Optional per-file hook; not called for directory entries.
* @param options Limits on the entry count and total bytes read.
* @returns A synthetic root node whose `children` are the zip's top-level
* entries.
* @throws {FileImportError} if the archive is unreadable or exceeds the
* given limits.
*/
public static async toFileTree(
filePath: string,
onFile?: (node: ZipTreeNode, entry: ZipEntryHandle) => Promise<void> | void,
options?: ZipWalkOptions
): Promise<ZipTreeNode> {
const root: ZipTreeNode = {
name: "",
title: "",
pathInZip: "",
children: [],
};
const isFiltered = (segment: string) =>
segment === "__MACOSX" || segment.startsWith(".");
const nodesByPath = new Map<string, ZipTreeNode>();
const resolveNode = (entryName: string): ZipTreeNode | null => {
// Drop empty segments and the path-no-op `.` (e.g. entries written as
// `./Collection/page.md`). `..` is preserved so the dotfile filter
// below rejects it — we never resolve path traversal in zip entries.
const segments = entryName
.split("/")
.filter((s) => s !== "" && s !== ".");
if (segments.length === 0) {
return null;
}
let current = root;
let pathSoFar = "";
for (let i = 0; i < segments.length; i++) {
const segment = segments[i];
if (isFiltered(segment)) {
return null;
}
pathSoFar = pathSoFar ? `${pathSoFar}/${segment}` : segment;
let next = nodesByPath.get(pathSoFar);
if (!next) {
next = {
name: segment,
title: deserializeFilename(path.parse(segment).name),
pathInZip: pathSoFar,
children: [],
};
current.children.push(next);
nodesByPath.set(pathSoFar, next);
}
current = next;
}
return current;
};
await this.walk(
filePath,
async (entry) => {
const node = resolveNode(entry.fileName);
if (!node || entry.isDirectory) {
return;
}
if (onFile) {
await onFile(node, entry);
}
},
options
);
return root;
}
private static toArchiveError(error: Error): Error {
// System errors carry a syscall, parser and zlib errors do not.
if ("syscall" in error || httpErrors.isHttpError(error)) {
return error;
}
return FileImportError("The file is not a valid zip archive");
}
private static archiveTooLargeError(maxTotalSize: number): Error {
return FileImportError(
`The archive is too large when uncompressed - the maximum is ${bytesToHumanReadable(
maxTotalSize
)}`
);
}
private static entryTooLargeError(fileName: string, maxSize: number): Error {
return ValidationError(
`${fileName} is too large - the maximum size is ${bytesToHumanReadable(
maxSize
)}`
);
}
}
/**
* Promisified wrapper around tmp.file.
*
* The descriptor tmp opens is discarded, as the file is written through a
* separate stream — retaining it would leak a descriptor per call.
*
* @param options options passed through to tmp.
* @returns the path of the created temporary file.
*/
const createTmpFile = (options: tmp.FileOptions) =>
new Promise<string>((resolve, reject) => {
tmp.file({ ...options, discardDescriptor: true }, (err, filePath) =>
err ? reject(err) : resolve(filePath)
);
});
/**
* Delete a temporary file, logging rather than throwing on failure.
*
* @param filePath the path of the file to remove.
*/
const removeTmpFile = (filePath: string) =>
fs.remove(filePath).catch((error) => {
Logger.error("Failed to remove tmp file", error);
});