mirror of
https://github.com/outline/outline.git
synced 2026-11-20 21:23:06 +03:00
416 lines
13 KiB
TypeScript
416 lines
13 KiB
TypeScript
import path from "node:path";
|
|
import { pipeline } from "node:stream/promises";
|
|
import fs from "fs-extra";
|
|
import httpErrors from "http-errors";
|
|
import tmp from "tmp";
|
|
import type { Entry } from "yauzl";
|
|
import yauzl, { validateFileName } from "yauzl";
|
|
import { ZipFile } from "yazl";
|
|
import { bytesToHumanReadable } from "@shared/utils/files";
|
|
import { FileImportError, ValidationError } from "@server/errors";
|
|
import Logger from "@server/logging/Logger";
|
|
import { trace } from "@server/logging/tracing";
|
|
import { deserializeFilename } from "./fs";
|
|
|
|
export interface ZipEntryHandle {
|
|
/** UTF-8 filename as recorded in the zip; directory entries end with `/`. */
|
|
fileName: string;
|
|
/** Size of the uncompressed entry in bytes. */
|
|
uncompressedSize: number;
|
|
/** True when this entry is a directory marker rather than a file. */
|
|
isDirectory: boolean;
|
|
/**
|
|
* Read the entry's contents into memory. Safe to skip — entries the caller
|
|
* does not read are simply advanced past.
|
|
*
|
|
* @param maxSize Maximum uncompressed size to read into memory, in bytes.
|
|
*/
|
|
readBuffer(maxSize: number): Promise<Buffer>;
|
|
}
|
|
|
|
export interface ZipWalkOptions {
|
|
/** Maximum number of entries the archive may contain. */
|
|
maxEntries?: number;
|
|
/** Maximum total bytes that may be read into memory across all entries. */
|
|
maxTotalSize?: number;
|
|
}
|
|
|
|
export interface ZipTreeNode {
|
|
/** The file name (last path segment) including extension. */
|
|
name: string;
|
|
/** Title derived from the file name (extension stripped, deserialized). */
|
|
title: string;
|
|
/** Path within the zip (no leading slash, segments joined with `/`). */
|
|
pathInZip: string;
|
|
/** Nested children — populated for directory entries. */
|
|
children: ZipTreeNode[];
|
|
}
|
|
|
|
/**
|
|
* Helper for creating and reading zip files.
|
|
*/
|
|
@trace()
|
|
export default class ZipHelper {
|
|
/**
|
|
* Write a zip file to a temporary disk location.
|
|
*
|
|
* Entries are added by the `addEntries` callback, which receives an archive
|
|
* that is already draining to disk. Adding entries only after a reader is
|
|
* attached keeps memory proportional to a single entry rather than to the
|
|
* size of the whole archive.
|
|
*
|
|
* @param addEntries Callback that populates the archive.
|
|
* @returns pathname of the temporary file where the zip was written to disk.
|
|
* @throws if the archive could not be built or written to disk.
|
|
*/
|
|
public static async toTmpFile(
|
|
addEntries: (zip: ZipFile) => Promise<void>
|
|
): Promise<string> {
|
|
Logger.debug("utils", "Creating tmp file…");
|
|
const filePath = await createTmpFile({
|
|
prefix: "export-",
|
|
postfix: ".zip",
|
|
});
|
|
|
|
const zip = new ZipFile();
|
|
const writeStream = fs.createWriteStream(filePath);
|
|
// yazl reports failures on the archive rather than on its output stream,
|
|
// so route them into the pipeline to get a single failure path.
|
|
zip.on("error", (error: Error) => writeStream.destroy(error));
|
|
|
|
const writing = pipeline(zip.outputStream, writeStream);
|
|
// Resolve rather than reject, so that a write failure while entries are
|
|
// still being added is never an unhandled rejection.
|
|
const written = writing.then(
|
|
() => undefined,
|
|
(error: Error) => error
|
|
);
|
|
|
|
try {
|
|
await addEntries(zip);
|
|
zip.end();
|
|
} catch (error) {
|
|
// Unblock the pipeline, which is still waiting on entries that will now
|
|
// never arrive.
|
|
writeStream.destroy();
|
|
await written;
|
|
await removeTmpFile(filePath);
|
|
throw error;
|
|
}
|
|
|
|
const writeError = await written;
|
|
if (writeError) {
|
|
await removeTmpFile(filePath);
|
|
throw writeError;
|
|
}
|
|
|
|
Logger.debug("utils", "Writing zip complete", { path: filePath });
|
|
return filePath;
|
|
}
|
|
|
|
/**
|
|
* Iterate through entries in a zip file without extracting it to disk.
|
|
* Entries are visited serially in archive order. `onEntry` may be async; the
|
|
* next entry is only read once the previous handler resolves.
|
|
*
|
|
* @param source The file path where the zip is located, or a Buffer holding
|
|
* the zip contents already in memory.
|
|
* @param onEntry Handler invoked for each entry. Skip an entry by returning
|
|
* without calling `entry.readBuffer(maxSize)`.
|
|
* @param options Limits on the entry count and total bytes read.
|
|
* @returns Promise that resolves once the archive has been fully walked.
|
|
* @throws {FileImportError} if the source is not a readable zip archive, or
|
|
* exceeds the given limits.
|
|
*/
|
|
public static walk(
|
|
source: string | Buffer,
|
|
onEntry: (entry: ZipEntryHandle) => Promise<void> | void,
|
|
options: ZipWalkOptions = {}
|
|
): Promise<void> {
|
|
const { maxEntries = Infinity, maxTotalSize = Infinity } = options;
|
|
|
|
return new Promise((resolve, reject) => {
|
|
const openOptions = {
|
|
lazyEntries: true,
|
|
autoClose: true,
|
|
decodeStrings: false,
|
|
};
|
|
|
|
const onOpen = function (
|
|
err: Error | null,
|
|
zipfile: yauzl.ZipFile
|
|
): void {
|
|
if (err) {
|
|
return reject(ZipHelper.toArchiveError(err));
|
|
}
|
|
|
|
// The count comes from the central directory, so it is known before
|
|
// any entry is read.
|
|
if (zipfile.entryCount > maxEntries) {
|
|
zipfile.close();
|
|
return reject(
|
|
FileImportError(
|
|
`The archive contains too many files - the maximum is ${maxEntries.toLocaleString()}`
|
|
)
|
|
);
|
|
}
|
|
|
|
let settled = false;
|
|
let totalRead = 0;
|
|
const fail = (error: Error) => {
|
|
if (settled) {
|
|
return;
|
|
}
|
|
settled = true;
|
|
zipfile.close();
|
|
reject(error);
|
|
};
|
|
|
|
zipfile.on("entry", (entry: Entry) => {
|
|
const fileName = Buffer.from(entry.fileName).toString("utf8");
|
|
|
|
if (validateFileName(fileName)) {
|
|
Logger.warn("Invalid zip entry", { fileName });
|
|
zipfile.readEntry();
|
|
return;
|
|
}
|
|
|
|
const handle: ZipEntryHandle = {
|
|
fileName,
|
|
uncompressedSize: entry.uncompressedSize,
|
|
isDirectory: fileName.endsWith("/"),
|
|
readBuffer: (maxSize) =>
|
|
new Promise<Buffer>((res, rej) => {
|
|
const remaining = maxTotalSize - totalRead;
|
|
const limit = Math.min(maxSize, remaining);
|
|
const tooLargeError = () =>
|
|
maxSize <= remaining
|
|
? ZipHelper.entryTooLargeError(fileName, maxSize)
|
|
: ZipHelper.archiveTooLargeError(maxTotalSize);
|
|
|
|
if (entry.uncompressedSize > limit) {
|
|
return rej(tooLargeError());
|
|
}
|
|
|
|
zipfile.openReadStream(entry, (rErr, readStream) => {
|
|
if (rErr) {
|
|
return rej(ZipHelper.toArchiveError(rErr));
|
|
}
|
|
const chunks: Buffer[] = [];
|
|
let bytesRead = 0;
|
|
let settled = false;
|
|
readStream.on("data", (chunk: Buffer) => {
|
|
bytesRead += chunk.length;
|
|
if (bytesRead > limit) {
|
|
readStream.destroy(tooLargeError());
|
|
return;
|
|
}
|
|
chunks.push(chunk);
|
|
});
|
|
readStream.on("end", () => {
|
|
if (!settled) {
|
|
settled = true;
|
|
totalRead += bytesRead;
|
|
res(Buffer.concat(chunks));
|
|
}
|
|
});
|
|
readStream.on("error", (err) => {
|
|
if (!settled) {
|
|
settled = true;
|
|
rej(ZipHelper.toArchiveError(err));
|
|
}
|
|
});
|
|
readStream.on("close", () => {
|
|
if (!settled) {
|
|
settled = true;
|
|
rej(
|
|
new Error(
|
|
`Stream closed before completing read of ${fileName}`
|
|
)
|
|
);
|
|
}
|
|
});
|
|
});
|
|
}),
|
|
};
|
|
|
|
Promise.resolve()
|
|
.then(() => onEntry(handle))
|
|
.then(() => {
|
|
if (!settled) {
|
|
zipfile.readEntry();
|
|
}
|
|
})
|
|
.catch(fail);
|
|
});
|
|
|
|
const done = () => {
|
|
if (!settled) {
|
|
settled = true;
|
|
resolve();
|
|
}
|
|
};
|
|
|
|
// A file-backed archive resolves on "close", so that its descriptor is
|
|
// released before the caller gets control back and can delete the file.
|
|
// A buffer-backed one has no descriptor and yauzl's BufferSlicer never
|
|
// emits "close", so "end" — every entry read — is the only signal.
|
|
if (typeof source === "string") {
|
|
zipfile.on("close", done);
|
|
} else {
|
|
zipfile.on("end", done);
|
|
}
|
|
|
|
zipfile.on("error", (error) => fail(ZipHelper.toArchiveError(error)));
|
|
zipfile.readEntry();
|
|
};
|
|
|
|
if (typeof source === "string") {
|
|
yauzl.open(source, openOptions, onOpen);
|
|
} else {
|
|
yauzl.fromBuffer(source, openOptions, onOpen);
|
|
}
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Walk a zip file once and build a tree of its entries without extracting
|
|
* to disk. macOS metadata directories (`__MACOSX`) and dotfiles are
|
|
* filtered out at any path segment.
|
|
*
|
|
* The optional `onFile` callback fires once per file entry as it is
|
|
* encountered, with both the materialized tree node and a handle to the
|
|
* raw entry. Callers that need to pre-load contents (e.g. small text
|
|
* files) can call `entry.readBuffer(maxSize)`; callers that only need the tree
|
|
* structure can omit the callback entirely.
|
|
*
|
|
* @param filePath Local filesystem path to the zip.
|
|
* @param onFile Optional per-file hook; not called for directory entries.
|
|
* @param options Limits on the entry count and total bytes read.
|
|
* @returns A synthetic root node whose `children` are the zip's top-level
|
|
* entries.
|
|
* @throws {FileImportError} if the archive is unreadable or exceeds the
|
|
* given limits.
|
|
*/
|
|
public static async toFileTree(
|
|
filePath: string,
|
|
onFile?: (node: ZipTreeNode, entry: ZipEntryHandle) => Promise<void> | void,
|
|
options?: ZipWalkOptions
|
|
): Promise<ZipTreeNode> {
|
|
const root: ZipTreeNode = {
|
|
name: "",
|
|
title: "",
|
|
pathInZip: "",
|
|
children: [],
|
|
};
|
|
|
|
const isFiltered = (segment: string) =>
|
|
segment === "__MACOSX" || segment.startsWith(".");
|
|
|
|
const nodesByPath = new Map<string, ZipTreeNode>();
|
|
|
|
const resolveNode = (entryName: string): ZipTreeNode | null => {
|
|
// Drop empty segments and the path-no-op `.` (e.g. entries written as
|
|
// `./Collection/page.md`). `..` is preserved so the dotfile filter
|
|
// below rejects it — we never resolve path traversal in zip entries.
|
|
const segments = entryName
|
|
.split("/")
|
|
.filter((s) => s !== "" && s !== ".");
|
|
if (segments.length === 0) {
|
|
return null;
|
|
}
|
|
|
|
let current = root;
|
|
let pathSoFar = "";
|
|
for (let i = 0; i < segments.length; i++) {
|
|
const segment = segments[i];
|
|
if (isFiltered(segment)) {
|
|
return null;
|
|
}
|
|
|
|
pathSoFar = pathSoFar ? `${pathSoFar}/${segment}` : segment;
|
|
let next = nodesByPath.get(pathSoFar);
|
|
if (!next) {
|
|
next = {
|
|
name: segment,
|
|
title: deserializeFilename(path.parse(segment).name),
|
|
pathInZip: pathSoFar,
|
|
children: [],
|
|
};
|
|
current.children.push(next);
|
|
nodesByPath.set(pathSoFar, next);
|
|
}
|
|
current = next;
|
|
}
|
|
|
|
return current;
|
|
};
|
|
|
|
await this.walk(
|
|
filePath,
|
|
async (entry) => {
|
|
const node = resolveNode(entry.fileName);
|
|
if (!node || entry.isDirectory) {
|
|
return;
|
|
}
|
|
if (onFile) {
|
|
await onFile(node, entry);
|
|
}
|
|
},
|
|
options
|
|
);
|
|
|
|
return root;
|
|
}
|
|
|
|
private static toArchiveError(error: Error): Error {
|
|
// System errors carry a syscall, parser and zlib errors do not.
|
|
if ("syscall" in error || httpErrors.isHttpError(error)) {
|
|
return error;
|
|
}
|
|
return FileImportError("The file is not a valid zip archive");
|
|
}
|
|
|
|
private static archiveTooLargeError(maxTotalSize: number): Error {
|
|
return FileImportError(
|
|
`The archive is too large when uncompressed - the maximum is ${bytesToHumanReadable(
|
|
maxTotalSize
|
|
)}`
|
|
);
|
|
}
|
|
|
|
private static entryTooLargeError(fileName: string, maxSize: number): Error {
|
|
return ValidationError(
|
|
`${fileName} is too large - the maximum size is ${bytesToHumanReadable(
|
|
maxSize
|
|
)}`
|
|
);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Promisified wrapper around tmp.file.
|
|
*
|
|
* The descriptor tmp opens is discarded, as the file is written through a
|
|
* separate stream — retaining it would leak a descriptor per call.
|
|
*
|
|
* @param options options passed through to tmp.
|
|
* @returns the path of the created temporary file.
|
|
*/
|
|
const createTmpFile = (options: tmp.FileOptions) =>
|
|
new Promise<string>((resolve, reject) => {
|
|
tmp.file({ ...options, discardDescriptor: true }, (err, filePath) =>
|
|
err ? reject(err) : resolve(filePath)
|
|
);
|
|
});
|
|
|
|
/**
|
|
* Delete a temporary file, logging rather than throwing on failure.
|
|
*
|
|
* @param filePath the path of the file to remove.
|
|
*/
|
|
const removeTmpFile = (filePath: string) =>
|
|
fs.remove(filePath).catch((error) => {
|
|
Logger.error("Failed to remove tmp file", error);
|
|
});
|