Files

358 lines
11 KiB
TypeScript

import { Readable } from "node:stream";
import { pipeline } from "node:stream/promises";
import fs from "fs-extra";
import tmp from "tmp";
import { vi } from "vitest";
import { ZipFile } from "yazl";
import { buildZip } from "@server/test/support";
import ZipHelper from "./ZipHelper";
async function writeZip(
entries: Record<string, string>,
postfix = ".zip"
): Promise<string> {
const zip = new ZipFile();
for (const [name, content] of Object.entries(entries)) {
zip.addBuffer(Buffer.from(content), name);
}
const zipPath = tmp.fileSync({ postfix }).name;
const writing = pipeline(zip.outputStream, fs.createWriteStream(zipPath));
zip.end();
await writing;
return zipPath;
}
/**
* Watch for the temporary file toTmpFile writes to, so that cleanup can be
* asserted against that exact path rather than by scanning the shared temp
* directory, which tests running in parallel also write to.
*/
function watchTmpFile() {
const createWriteStream = vi.spyOn(fs, "createWriteStream");
return () => {
expect(createWriteStream).toHaveBeenCalledTimes(1);
return String(createWriteStream.mock.calls[0][0]);
};
}
async function readZip(filePath: string): Promise<Record<string, string>> {
const contents: Record<string, string> = {};
await ZipHelper.walk(filePath, async (entry) => {
if (!entry.isDirectory) {
contents[entry.fileName] = (await entry.readBuffer(1024)).toString(
"utf8"
);
}
});
return contents;
}
describe("ZipHelper.toTmpFile", () => {
afterEach(() => {
vi.restoreAllMocks();
});
it("writes buffered and streamed entries to a temporary file", async () => {
const filePath = await ZipHelper.toTmpFile(async (zip) => {
zip.addBuffer(Buffer.from("hello"), "a.txt");
zip.addReadStreamLazy("b.txt", {}, (callback) =>
callback(null, Readable.from(["wo", "rld"]))
);
await Promise.resolve();
});
expect(await readZip(filePath)).toEqual({
"a.txt": "hello",
"b.txt": "world",
});
await fs.remove(filePath);
});
it("opens each lazy stream only when its entry is written", async () => {
const opened: string[] = [];
const filePath = await ZipHelper.toTmpFile(async (zip) => {
for (const name of ["a.txt", "b.txt", "c.txt"]) {
zip.addReadStreamLazy(name, {}, (callback) => {
opened.push(name);
callback(null, Readable.from([name]));
});
}
await Promise.resolve();
// Nothing is read up front — sources are opened one at a time as the
// archive is pumped, so only one file is ever held open.
expect(opened.length).toBeLessThan(3);
});
expect(opened).toEqual(["a.txt", "b.txt", "c.txt"]);
await fs.remove(filePath);
});
it("propagates errors thrown while adding entries and cleans up", async () => {
const tmpFile = watchTmpFile();
await expect(
ZipHelper.toTmpFile(async (zip) => {
zip.addBuffer(Buffer.from("partial"), "a.txt");
await Promise.resolve();
throw new Error("boom");
})
).rejects.toThrow("boom");
expect(await fs.pathExists(tmpFile())).toBe(false);
});
it("propagates errors raised when opening an entry and cleans up", async () => {
const tmpFile = watchTmpFile();
await expect(
ZipHelper.toTmpFile(async (zip) => {
zip.addReadStreamLazy("a.txt", {}, (callback) =>
callback(new Error("stream unavailable"), Readable.from([]))
);
await Promise.resolve();
})
).rejects.toThrow("stream unavailable");
expect(await fs.pathExists(tmpFile())).toBe(false);
});
it("fails rather than stalls when an entry's stream errors mid-read", async () => {
const tmpFile = watchTmpFile();
await expect(
ZipHelper.toTmpFile(async (zip) => {
const source = new Readable({
read() {
this.push("partial");
this.destroy(new Error("connection reset"));
},
});
// Mirrors how ExportTask surfaces an interrupted read, which yazl
// does not watch for on the streams it is given.
source.on("error", (err) => zip.emit("error", err));
zip.addReadStreamLazy("a.txt", {}, (callback) =>
callback(null, source)
);
await Promise.resolve();
})
).rejects.toThrow("connection reset");
expect(await fs.pathExists(tmpFile())).toBe(false);
});
});
describe("ZipHelper.toFileTree", () => {
it("builds a nested tree with normalized pathInZip", async () => {
const zipPath = await writeZip({
"Collection/sub/page.md": "# hi",
"Collection/other.md": "other",
});
const root = await ZipHelper.toFileTree(zipPath);
expect(root.children).toHaveLength(1);
const collection = root.children[0];
expect(collection.name).toBe("Collection");
expect(collection.pathInZip).toBe("Collection");
expect(collection.children.map((c) => c.name).sort()).toEqual([
"other.md",
"sub",
]);
const sub = collection.children.find((c) => c.name === "sub");
expect(sub?.children[0].pathInZip).toBe("Collection/sub/page.md");
});
it("normalizes `./`-prefixed entries instead of dropping them", async () => {
const zipPath = await writeZip({
"./Collection/page.md": "body",
});
const root = await ZipHelper.toFileTree(zipPath);
expect(root.children).toHaveLength(1);
expect(root.children[0].name).toBe("Collection");
expect(root.children[0].children[0].pathInZip).toBe("Collection/page.md");
});
it("filters macOS metadata and dotfiles at any depth", async () => {
const zipPath = await writeZip({
"__MACOSX/Collection/page.md": "junk",
"Collection/.DS_Store": "junk",
"Collection/page.md": "body",
});
const root = await ZipHelper.toFileTree(zipPath);
expect(root.children).toHaveLength(1);
expect(root.children[0].name).toBe("Collection");
expect(root.children[0].children.map((c) => c.name)).toEqual(["page.md"]);
});
it("invokes onFile for each file entry with a readable handle", async () => {
const zipPath = await writeZip({
"Collection/page.md": "hello world",
"Collection/image.png": "binary",
});
const seen: Record<string, string> = {};
await ZipHelper.toFileTree(zipPath, async (node, entry) => {
if (node.name.endsWith(".md")) {
const buf = await entry.readBuffer(100);
seen[node.pathInZip] = buf.toString("utf8");
}
});
expect(seen).toEqual({ "Collection/page.md": "hello world" });
});
it("rejects reads larger than the provided max size", async () => {
const zipPath = await writeZip({
"Collection/page.md": "hello world",
});
await expect(
ZipHelper.toFileTree(zipPath, async (_node, entry) => {
await entry.readBuffer(10);
})
).rejects.toThrow("Collection/page.md is too large");
});
it("exposes entry sizes before the entry is read", async () => {
const zipPath = await writeZip({
"Collection/page.md": "hello world",
});
const sizes: Record<string, number> = {};
await ZipHelper.toFileTree(zipPath, (node, entry) => {
sizes[node.pathInZip] = entry.uncompressedSize;
});
expect(sizes).toEqual({ "Collection/page.md": 11 });
});
});
describe("ZipHelper.walk", () => {
it("walks an archive held in memory", async () => {
const buffer = await buildZip({
"Collection/page.md": "hello world",
"Collection/image.png": "binary",
});
const contents: Record<string, string> = {};
await ZipHelper.walk(buffer, async (entry) => {
if (!entry.isDirectory) {
contents[entry.fileName] = (await entry.readBuffer(1024)).toString(
"utf8"
);
}
});
expect(contents).toEqual({
"Collection/page.md": "hello world",
"Collection/image.png": "binary",
});
});
it("rejects an oversized entry in an archive held in memory", async () => {
const buffer = await buildZip({ "Collection/page.md": "hello world" });
await expect(
ZipHelper.walk(buffer, async (entry) => {
await entry.readBuffer(4);
})
).rejects.toThrow("Collection/page.md is too large");
});
it("rejects an archive with more entries than allowed", async () => {
const buffer = await buildZip({ "a.md": "a", "b.md": "b", "c.md": "c" });
const onEntry = vi.fn();
await expect(
ZipHelper.walk(buffer, onEntry, { maxEntries: 2 })
).rejects.toMatchObject({
id: "import_error",
message: "The archive contains too many files - the maximum is 2",
});
expect(onEntry).not.toHaveBeenCalled();
});
it("rejects reads that exceed the total size across entries", async () => {
const buffer = await buildZip({
"a.md": "x".repeat(600),
"b.md": "x".repeat(600),
});
await expect(
ZipHelper.walk(
buffer,
async (entry) => {
await entry.readBuffer(1024);
},
{ maxTotalSize: 1024 }
)
).rejects.toMatchObject({
id: "import_error",
message: expect.stringContaining("too large when uncompressed"),
});
});
it("does not count skipped entries toward the total size", async () => {
const buffer = await buildZip({
"a.md": "x".repeat(600),
"b.png": "x".repeat(600),
});
await expect(
ZipHelper.walk(
buffer,
async (entry) => {
if (entry.fileName.endsWith(".md")) {
await entry.readBuffer(1024);
}
},
{ maxTotalSize: 1024 }
)
).resolves.toBeUndefined();
});
it("rejects a file that is not a zip archive as an import error", async () => {
const filePath = tmp.fileSync({ postfix: ".zip" }).name;
await fs.writeFile(filePath, "not a zip file");
await expect(ZipHelper.walk(filePath, () => {})).rejects.toMatchObject({
id: "import_error",
message: "The file is not a valid zip archive",
});
await fs.remove(filePath);
});
it("rejects a buffer that is not a zip archive as an import error", async () => {
await expect(
ZipHelper.walk(Buffer.from("not a zip file"), () => {})
).rejects.toMatchObject({
id: "import_error",
message: "The file is not a valid zip archive",
});
});
it("rejects an entry with corrupt compressed data as an import error", async () => {
const buffer = await buildZip({ "page.md": "hello world ".repeat(100) });
// An invalid deflate block type follows the local header of the entry.
buffer[30 + "page.md".length] = 0xff;
await expect(
ZipHelper.walk(buffer, async (entry) => {
await entry.readBuffer(4096);
})
).rejects.toMatchObject({
id: "import_error",
message: "The file is not a valid zip archive",
});
});
it("keeps file system errors when the archive cannot be opened", async () => {
await expect(
ZipHelper.walk("/does/not/exist.zip", () => {})
).rejects.toMatchObject({ code: "ENOENT" });
});
});