import { Readable } from "node:stream"; import { pipeline } from "node:stream/promises"; import fs from "fs-extra"; import tmp from "tmp"; import { vi } from "vitest"; import { ZipFile } from "yazl"; import { buildZip } from "@server/test/support"; import ZipHelper from "./ZipHelper"; async function writeZip( entries: Record, postfix = ".zip" ): Promise { const zip = new ZipFile(); for (const [name, content] of Object.entries(entries)) { zip.addBuffer(Buffer.from(content), name); } const zipPath = tmp.fileSync({ postfix }).name; const writing = pipeline(zip.outputStream, fs.createWriteStream(zipPath)); zip.end(); await writing; return zipPath; } /** * Watch for the temporary file toTmpFile writes to, so that cleanup can be * asserted against that exact path rather than by scanning the shared temp * directory, which tests running in parallel also write to. */ function watchTmpFile() { const createWriteStream = vi.spyOn(fs, "createWriteStream"); return () => { expect(createWriteStream).toHaveBeenCalledTimes(1); return String(createWriteStream.mock.calls[0][0]); }; } async function readZip(filePath: string): Promise> { const contents: Record = {}; await ZipHelper.walk(filePath, async (entry) => { if (!entry.isDirectory) { contents[entry.fileName] = (await entry.readBuffer(1024)).toString( "utf8" ); } }); return contents; } describe("ZipHelper.toTmpFile", () => { afterEach(() => { vi.restoreAllMocks(); }); it("writes buffered and streamed entries to a temporary file", async () => { const filePath = await ZipHelper.toTmpFile(async (zip) => { zip.addBuffer(Buffer.from("hello"), "a.txt"); zip.addReadStreamLazy("b.txt", {}, (callback) => callback(null, Readable.from(["wo", "rld"])) ); await Promise.resolve(); }); expect(await readZip(filePath)).toEqual({ "a.txt": "hello", "b.txt": "world", }); await fs.remove(filePath); }); it("opens each lazy stream only when its entry is written", async () => { const opened: string[] = []; const filePath = await ZipHelper.toTmpFile(async (zip) => { for (const name of ["a.txt", "b.txt", "c.txt"]) { zip.addReadStreamLazy(name, {}, (callback) => { opened.push(name); callback(null, Readable.from([name])); }); } await Promise.resolve(); // Nothing is read up front — sources are opened one at a time as the // archive is pumped, so only one file is ever held open. expect(opened.length).toBeLessThan(3); }); expect(opened).toEqual(["a.txt", "b.txt", "c.txt"]); await fs.remove(filePath); }); it("propagates errors thrown while adding entries and cleans up", async () => { const tmpFile = watchTmpFile(); await expect( ZipHelper.toTmpFile(async (zip) => { zip.addBuffer(Buffer.from("partial"), "a.txt"); await Promise.resolve(); throw new Error("boom"); }) ).rejects.toThrow("boom"); expect(await fs.pathExists(tmpFile())).toBe(false); }); it("propagates errors raised when opening an entry and cleans up", async () => { const tmpFile = watchTmpFile(); await expect( ZipHelper.toTmpFile(async (zip) => { zip.addReadStreamLazy("a.txt", {}, (callback) => callback(new Error("stream unavailable"), Readable.from([])) ); await Promise.resolve(); }) ).rejects.toThrow("stream unavailable"); expect(await fs.pathExists(tmpFile())).toBe(false); }); it("fails rather than stalls when an entry's stream errors mid-read", async () => { const tmpFile = watchTmpFile(); await expect( ZipHelper.toTmpFile(async (zip) => { const source = new Readable({ read() { this.push("partial"); this.destroy(new Error("connection reset")); }, }); // Mirrors how ExportTask surfaces an interrupted read, which yazl // does not watch for on the streams it is given. source.on("error", (err) => zip.emit("error", err)); zip.addReadStreamLazy("a.txt", {}, (callback) => callback(null, source) ); await Promise.resolve(); }) ).rejects.toThrow("connection reset"); expect(await fs.pathExists(tmpFile())).toBe(false); }); }); describe("ZipHelper.toFileTree", () => { it("builds a nested tree with normalized pathInZip", async () => { const zipPath = await writeZip({ "Collection/sub/page.md": "# hi", "Collection/other.md": "other", }); const root = await ZipHelper.toFileTree(zipPath); expect(root.children).toHaveLength(1); const collection = root.children[0]; expect(collection.name).toBe("Collection"); expect(collection.pathInZip).toBe("Collection"); expect(collection.children.map((c) => c.name).sort()).toEqual([ "other.md", "sub", ]); const sub = collection.children.find((c) => c.name === "sub"); expect(sub?.children[0].pathInZip).toBe("Collection/sub/page.md"); }); it("normalizes `./`-prefixed entries instead of dropping them", async () => { const zipPath = await writeZip({ "./Collection/page.md": "body", }); const root = await ZipHelper.toFileTree(zipPath); expect(root.children).toHaveLength(1); expect(root.children[0].name).toBe("Collection"); expect(root.children[0].children[0].pathInZip).toBe("Collection/page.md"); }); it("filters macOS metadata and dotfiles at any depth", async () => { const zipPath = await writeZip({ "__MACOSX/Collection/page.md": "junk", "Collection/.DS_Store": "junk", "Collection/page.md": "body", }); const root = await ZipHelper.toFileTree(zipPath); expect(root.children).toHaveLength(1); expect(root.children[0].name).toBe("Collection"); expect(root.children[0].children.map((c) => c.name)).toEqual(["page.md"]); }); it("invokes onFile for each file entry with a readable handle", async () => { const zipPath = await writeZip({ "Collection/page.md": "hello world", "Collection/image.png": "binary", }); const seen: Record = {}; await ZipHelper.toFileTree(zipPath, async (node, entry) => { if (node.name.endsWith(".md")) { const buf = await entry.readBuffer(100); seen[node.pathInZip] = buf.toString("utf8"); } }); expect(seen).toEqual({ "Collection/page.md": "hello world" }); }); it("rejects reads larger than the provided max size", async () => { const zipPath = await writeZip({ "Collection/page.md": "hello world", }); await expect( ZipHelper.toFileTree(zipPath, async (_node, entry) => { await entry.readBuffer(10); }) ).rejects.toThrow("Collection/page.md is too large"); }); it("exposes entry sizes before the entry is read", async () => { const zipPath = await writeZip({ "Collection/page.md": "hello world", }); const sizes: Record = {}; await ZipHelper.toFileTree(zipPath, (node, entry) => { sizes[node.pathInZip] = entry.uncompressedSize; }); expect(sizes).toEqual({ "Collection/page.md": 11 }); }); }); describe("ZipHelper.walk", () => { it("walks an archive held in memory", async () => { const buffer = await buildZip({ "Collection/page.md": "hello world", "Collection/image.png": "binary", }); const contents: Record = {}; await ZipHelper.walk(buffer, async (entry) => { if (!entry.isDirectory) { contents[entry.fileName] = (await entry.readBuffer(1024)).toString( "utf8" ); } }); expect(contents).toEqual({ "Collection/page.md": "hello world", "Collection/image.png": "binary", }); }); it("rejects an oversized entry in an archive held in memory", async () => { const buffer = await buildZip({ "Collection/page.md": "hello world" }); await expect( ZipHelper.walk(buffer, async (entry) => { await entry.readBuffer(4); }) ).rejects.toThrow("Collection/page.md is too large"); }); it("rejects an archive with more entries than allowed", async () => { const buffer = await buildZip({ "a.md": "a", "b.md": "b", "c.md": "c" }); const onEntry = vi.fn(); await expect( ZipHelper.walk(buffer, onEntry, { maxEntries: 2 }) ).rejects.toMatchObject({ id: "import_error", message: "The archive contains too many files - the maximum is 2", }); expect(onEntry).not.toHaveBeenCalled(); }); it("rejects reads that exceed the total size across entries", async () => { const buffer = await buildZip({ "a.md": "x".repeat(600), "b.md": "x".repeat(600), }); await expect( ZipHelper.walk( buffer, async (entry) => { await entry.readBuffer(1024); }, { maxTotalSize: 1024 } ) ).rejects.toMatchObject({ id: "import_error", message: expect.stringContaining("too large when uncompressed"), }); }); it("does not count skipped entries toward the total size", async () => { const buffer = await buildZip({ "a.md": "x".repeat(600), "b.png": "x".repeat(600), }); await expect( ZipHelper.walk( buffer, async (entry) => { if (entry.fileName.endsWith(".md")) { await entry.readBuffer(1024); } }, { maxTotalSize: 1024 } ) ).resolves.toBeUndefined(); }); it("rejects a file that is not a zip archive as an import error", async () => { const filePath = tmp.fileSync({ postfix: ".zip" }).name; await fs.writeFile(filePath, "not a zip file"); await expect(ZipHelper.walk(filePath, () => {})).rejects.toMatchObject({ id: "import_error", message: "The file is not a valid zip archive", }); await fs.remove(filePath); }); it("rejects a buffer that is not a zip archive as an import error", async () => { await expect( ZipHelper.walk(Buffer.from("not a zip file"), () => {}) ).rejects.toMatchObject({ id: "import_error", message: "The file is not a valid zip archive", }); }); it("rejects an entry with corrupt compressed data as an import error", async () => { const buffer = await buildZip({ "page.md": "hello world ".repeat(100) }); // An invalid deflate block type follows the local header of the entry. buffer[30 + "page.md".length] = 0xff; await expect( ZipHelper.walk(buffer, async (entry) => { await entry.readBuffer(4096); }) ).rejects.toMatchObject({ id: "import_error", message: "The file is not a valid zip archive", }); }); it("keeps file system errors when the archive cannot be opened", async () => { await expect( ZipHelper.walk("/does/not/exist.zip", () => {}) ).rejects.toMatchObject({ code: "ENOENT" }); }); });