mirror of
https://github.com/outline/outline.git
synced 2026-11-20 21:23:06 +03:00
358 lines
11 KiB
TypeScript
358 lines
11 KiB
TypeScript
import { Readable } from "node:stream";
|
|
import { pipeline } from "node:stream/promises";
|
|
import fs from "fs-extra";
|
|
import tmp from "tmp";
|
|
import { vi } from "vitest";
|
|
import { ZipFile } from "yazl";
|
|
import { buildZip } from "@server/test/support";
|
|
import ZipHelper from "./ZipHelper";
|
|
|
|
async function writeZip(
|
|
entries: Record<string, string>,
|
|
postfix = ".zip"
|
|
): Promise<string> {
|
|
const zip = new ZipFile();
|
|
for (const [name, content] of Object.entries(entries)) {
|
|
zip.addBuffer(Buffer.from(content), name);
|
|
}
|
|
const zipPath = tmp.fileSync({ postfix }).name;
|
|
const writing = pipeline(zip.outputStream, fs.createWriteStream(zipPath));
|
|
zip.end();
|
|
await writing;
|
|
return zipPath;
|
|
}
|
|
|
|
/**
|
|
* Watch for the temporary file toTmpFile writes to, so that cleanup can be
|
|
* asserted against that exact path rather than by scanning the shared temp
|
|
* directory, which tests running in parallel also write to.
|
|
*/
|
|
function watchTmpFile() {
|
|
const createWriteStream = vi.spyOn(fs, "createWriteStream");
|
|
return () => {
|
|
expect(createWriteStream).toHaveBeenCalledTimes(1);
|
|
return String(createWriteStream.mock.calls[0][0]);
|
|
};
|
|
}
|
|
|
|
async function readZip(filePath: string): Promise<Record<string, string>> {
|
|
const contents: Record<string, string> = {};
|
|
await ZipHelper.walk(filePath, async (entry) => {
|
|
if (!entry.isDirectory) {
|
|
contents[entry.fileName] = (await entry.readBuffer(1024)).toString(
|
|
"utf8"
|
|
);
|
|
}
|
|
});
|
|
return contents;
|
|
}
|
|
|
|
describe("ZipHelper.toTmpFile", () => {
|
|
afterEach(() => {
|
|
vi.restoreAllMocks();
|
|
});
|
|
|
|
it("writes buffered and streamed entries to a temporary file", async () => {
|
|
const filePath = await ZipHelper.toTmpFile(async (zip) => {
|
|
zip.addBuffer(Buffer.from("hello"), "a.txt");
|
|
zip.addReadStreamLazy("b.txt", {}, (callback) =>
|
|
callback(null, Readable.from(["wo", "rld"]))
|
|
);
|
|
await Promise.resolve();
|
|
});
|
|
|
|
expect(await readZip(filePath)).toEqual({
|
|
"a.txt": "hello",
|
|
"b.txt": "world",
|
|
});
|
|
await fs.remove(filePath);
|
|
});
|
|
|
|
it("opens each lazy stream only when its entry is written", async () => {
|
|
const opened: string[] = [];
|
|
const filePath = await ZipHelper.toTmpFile(async (zip) => {
|
|
for (const name of ["a.txt", "b.txt", "c.txt"]) {
|
|
zip.addReadStreamLazy(name, {}, (callback) => {
|
|
opened.push(name);
|
|
callback(null, Readable.from([name]));
|
|
});
|
|
}
|
|
await Promise.resolve();
|
|
// Nothing is read up front — sources are opened one at a time as the
|
|
// archive is pumped, so only one file is ever held open.
|
|
expect(opened.length).toBeLessThan(3);
|
|
});
|
|
|
|
expect(opened).toEqual(["a.txt", "b.txt", "c.txt"]);
|
|
await fs.remove(filePath);
|
|
});
|
|
|
|
it("propagates errors thrown while adding entries and cleans up", async () => {
|
|
const tmpFile = watchTmpFile();
|
|
|
|
await expect(
|
|
ZipHelper.toTmpFile(async (zip) => {
|
|
zip.addBuffer(Buffer.from("partial"), "a.txt");
|
|
await Promise.resolve();
|
|
throw new Error("boom");
|
|
})
|
|
).rejects.toThrow("boom");
|
|
|
|
expect(await fs.pathExists(tmpFile())).toBe(false);
|
|
});
|
|
|
|
it("propagates errors raised when opening an entry and cleans up", async () => {
|
|
const tmpFile = watchTmpFile();
|
|
|
|
await expect(
|
|
ZipHelper.toTmpFile(async (zip) => {
|
|
zip.addReadStreamLazy("a.txt", {}, (callback) =>
|
|
callback(new Error("stream unavailable"), Readable.from([]))
|
|
);
|
|
await Promise.resolve();
|
|
})
|
|
).rejects.toThrow("stream unavailable");
|
|
|
|
expect(await fs.pathExists(tmpFile())).toBe(false);
|
|
});
|
|
|
|
it("fails rather than stalls when an entry's stream errors mid-read", async () => {
|
|
const tmpFile = watchTmpFile();
|
|
|
|
await expect(
|
|
ZipHelper.toTmpFile(async (zip) => {
|
|
const source = new Readable({
|
|
read() {
|
|
this.push("partial");
|
|
this.destroy(new Error("connection reset"));
|
|
},
|
|
});
|
|
// Mirrors how ExportTask surfaces an interrupted read, which yazl
|
|
// does not watch for on the streams it is given.
|
|
source.on("error", (err) => zip.emit("error", err));
|
|
zip.addReadStreamLazy("a.txt", {}, (callback) =>
|
|
callback(null, source)
|
|
);
|
|
await Promise.resolve();
|
|
})
|
|
).rejects.toThrow("connection reset");
|
|
|
|
expect(await fs.pathExists(tmpFile())).toBe(false);
|
|
});
|
|
});
|
|
|
|
describe("ZipHelper.toFileTree", () => {
|
|
it("builds a nested tree with normalized pathInZip", async () => {
|
|
const zipPath = await writeZip({
|
|
"Collection/sub/page.md": "# hi",
|
|
"Collection/other.md": "other",
|
|
});
|
|
|
|
const root = await ZipHelper.toFileTree(zipPath);
|
|
expect(root.children).toHaveLength(1);
|
|
|
|
const collection = root.children[0];
|
|
expect(collection.name).toBe("Collection");
|
|
expect(collection.pathInZip).toBe("Collection");
|
|
expect(collection.children.map((c) => c.name).sort()).toEqual([
|
|
"other.md",
|
|
"sub",
|
|
]);
|
|
|
|
const sub = collection.children.find((c) => c.name === "sub");
|
|
expect(sub?.children[0].pathInZip).toBe("Collection/sub/page.md");
|
|
});
|
|
|
|
it("normalizes `./`-prefixed entries instead of dropping them", async () => {
|
|
const zipPath = await writeZip({
|
|
"./Collection/page.md": "body",
|
|
});
|
|
|
|
const root = await ZipHelper.toFileTree(zipPath);
|
|
expect(root.children).toHaveLength(1);
|
|
expect(root.children[0].name).toBe("Collection");
|
|
expect(root.children[0].children[0].pathInZip).toBe("Collection/page.md");
|
|
});
|
|
|
|
it("filters macOS metadata and dotfiles at any depth", async () => {
|
|
const zipPath = await writeZip({
|
|
"__MACOSX/Collection/page.md": "junk",
|
|
"Collection/.DS_Store": "junk",
|
|
"Collection/page.md": "body",
|
|
});
|
|
|
|
const root = await ZipHelper.toFileTree(zipPath);
|
|
expect(root.children).toHaveLength(1);
|
|
expect(root.children[0].name).toBe("Collection");
|
|
expect(root.children[0].children.map((c) => c.name)).toEqual(["page.md"]);
|
|
});
|
|
|
|
it("invokes onFile for each file entry with a readable handle", async () => {
|
|
const zipPath = await writeZip({
|
|
"Collection/page.md": "hello world",
|
|
"Collection/image.png": "binary",
|
|
});
|
|
|
|
const seen: Record<string, string> = {};
|
|
await ZipHelper.toFileTree(zipPath, async (node, entry) => {
|
|
if (node.name.endsWith(".md")) {
|
|
const buf = await entry.readBuffer(100);
|
|
seen[node.pathInZip] = buf.toString("utf8");
|
|
}
|
|
});
|
|
|
|
expect(seen).toEqual({ "Collection/page.md": "hello world" });
|
|
});
|
|
|
|
it("rejects reads larger than the provided max size", async () => {
|
|
const zipPath = await writeZip({
|
|
"Collection/page.md": "hello world",
|
|
});
|
|
|
|
await expect(
|
|
ZipHelper.toFileTree(zipPath, async (_node, entry) => {
|
|
await entry.readBuffer(10);
|
|
})
|
|
).rejects.toThrow("Collection/page.md is too large");
|
|
});
|
|
|
|
it("exposes entry sizes before the entry is read", async () => {
|
|
const zipPath = await writeZip({
|
|
"Collection/page.md": "hello world",
|
|
});
|
|
|
|
const sizes: Record<string, number> = {};
|
|
await ZipHelper.toFileTree(zipPath, (node, entry) => {
|
|
sizes[node.pathInZip] = entry.uncompressedSize;
|
|
});
|
|
|
|
expect(sizes).toEqual({ "Collection/page.md": 11 });
|
|
});
|
|
});
|
|
|
|
describe("ZipHelper.walk", () => {
|
|
it("walks an archive held in memory", async () => {
|
|
const buffer = await buildZip({
|
|
"Collection/page.md": "hello world",
|
|
"Collection/image.png": "binary",
|
|
});
|
|
|
|
const contents: Record<string, string> = {};
|
|
await ZipHelper.walk(buffer, async (entry) => {
|
|
if (!entry.isDirectory) {
|
|
contents[entry.fileName] = (await entry.readBuffer(1024)).toString(
|
|
"utf8"
|
|
);
|
|
}
|
|
});
|
|
|
|
expect(contents).toEqual({
|
|
"Collection/page.md": "hello world",
|
|
"Collection/image.png": "binary",
|
|
});
|
|
});
|
|
|
|
it("rejects an oversized entry in an archive held in memory", async () => {
|
|
const buffer = await buildZip({ "Collection/page.md": "hello world" });
|
|
|
|
await expect(
|
|
ZipHelper.walk(buffer, async (entry) => {
|
|
await entry.readBuffer(4);
|
|
})
|
|
).rejects.toThrow("Collection/page.md is too large");
|
|
});
|
|
|
|
it("rejects an archive with more entries than allowed", async () => {
|
|
const buffer = await buildZip({ "a.md": "a", "b.md": "b", "c.md": "c" });
|
|
const onEntry = vi.fn();
|
|
|
|
await expect(
|
|
ZipHelper.walk(buffer, onEntry, { maxEntries: 2 })
|
|
).rejects.toMatchObject({
|
|
id: "import_error",
|
|
message: "The archive contains too many files - the maximum is 2",
|
|
});
|
|
expect(onEntry).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it("rejects reads that exceed the total size across entries", async () => {
|
|
const buffer = await buildZip({
|
|
"a.md": "x".repeat(600),
|
|
"b.md": "x".repeat(600),
|
|
});
|
|
|
|
await expect(
|
|
ZipHelper.walk(
|
|
buffer,
|
|
async (entry) => {
|
|
await entry.readBuffer(1024);
|
|
},
|
|
{ maxTotalSize: 1024 }
|
|
)
|
|
).rejects.toMatchObject({
|
|
id: "import_error",
|
|
message: expect.stringContaining("too large when uncompressed"),
|
|
});
|
|
});
|
|
|
|
it("does not count skipped entries toward the total size", async () => {
|
|
const buffer = await buildZip({
|
|
"a.md": "x".repeat(600),
|
|
"b.png": "x".repeat(600),
|
|
});
|
|
|
|
await expect(
|
|
ZipHelper.walk(
|
|
buffer,
|
|
async (entry) => {
|
|
if (entry.fileName.endsWith(".md")) {
|
|
await entry.readBuffer(1024);
|
|
}
|
|
},
|
|
{ maxTotalSize: 1024 }
|
|
)
|
|
).resolves.toBeUndefined();
|
|
});
|
|
|
|
it("rejects a file that is not a zip archive as an import error", async () => {
|
|
const filePath = tmp.fileSync({ postfix: ".zip" }).name;
|
|
await fs.writeFile(filePath, "not a zip file");
|
|
|
|
await expect(ZipHelper.walk(filePath, () => {})).rejects.toMatchObject({
|
|
id: "import_error",
|
|
message: "The file is not a valid zip archive",
|
|
});
|
|
await fs.remove(filePath);
|
|
});
|
|
|
|
it("rejects a buffer that is not a zip archive as an import error", async () => {
|
|
await expect(
|
|
ZipHelper.walk(Buffer.from("not a zip file"), () => {})
|
|
).rejects.toMatchObject({
|
|
id: "import_error",
|
|
message: "The file is not a valid zip archive",
|
|
});
|
|
});
|
|
|
|
it("rejects an entry with corrupt compressed data as an import error", async () => {
|
|
const buffer = await buildZip({ "page.md": "hello world ".repeat(100) });
|
|
// An invalid deflate block type follows the local header of the entry.
|
|
buffer[30 + "page.md".length] = 0xff;
|
|
|
|
await expect(
|
|
ZipHelper.walk(buffer, async (entry) => {
|
|
await entry.readBuffer(4096);
|
|
})
|
|
).rejects.toMatchObject({
|
|
id: "import_error",
|
|
message: "The file is not a valid zip archive",
|
|
});
|
|
});
|
|
|
|
it("keeps file system errors when the archive cannot be opened", async () => {
|
|
await expect(
|
|
ZipHelper.walk("/does/not/exist.zip", () => {})
|
|
).rejects.toMatchObject({ code: "ENOENT" });
|
|
});
|
|
});
|