mirror of
https://github.com/outline/outline.git
synced 2026-11-20 21:23:06 +03:00
* chore: Replace tunnel-agent with https-proxy-agent * fix: Keep the target host in proxied HTTP request URLs * test: Move test proxy into test support
267 lines
8.0 KiB
TypeScript
267 lines
8.0 KiB
TypeScript
/* oxlint-disable no-restricted-imports, react/rules-of-hooks */
|
|
import { promises as dns } from "node:dns";
|
|
import type http from "node:http";
|
|
import type https from "node:https";
|
|
import * as net from "node:net";
|
|
import nodeFetch, {
|
|
Headers,
|
|
type RequestInit,
|
|
type Response,
|
|
} from "node-fetch";
|
|
import { HttpProxyAgent } from "http-proxy-agent";
|
|
import { HttpsProxyAgent } from "https-proxy-agent";
|
|
import { getProxyForUrl } from "proxy-from-env";
|
|
import { errToString } from "@shared/utils/error";
|
|
import env from "@server/env";
|
|
import { InvalidRequestError } from "@server/errors";
|
|
import Logger from "@server/logging/Logger";
|
|
import {
|
|
type RequestFilteringAgentOptions,
|
|
useAgent as useFilteringAgent,
|
|
validateIPAddress,
|
|
} from "./requestFilteringAgent";
|
|
|
|
const DefaultOptions = {
|
|
keepAlive: true,
|
|
timeout: 1000,
|
|
keepAliveMsecs: 500,
|
|
maxSockets: 200,
|
|
maxFreeSockets: 5,
|
|
maxCachedSessions: 500,
|
|
};
|
|
|
|
export type { RequestInit, Response } from "node-fetch";
|
|
export { Headers };
|
|
|
|
/**
|
|
* Default user agent string for outgoing requests.
|
|
*/
|
|
export const outlineUserAgent = `Outline-${
|
|
env.VERSION ? `/${env.VERSION.slice(0, 7)}` : ""
|
|
}`;
|
|
|
|
/**
|
|
* Fake Chrome user agent string for use in fetch requests to
|
|
* improve reliability.
|
|
*/
|
|
export const chromeUserAgent =
|
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36";
|
|
|
|
/**
|
|
* Resolves the hostname and validates every returned address against the
|
|
* filtering rules.
|
|
*
|
|
* @param req The request that will connect to the host.
|
|
* @param host The target hostname or IP address.
|
|
* @param options Allow/deny rules to apply.
|
|
* @throws An error if any resolved address is disallowed, or if the request
|
|
* was aborted during the lookup.
|
|
*/
|
|
const validateTarget = async (
|
|
req: http.ClientRequest,
|
|
host: string,
|
|
options: RequestFilteringAgentOptions
|
|
): Promise<void> => {
|
|
const addresses =
|
|
net.isIP(host) !== 0
|
|
? [{ address: host, family: net.isIP(host) }]
|
|
: await dns.lookup(host, { all: true });
|
|
|
|
for (const { address, family } of addresses) {
|
|
const err = validateIPAddress({ address, family, host }, options);
|
|
if (err) {
|
|
throw err;
|
|
}
|
|
}
|
|
|
|
if (req.destroyed) {
|
|
throw new Error("Request aborted");
|
|
}
|
|
};
|
|
|
|
/**
|
|
* Proxy agents connect to the proxy rather than the target, so they skip the
|
|
* filtering agent's DNS hook. These validate the target of every request,
|
|
* including redirects, before connecting. The proxy resolves DNS again, so
|
|
* only egress rules on the proxy can fully prevent DNS rebinding.
|
|
*/
|
|
class FilteringHttpProxyAgent extends HttpProxyAgent<string> {
|
|
constructor(
|
|
proxy: string,
|
|
private filtering: RequestFilteringAgentOptions
|
|
) {
|
|
super(proxy, DefaultOptions);
|
|
}
|
|
|
|
setRequestProps(
|
|
...args: Parameters<HttpProxyAgent<string>["setRequestProps"]>
|
|
) {
|
|
const [req, opts] = args;
|
|
const path = req.path;
|
|
super.setRequestProps(...args);
|
|
|
|
// The parent resolves the path against the Host header, so a path such
|
|
// as "//other.host/" or a custom Host header would change the target.
|
|
if (path.startsWith("/")) {
|
|
const host = net.isIPv6(opts.host ?? "") ? `[${opts.host}]` : opts.host;
|
|
const protocol = opts.secureEndpoint ? "https:" : "http:";
|
|
const defaultPort = opts.secureEndpoint ? 443 : 80;
|
|
const port =
|
|
opts.port && opts.port !== defaultPort ? `:${opts.port}` : "";
|
|
req.path = `${protocol}//${host}${port}${path}`;
|
|
}
|
|
}
|
|
|
|
async connect(...args: Parameters<HttpProxyAgent<string>["connect"]>) {
|
|
await validateTarget(args[0], args[1].host ?? "", this.filtering);
|
|
return super.connect(...args);
|
|
}
|
|
}
|
|
|
|
class FilteringHttpsProxyAgent extends HttpsProxyAgent<string> {
|
|
constructor(
|
|
proxy: string,
|
|
private filtering: RequestFilteringAgentOptions
|
|
) {
|
|
super(proxy, DefaultOptions);
|
|
}
|
|
|
|
async connect(...args: Parameters<HttpsProxyAgent<string>["connect"]>) {
|
|
await validateTarget(args[0], args[1].host ?? "", this.filtering);
|
|
return super.connect(...args);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Wrapper around fetch that filters requests to private IP addresses, and sends
|
|
* requests through a proxy when one is configured in the environment.
|
|
*
|
|
* @param url The url to fetch
|
|
* @param init The fetch init object
|
|
* @param init.allowIPAddressList Private IP addresses or CIDR ranges that may
|
|
* be requested. Defaults to the ALLOWED_PRIVATE_IP_ADDRESSES environment
|
|
* variable; pass an empty list to block every private address.
|
|
* @param init.timeout Time limit in milliseconds.
|
|
* @returns The response
|
|
*/
|
|
export default async function fetch(
|
|
url: string,
|
|
init?: RequestInit & {
|
|
allowPrivateIPAddress?: boolean;
|
|
allowIPAddressList?: string[];
|
|
timeout?: number;
|
|
}
|
|
): Promise<Response> {
|
|
Logger.silly("http", `Network request to ${url}`, init);
|
|
|
|
const {
|
|
allowPrivateIPAddress,
|
|
allowIPAddressList = env.ALLOWED_PRIVATE_IP_ADDRESSES ?? [],
|
|
timeout,
|
|
...rest
|
|
} = init || {};
|
|
|
|
// The timeout signal stays active after fetch returns so it also limits
|
|
// reading the body.
|
|
const timeoutSignal = timeout ? AbortSignal.timeout(timeout) : undefined;
|
|
const signal =
|
|
rest.signal && timeoutSignal
|
|
? AbortSignal.any([rest.signal, timeoutSignal])
|
|
: (rest.signal ?? timeoutSignal);
|
|
|
|
try {
|
|
const headers = new Headers({ "User-Agent": outlineUserAgent });
|
|
new Headers(rest?.headers).forEach((value, key) => {
|
|
headers.set(key, value);
|
|
});
|
|
|
|
const response = await nodeFetch(url, {
|
|
...rest,
|
|
headers,
|
|
signal,
|
|
// A function so that redirects get an agent for their own protocol.
|
|
agent: (requestURL) =>
|
|
buildAgent(requestURL.href, {
|
|
signal,
|
|
allowPrivateIPAddress,
|
|
allowIPAddressList,
|
|
}),
|
|
});
|
|
|
|
if (!response.ok) {
|
|
Logger.silly("http", `Network request failed`, {
|
|
url,
|
|
status: response.status,
|
|
statusText: response.statusText,
|
|
headers: response.headers.raw(),
|
|
});
|
|
}
|
|
|
|
return response;
|
|
} catch (err) {
|
|
if (
|
|
timeoutSignal?.aborted &&
|
|
err instanceof Error &&
|
|
err.name === "AbortError"
|
|
) {
|
|
throw new Error(`Request timeout after ${timeout}ms`);
|
|
}
|
|
const message = errToString(err);
|
|
if (message.includes("DNS lookup")) {
|
|
throw InvalidRequestError(
|
|
env.isCloudHosted
|
|
? message
|
|
: `${message}\n\nTo allow this request, add the IP address or CIDR range to the ALLOWED_PRIVATE_IP_ADDRESSES environment variable.`
|
|
);
|
|
}
|
|
throw err;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Creates a http or https agent for the given URL, applying request filtering
|
|
* if necessary. If a proxy is detected in the environment, the request is sent
|
|
* through that proxy instead.
|
|
*
|
|
* @param url The URL to fetch.
|
|
* @param options Options controlling agent behavior.
|
|
* @param options.signal An abort signal used to destroy the agent on cancellation or timeout.
|
|
* @param options.allowPrivateIPAddress Whether to allow requests to private IP addresses.
|
|
* @param options.allowIPAddressList Private IP addresses or CIDR ranges that may be requested.
|
|
* @returns An http or https agent configured for the URL.
|
|
*/
|
|
function buildAgent(
|
|
url: string,
|
|
options: {
|
|
signal?: AbortSignal | null;
|
|
allowPrivateIPAddress?: boolean;
|
|
allowIPAddressList?: string[];
|
|
} = {}
|
|
) {
|
|
const proxyURL = getProxyForUrl(url);
|
|
const filtering = {
|
|
allowPrivateIPAddress: options.allowPrivateIPAddress,
|
|
allowIPAddressList:
|
|
options.allowIPAddressList ?? env.ALLOWED_PRIVATE_IP_ADDRESSES ?? [],
|
|
};
|
|
let agent: https.Agent | http.Agent;
|
|
|
|
if (proxyURL) {
|
|
Logger.silly("http", `Using proxy for request`, { url });
|
|
agent =
|
|
new URL(url).protocol === "https:"
|
|
? new FilteringHttpsProxyAgent(proxyURL, filtering)
|
|
: new FilteringHttpProxyAgent(proxyURL, filtering);
|
|
} else {
|
|
agent = useFilteringAgent(url, { ...DefaultOptions, ...filtering });
|
|
}
|
|
|
|
if (options.signal) {
|
|
options.signal.addEventListener("abort", () => {
|
|
agent.destroy();
|
|
});
|
|
}
|
|
|
|
return agent;
|
|
}
|