From 036b1fd66d8e18091c826e644aef872aee92ffcc Mon Sep 17 00:00:00 2001 From: Joyee Cheung Date: Wed, 2 Jul 2025 01:17:07 +0200 Subject: [PATCH] http,https: add built-in proxy support in http/https.request and Agent This patch implements proxy support for HTTP and HTTPS clients and agents in the `http` and `https` built-ins`. When NODE_USE_ENV_PROXY is set to 1, the default global agent would parse the HTTP_PROXY/http_proxy, HTTPS_PROXY/https_proxy, NO_PROXY/no_proxy settings from the environment variables, and proxy the requests sent through the built-in http/https client accordingly. To support this, `http.Agent` and `https.Agent` now accept a few new options: - `proxyEnv`: when it's an object, the agent would read and parse the HTTP_PROXY/http_proxy, HTTPS_PROXY/https_proxy, NO_PROXY/no_proxy properties from it, and apply them based on the protocol it uses to send requests. This option allows custom agents to reuse built-in proxy support by composing options. Global agents set this to `process.env` when NODE_USE_ENV_PROXY is 1. - `defaultPort` and `protocol`: these allow setting of the default port and protocol of the agents. We also need these when configuring proxy settings and deciding whether a request should be proxied. Implementation-wise, this adds a `ProxyConfig` internal class to handle parsing and application of proxy configurations. The configuration is parsed during agent construction. When requests are made, the `createConnection()` methods on the agents would check whether the request should be proxied. If yes, they either connect to the proxy server (in the case of HTTP reqeusts) or establish a tunnel (in the case of HTTPS requests) through either a TCP socket (if the proxy uses HTTP) or a TLS socket (if the proxy uses HTTPS). When proxying HTTPS requests through a tunnel, the connection listener is invoked after the tunnel is established. Tunnel establishment uses the timeout of the request options, if there is one. Otherwise it uses the timeout of the agent. If an error is encountered during tunnel establishment, an ERR_PROXY_TUNNEL would be emitted on the returned socket. If the proxy server sends a errored status code, the error would contain an `statusCode` property. If the error is caused by timeout, the error would contain a `proxyTunnelTimeout` property. This implementation honors the built-in socket pool and socket limits. Pooled sockets are still keyed by request endpoints, they are just connected to the proxy server instead, and the persistence of the connection can be maintained as long as the proxy server respects connection/proxy-connection or persist by default (HTTP/1.1) PR-URL: https://github.com/nodejs/node/pull/58980 Refs: https://github.com/nodejs/node/issues/57872 Refs: https://github.com/nodejs/node/issues/8381 Refs: https://github.com/nodejs/node/issues/15620 Reviewed-By: Matteo Collina --- doc/api/errors.md | 12 + doc/api/http.md | 114 +++++++ doc/api/https.md | 8 + lib/_http_agent.js | 110 ++++++- lib/_http_client.js | 56 ++++ lib/https.js | 282 ++++++++++++++++-- lib/internal/errors.js | 2 + lib/internal/http.js | 192 ++++++++++++ lib/internal/process/pre_execution.js | 5 +- ...-http-proxy-request-connection-refused.mjs | 39 +++ .../test-http-proxy-request-https-proxy.mjs | 54 ++++ ...-proxy-request-invalid-char-in-options.mjs | 44 +++ ...http-proxy-request-invalid-char-in-url.mjs | 93 ++++++ ...http-proxy-request-invalid-credentials.mjs | 49 +++ .../test-http-proxy-request-invalid-url.mjs | 30 ++ .../test-http-proxy-request-max-sockets.mjs | 123 ++++++++ ...t-http-proxy-request-no-proxy-asterisk.mjs | 42 +++ ...est-http-proxy-request-no-proxy-domain.mjs | 81 +++++ .../test-http-proxy-request-no-proxy-ip.mjs | 59 ++++ ...p-proxy-request-no-proxy-port-specific.mjs | 80 +++++ .../test-http-proxy-request-no-proxy.mjs | 60 ++++ ...t-http-proxy-request-proxy-failure-500.mjs | 55 ++++ ...tp-proxy-request-proxy-failure-hang-up.mjs | 39 +++ ...t-http-proxy-request-socket-keep-alive.mjs | 102 +++++++ test/client-proxy/test-http-proxy-request.mjs | 93 ++++++ ...http-request-proxy-post-server-failure.mjs | 60 ++++ .../test-http-request-proxy-post.mjs | 62 ++++ test/client-proxy/test-https-proxy-fetch.mjs | 2 +- .../test-https-proxy-request-auth-failure.mjs | 60 ++++ ...https-proxy-request-connection-refused.mjs | 48 +++ ...est-https-proxy-request-empty-response.mjs | 51 ++++ ...-https-proxy-request-handshake-failure.mjs | 57 ++++ .../test-https-proxy-request-https-proxy.mjs | 59 ++++ ...https-proxy-request-incomplete-headers.mjs | 53 ++++ ...-proxy-request-invalid-char-in-options.mjs | 55 ++++ ...ttps-proxy-request-invalid-char-in-url.mjs | 84 ++++++ ...ttps-proxy-request-invalid-credentials.mjs | 56 ++++ .../test-https-proxy-request-invalid-url.mjs | 40 +++ ...https-proxy-request-malformed-response.mjs | 52 ++++ .../test-https-proxy-request-max-sockets.mjs | 120 ++++++++ .../test-https-proxy-request-no-proxy.mjs | 54 ++++ ...-https-proxy-request-proxy-failure-404.mjs | 52 ++++ ...-https-proxy-request-proxy-failure-500.mjs | 53 ++++ ...-https-proxy-request-proxy-failure-502.mjs | 51 ++++ ...ps-proxy-request-proxy-failure-hang-up.mjs | 51 ++++ ...s-proxy-request-server-failure-hang-up.mjs | 61 ++++ ...-https-proxy-request-socket-keep-alive.mjs | 101 +++++++ ...tps-proxy-request-tunnel-timeout-agent.mjs | 54 ++++ ...est-https-proxy-request-tunnel-timeout.mjs | 54 ++++ .../client-proxy/test-https-proxy-request.mjs | 107 +++++++ .../test-https-request-proxy-post.mjs | 74 +++++ test/common/proxy-server.js | 29 +- test/fixtures/post-resource-and-log.js | 50 ++++ test/fixtures/proxy-server-worker.js | 24 ++ test/fixtures/request-and-log.js | 55 ++++ 55 files changed, 3516 insertions(+), 37 deletions(-) create mode 100644 test/client-proxy/test-http-proxy-request-connection-refused.mjs create mode 100644 test/client-proxy/test-http-proxy-request-https-proxy.mjs create mode 100644 test/client-proxy/test-http-proxy-request-invalid-char-in-options.mjs create mode 100644 test/client-proxy/test-http-proxy-request-invalid-char-in-url.mjs create mode 100644 test/client-proxy/test-http-proxy-request-invalid-credentials.mjs create mode 100644 test/client-proxy/test-http-proxy-request-invalid-url.mjs create mode 100644 test/client-proxy/test-http-proxy-request-max-sockets.mjs create mode 100644 test/client-proxy/test-http-proxy-request-no-proxy-asterisk.mjs create mode 100644 test/client-proxy/test-http-proxy-request-no-proxy-domain.mjs create mode 100644 test/client-proxy/test-http-proxy-request-no-proxy-ip.mjs create mode 100644 test/client-proxy/test-http-proxy-request-no-proxy-port-specific.mjs create mode 100644 test/client-proxy/test-http-proxy-request-no-proxy.mjs create mode 100644 test/client-proxy/test-http-proxy-request-proxy-failure-500.mjs create mode 100644 test/client-proxy/test-http-proxy-request-proxy-failure-hang-up.mjs create mode 100644 test/client-proxy/test-http-proxy-request-socket-keep-alive.mjs create mode 100644 test/client-proxy/test-http-proxy-request.mjs create mode 100644 test/client-proxy/test-http-request-proxy-post-server-failure.mjs create mode 100644 test/client-proxy/test-http-request-proxy-post.mjs create mode 100644 test/client-proxy/test-https-proxy-request-auth-failure.mjs create mode 100644 test/client-proxy/test-https-proxy-request-connection-refused.mjs create mode 100644 test/client-proxy/test-https-proxy-request-empty-response.mjs create mode 100644 test/client-proxy/test-https-proxy-request-handshake-failure.mjs create mode 100644 test/client-proxy/test-https-proxy-request-https-proxy.mjs create mode 100644 test/client-proxy/test-https-proxy-request-incomplete-headers.mjs create mode 100644 test/client-proxy/test-https-proxy-request-invalid-char-in-options.mjs create mode 100644 test/client-proxy/test-https-proxy-request-invalid-char-in-url.mjs create mode 100644 test/client-proxy/test-https-proxy-request-invalid-credentials.mjs create mode 100644 test/client-proxy/test-https-proxy-request-invalid-url.mjs create mode 100644 test/client-proxy/test-https-proxy-request-malformed-response.mjs create mode 100644 test/client-proxy/test-https-proxy-request-max-sockets.mjs create mode 100644 test/client-proxy/test-https-proxy-request-no-proxy.mjs create mode 100644 test/client-proxy/test-https-proxy-request-proxy-failure-404.mjs create mode 100644 test/client-proxy/test-https-proxy-request-proxy-failure-500.mjs create mode 100644 test/client-proxy/test-https-proxy-request-proxy-failure-502.mjs create mode 100644 test/client-proxy/test-https-proxy-request-proxy-failure-hang-up.mjs create mode 100644 test/client-proxy/test-https-proxy-request-server-failure-hang-up.mjs create mode 100644 test/client-proxy/test-https-proxy-request-socket-keep-alive.mjs create mode 100644 test/client-proxy/test-https-proxy-request-tunnel-timeout-agent.mjs create mode 100644 test/client-proxy/test-https-proxy-request-tunnel-timeout.mjs create mode 100644 test/client-proxy/test-https-proxy-request.mjs create mode 100644 test/client-proxy/test-https-request-proxy-post.mjs create mode 100644 test/fixtures/post-resource-and-log.js create mode 100644 test/fixtures/proxy-server-worker.js create mode 100644 test/fixtures/request-and-log.js diff --git a/doc/api/errors.md b/doc/api/errors.md index df0fe46e32..687720d7f4 100644 --- a/doc/api/errors.md +++ b/doc/api/errors.md @@ -2491,6 +2491,18 @@ Accessing `Object.prototype.__proto__` has been forbidden using [`Object.setPrototypeOf`][] should be used to get and set the prototype of an object. + + +### `ERR_PROXY_INVALID_CONFIG` + +Failed to proxy a request because the proxy configuration is invalid. + + + +### `ERR_PROXY_TUNNEL` + +Failed to establish proxy tunnel when `NODE_USE_ENV_PROXY` is enabled. + ### `ERR_QUIC_APPLICATION_ERROR` diff --git a/doc/api/http.md b/doc/api/http.md index 29976d9b50..af7a121873 100644 --- a/doc/api/http.md +++ b/doc/api/http.md @@ -116,6 +116,14 @@ http.get({ + +> Stability: 1.1 - Active development + +When Node.js creates the global agent, it checks the `NODE_USE_ENV_PROXY` +environment variable. If it is set to `1`, the global agent will be constructed +with `proxyEnv: process.env`, enabling proxy support based on the environment variables. + +Custom agents can also be created with proxy support by passing a +`proxyEnv` option when constructing the agent. The value can be `process.env` +if they just want to inherit the configuration from the environment variables, +or an object with specific setting overriding the environment. + +The following properties of the `proxyEnv` are checked to configure proxy +support. + +* `HTTP_PROXY` or `http_proxy`: Proxy server URL for HTTP requests. If both are set, + `http_proxy` takes precedence. +* `HTTPS_PROXY` or `https_proxy`: Proxy server URL for HTTPS requests. If both are set, + `https_proxy` takes precedence. +* `NO_PROXY` or `no_proxy`: Comma-separated list of hosts to bypass the proxy. If both are set, + `no_proxy` takes precedence. + +If the request is made to a Unix domain socket, the proxy settings will be ignored. + +### Proxy URL Format + +Proxy URLs can use either HTTP or HTTPS protocols: + +* HTTP proxy: `http://proxy.example.com:8080` +* HTTPS proxy: `https://proxy.example.com:8080` +* Proxy with authentication: `http://username:password@proxy.example.com:8080` + +### `NO_PROXY` Format + +The `NO_PROXY` environment variable supports several formats: + +* `*` - Bypass proxy for all hosts +* `example.com` - Exact host name match +* `.example.com` - Domain suffix match (matches `sub.example.com`) +* `*.example.com` - Wildcard domain match +* `192.168.1.100` - Exact IP address match +* `192.168.1.1-192.168.1.100` - IP address range +* `example.com:8080` - Hostname with specific port + +Multiple entries should be separated by commas. + +### Example + +Starting a Node.js process with proxy support enabled for all requests sent +through the default global agent: + +```console +NODE_USE_ENV_PROXY=1 HTTP_PROXY=http://proxy.example.com:8080 NO_PROXY=localhost,127.0.0.1 node client.js +``` + +To create a custom agent with built-in proxy support: + +```cjs +const http = require('node:http'); + +// Creating a custom agent with custom proxy support. +const agent = new http.Agent({ proxyEnv: { HTTP_PROXY: 'http://proxy.example.com:8080' } }); + +http.request({ + hostname: 'www.example.com', + port: 80, + path: '/', + agent, +}, (res) => { + // This request will be proxied through proxy.example.com:8080 using the HTTP protocol. + console.log(`STATUS: ${res.statusCode}`); +}); +``` + +Alternatively, the following also works: + +```cjs +const http = require('node:http'); +// Use lower-cased option name. +const agent1 = new http.Agent({ proxyEnv: { http_proxy: 'http://proxy.example.com:8080' } }); +// Use values inherited from the environment variables, if the process is started with +// HTTP_PROXY=http://proxy.example.com:8080 this will use the proxy server specified +// in process.env.HTTP_PROXY. +const agent2 = new http.Agent({ proxyEnv: process.env }); +``` + +[Built-in Proxy Support]: #built-in-proxy-support [RFC 8187]: https://www.rfc-editor.org/rfc/rfc8187.txt [`'ERR_HTTP_CONTENT_LENGTH_MISMATCH'`]: errors.md#err_http_content_length_mismatch [`'checkContinue'`]: #event-checkcontinue diff --git a/doc/api/https.md b/doc/api/https.md index 04a57143ad..e8560c26ba 100644 --- a/doc/api/https.md +++ b/doc/api/https.md @@ -65,6 +65,14 @@ An [`Agent`][] object for HTTPS similar to [`http.Agent`][]. See