From d0f5856ad8d75b8c0da4a89840c11e6d718fa70c Mon Sep 17 00:00:00 2001 From: ruby Date: Wed, 15 Apr 2026 23:45:24 +0400 Subject: [PATCH 1/3] fix: improve timeout handling, abort behavior, and project linting --- .github/workflows/test.yml | 11 ++ docs/BUILD.md | 2 +- package-lock.json | 86 +++++++++- package.json | 18 ++- rust/src/emulation/builders.rs | 2 +- rust/src/napi/convert.rs | 36 +++-- rust/src/napi/request.rs | 34 +++- rust/src/store/body_store.rs | 76 +++++---- rust/src/store/mod.rs | 1 + rust/src/store/request_store.rs | 42 +++++ rust/src/transport/dns.rs | 2 +- rust/src/transport/mod.rs | 2 +- rust/src/transport/request.rs | 15 +- rust/src/transport/tls.rs | 2 +- rust/src/transport/types.rs | 9 +- rust/src/transport/websocket.rs | 7 +- scripts/generate-browser-profiles.mjs | 11 +- scripts/install-git-hooks.mjs | 2 +- scripts/platform-targets.mjs | 70 ++++---- scripts/postbuild.mjs | 56 +++---- scripts/prepare-main-package.mjs | 32 ++-- scripts/prepare-platform-package.mjs | 36 ++--- scripts/update-wreq-upstream.mjs | 8 +- src/http/fetch.ts | 6 +- src/http/pipeline/dispatch.ts | 20 ++- src/http/pipeline/options.ts | 15 +- src/http/request.ts | 1 + src/native.ts | 219 ++++++++++++++++++++++++++ src/native/binding.ts | 6 +- src/native/request.ts | 58 ++++++- src/test/helpers/local-server.ts | 4 +- src/test/http-client.spec.ts | 69 ++++++++ src/test/transport-features.spec.ts | 25 ++- src/websocket/index.ts | 16 +- 34 files changed, 802 insertions(+), 197 deletions(-) create mode 100644 rust/src/store/request_store.rs create mode 100644 src/native.ts diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 096079f..85711c3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -79,6 +79,11 @@ jobs: - name: Install dependencies run: npm install + - name: Install Rust linting components + uses: dtolnay/rust-toolchain@stable + with: + components: rustfmt, clippy + - name: Check TypeScript types run: npx tsc --noEmit @@ -87,3 +92,9 @@ jobs: - name: Check code formatting run: npm run format:check + + - name: Check Rust formatting + run: npm run format:rust:check + + - name: Run clippy + run: npm run lint:rust diff --git a/docs/BUILD.md b/docs/BUILD.md index b4f0bee..7297b3f 100644 --- a/docs/BUILD.md +++ b/docs/BUILD.md @@ -147,7 +147,7 @@ npm run build:rust npm install # Clean and rebuild -rm -rf rust/target +npx rimraf rust/target npm run build:rust ``` diff --git a/package-lock.json b/package-lock.json index db030de..cb04c88 100644 --- a/package-lock.json +++ b/package-lock.json @@ -24,6 +24,7 @@ "@types/ws": "^8.18.1", "oxfmt": "^0.45.0", "oxlint": "^1.60.0", + "rimraf": "^6.1.3", "typescript": "^5.0.0", "ws": "^8.20.0" }, @@ -987,7 +988,6 @@ "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": "18 || 20 || >=22" } @@ -998,7 +998,6 @@ "integrity": "sha512-VZznLgtwhn+Mact9tfiwx64fA9erHH/MCXEUfB/0bX/6Fz6ny5EGTXYltMocqg4xFAQZtnO3DHWWXi8RiuN7cQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "balanced-match": "^4.0.2" }, @@ -1332,6 +1331,24 @@ "license": "ISC", "peer": true }, + "node_modules/glob": { + "version": "13.0.6", + "resolved": "https://registry.npmjs.org/glob/-/glob-13.0.6.tgz", + "integrity": "sha512-Wjlyrolmm8uDpm/ogGyXZXb1Z+Ca2B8NbJwqBVg0axK9GbBeoS7yGV6vjXnYdGm6X53iehEuxxbyiKp8QmN4Vw==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "minimatch": "^10.2.2", + "minipass": "^7.1.3", + "path-scurry": "^2.0.2" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/glob-parent": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", @@ -1468,13 +1485,22 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/lru-cache": { + "version": "11.3.5", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.3.5.tgz", + "integrity": "sha512-NxVFwLAnrd9i7KUBxC4DrUhmgjzOs+1Qm50D3oF1/oL+r1NpZ4gA7xvG0/zJ8evR7zIKn4vLf7qTNduWFtCrRw==", + "dev": true, + "license": "BlueOak-1.0.0", + "engines": { + "node": "20 || >=22" + } + }, "node_modules/minimatch": { "version": "10.2.5", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz", "integrity": "sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==", "dev": true, "license": "BlueOak-1.0.0", - "peer": true, "dependencies": { "brace-expansion": "^5.0.5" }, @@ -1485,6 +1511,16 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/minipass": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz", + "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", + "dev": true, + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", @@ -1639,6 +1675,13 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/package-json-from-dist": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", + "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==", + "dev": true, + "license": "BlueOak-1.0.0" + }, "node_modules/path-exists": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", @@ -1661,6 +1704,23 @@ "node": ">=8" } }, + "node_modules/path-scurry": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.2.tgz", + "integrity": "sha512-3O/iVVsJAPsOnpwWIeD+d6z/7PmqApyQePUtCndjatj/9I5LylHvt5qluFaBT3I5h3r1ejfR056c+FCv+NnNXg==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "lru-cache": "^11.0.0", + "minipass": "^7.1.2" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/picomatch": { "version": "4.0.4", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", @@ -1696,6 +1756,26 @@ "node": ">=6" } }, + "node_modules/rimraf": { + "version": "6.1.3", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-6.1.3.tgz", + "integrity": "sha512-LKg+Cr2ZF61fkcaK1UdkH2yEBBKnYjTyWzTJT6KNPcSPaiT7HSdhtMXQuN5wkTX0Xu72KQ1l8S42rlmexS2hSA==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "glob": "^13.0.3", + "package-json-from-dist": "^1.0.1" + }, + "bin": { + "rimraf": "dist/esm/bin.mjs" + }, + "engines": { + "node": "20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/shebang-command": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", diff --git a/package.json b/package.json index f756e65..4a93a01 100644 --- a/package.json +++ b/package.json @@ -17,18 +17,21 @@ "scripts": { "build": "npm run build:rust && npm run build:ts", "build:rust": "napi build --platform --release --cargo-cwd rust rust", - "build:ts": "node ./scripts/generate-browser-profiles.mjs && tsc && node ./scripts/postbuild.mjs", + "build:ts": "rimraf dist && node ./scripts/generate-browser-profiles.mjs && tsc && node ./scripts/postbuild.mjs", "deps:wreq": "node ./scripts/update-wreq-upstream.mjs", "prepare": "node ./scripts/install-git-hooks.mjs", "prepare:publish:main": "node ./scripts/prepare-main-package.mjs", "prepare:publish:platform": "node ./scripts/prepare-platform-package.mjs", "artifacts": "napi artifacts", - "clean": "rm -rf dist rust/target rust/*.node", + "clean": "rimraf dist rust/target && rimraf --glob \"rust/*.node\"", "test": "npm run build && node --test dist/test/node-wreq.spec.js", - "lint": "oxlint --deny-warnings src", - "lint:fix": "oxlint --fix src", - "format": "oxfmt --write \"src/**/*.ts\"", - "format:check": "oxfmt --check \"src/**/*.ts\"" + "lint": "oxlint --deny-warnings src scripts", + "lint:fix": "oxlint --fix src scripts", + "lint:rust": "cargo clippy --manifest-path rust/Cargo.toml --all-targets -- -D warnings", + "format": "oxfmt --write src scripts", + "format:check": "oxfmt --check src scripts", + "format:rust": "cargo fmt --manifest-path rust/Cargo.toml", + "format:rust:check": "cargo fmt --manifest-path rust/Cargo.toml --check" }, "keywords": [ "anti-bot", @@ -44,7 +47,7 @@ "scrapper", "fingerprint", "tls", - "tls-fingerprint", + "tls-fingerprint", "http2", "fetch", "websocket", @@ -72,6 +75,7 @@ "@types/ws": "^8.18.1", "oxfmt": "^0.45.0", "oxlint": "^1.60.0", + "rimraf": "^6.1.3", "typescript": "^5.0.0", "ws": "^8.20.0" }, diff --git a/rust/src/emulation/builders.rs b/rust/src/emulation/builders.rs index 31ec2e3..7f1af5f 100644 --- a/rust/src/emulation/builders.rs +++ b/rust/src/emulation/builders.rs @@ -264,7 +264,7 @@ fn build_settings_order(settings_order: Vec) -> Result { for setting in settings_order { let setting_id = parse_http2_setting_id(&setting)?; - if !seen.insert(setting_id.clone()) { + if !seen.insert(setting_id) { bail!("Duplicate emulation http2Options.settingsOrder entry: {setting}"); } builder = builder.push(setting_id); diff --git a/rust/src/napi/convert.rs b/rust/src/napi/convert.rs index 552f6a4..05f4d2a 100644 --- a/rust/src/napi/convert.rs +++ b/rust/src/napi/convert.rs @@ -5,8 +5,18 @@ use crate::transport::types::{ WebSocketConnectOptions, WebSocketConnection, }; use neon::prelude::*; -use neon::types::JsBuffer; use neon::types::buffer::TypedArray; +use neon::types::JsBuffer; + +fn js_value_to_timeout_ms(cx: &mut FunctionContext, value: Handle) -> NeonResult { + let value = value.downcast::(cx).or_throw(cx)?.value(cx); + + if !value.is_finite() || value < 0.0 { + return cx.throw_type_error("timeout must be a finite non-negative number"); + } + + Ok(if value == 0.0 { 0 } else { value.ceil() as u64 }) +} pub(crate) fn js_value_to_string_array( cx: &mut FunctionContext, @@ -101,12 +111,16 @@ pub(crate) fn js_object_to_request_options( .map(|v| v.value(cx)) .unwrap_or(false); let dns = js_object_to_dns_options(cx, obj)?; - let timeout = obj .get_opt(cx, "timeout")? - .and_then(|v: Handle| v.downcast::(cx).ok()) - .map(|v| v.value(cx) as u64) - .unwrap_or(30000); + .map(|v| js_value_to_timeout_ms(cx, v)) + .transpose()?; + + let timeout = match timeout { + Some(0) => None, + Some(timeout) => Some(timeout), + None => Some(30000), + }; let disable_default_headers = obj .get_opt(cx, "disableDefaultHeaders")? @@ -186,12 +200,16 @@ pub(crate) fn js_object_to_websocket_options( .map(|v| v.value(cx)) .unwrap_or(false); let dns = js_object_to_dns_options(cx, obj)?; - let timeout = obj .get_opt(cx, "timeout")? - .and_then(|v: Handle| v.downcast::(cx).ok()) - .map(|v| v.value(cx) as u64) - .unwrap_or(30000); + .map(|v| js_value_to_timeout_ms(cx, v)) + .transpose()?; + + let timeout = match timeout { + Some(0) => None, + Some(timeout) => Some(timeout), + None => Some(30000), + }; let disable_default_headers = obj .get_opt(cx, "disableDefaultHeaders")? diff --git a/rust/src/napi/request.rs b/rust/src/napi/request.rs index 1447b2a..2f80564 100644 --- a/rust/src/napi/request.rs +++ b/rust/src/napi/request.rs @@ -1,16 +1,29 @@ use crate::napi::convert::{js_object_to_request_options, response_to_js_object}; -use crate::transport::execute_request; +use crate::store::request_store::{ + cancel_request as cancel_request_handle, insert_request, remove_request, +}; +use crate::store::runtime::runtime; +use crate::transport::make_request; use neon::prelude::*; -fn request(mut cx: FunctionContext) -> JsResult { +fn request(mut cx: FunctionContext) -> JsResult { let options_obj = cx.argument::(0)?; let options = js_object_to_request_options(&mut cx, options_obj)?; let channel = cx.channel(); let (deferred, promise) = cx.promise(); + let (cancel_tx, cancel_rx) = tokio::sync::oneshot::channel::<()>(); + let handle = insert_request(cancel_tx); std::thread::spawn(move || { - let result = execute_request(options); + let result = runtime().block_on(async move { + tokio::select! { + result = make_request(options) => result, + _ = cancel_rx => Err(anyhow::anyhow!("Request aborted")), + } + }); + + remove_request(handle); deferred.settle_with(&channel, move |mut cx| match result { Ok(response) => response_to_js_object(&mut cx, response), @@ -18,10 +31,23 @@ fn request(mut cx: FunctionContext) -> JsResult { }); }); - Ok(promise) + let result = JsObject::new(&mut cx); + let handle_value = cx.number(handle as f64); + + result.set(&mut cx, "handle", handle_value)?; + result.set(&mut cx, "promise", promise)?; + + Ok(result) +} + +fn cancel_request(mut cx: FunctionContext) -> JsResult { + let handle = cx.argument::(0)?.value(&mut cx) as u64; + + Ok(cx.boolean(cancel_request_handle(handle))) } pub fn register(cx: &mut ModuleContext) -> NeonResult<()> { cx.export_function("request", request)?; + cx.export_function("cancelRequest", cancel_request)?; Ok(()) } diff --git a/rust/src/store/body_store.rs b/rust/src/store/body_store.rs index e00e62e..c26cb30 100644 --- a/rust/src/store/body_store.rs +++ b/rust/src/store/body_store.rs @@ -3,7 +3,7 @@ use anyhow::{Context, Result}; use std::collections::HashMap; use std::sync::{ atomic::{AtomicU64, Ordering}, - Mutex, OnceLock, + Arc, Mutex, OnceLock, }; #[derive(Debug)] @@ -11,36 +11,54 @@ struct StoredBody { response: wreq::Response, } +type SharedBody = Arc>; + static NEXT_BODY_HANDLE: AtomicU64 = AtomicU64::new(1); -static BODY_STORE: OnceLock>> = OnceLock::new(); +static BODY_STORE: OnceLock>> = OnceLock::new(); -fn body_store() -> &'static Mutex> { +fn body_store() -> &'static Mutex> { BODY_STORE.get_or_init(|| Mutex::new(HashMap::new())) } pub fn store_body(response: wreq::Response) -> u64 { let handle = NEXT_BODY_HANDLE.fetch_add(1, Ordering::Relaxed); - body_store() - .lock() - .expect("body store poisoned") - .insert(handle, StoredBody { response }); + body_store().lock().expect("body store poisoned").insert( + handle, + Arc::new(tokio::sync::Mutex::new(StoredBody { response })), + ); handle } -pub fn read_body_chunk(handle: u64, _size: usize) -> Result<(Vec, bool)> { - let mut store = body_store() +fn get_body(handle: u64) -> Result { + let store = body_store() .lock() .map_err(|_| anyhow::anyhow!("body store poisoned"))?; - let Some(body) = store.get_mut(&handle) else { - return Err(anyhow::anyhow!("Unknown body handle: {}", handle)); - }; - let chunk = runtime() - .block_on(body.response.chunk()) - .context("Failed to read response body chunk")?; + store + .get(&handle) + .cloned() + .ok_or_else(|| anyhow::anyhow!("Unknown body handle: {}", handle)) +} + +fn remove_body(handle: u64) -> Option { + body_store() + .lock() + .expect("body store poisoned") + .remove(&handle) +} + +pub fn read_body_chunk(handle: u64, _size: usize) -> Result<(Vec, bool)> { + let body = get_body(handle)?; + let chunk = runtime().block_on(async { + let mut body = body.lock().await; + body.response + .chunk() + .await + .context("Failed to read response body chunk") + })?; let Some(chunk) = chunk else { - store.remove(&handle); + remove_body(handle); return Ok((Vec::new(), true)); }; @@ -48,18 +66,16 @@ pub fn read_body_chunk(handle: u64, _size: usize) -> Result<(Vec, bool)> { } pub fn read_body_all(handle: u64) -> Result> { - let mut store = body_store() - .lock() - .map_err(|_| anyhow::anyhow!("body store poisoned"))?; - let Some(body) = store.remove(&handle) else { + let Some(body) = remove_body(handle) else { return Err(anyhow::anyhow!("Unknown body handle: {}", handle)); }; - let mut bytes = Vec::new(); - let mut response = body.response; + runtime().block_on(async move { + let mut body = body.lock().await; + let mut bytes = Vec::new(); - runtime().block_on(async { - while let Some(chunk) = response + while let Some(chunk) = body + .response .chunk() .await .context("Failed to read response body chunk")? @@ -67,16 +83,10 @@ pub fn read_body_all(handle: u64) -> Result> { bytes.extend_from_slice(&chunk); } - Ok::<(), anyhow::Error>(()) - })?; - - Ok(bytes) + Ok::, anyhow::Error>(bytes) + }) } pub fn cancel_body(handle: u64) -> bool { - body_store() - .lock() - .expect("body store poisoned") - .remove(&handle) - .is_some() + remove_body(handle).is_some() } diff --git a/rust/src/store/mod.rs b/rust/src/store/mod.rs index b27735a..47456a7 100644 --- a/rust/src/store/mod.rs +++ b/rust/src/store/mod.rs @@ -1,3 +1,4 @@ pub mod body_store; +pub mod request_store; pub mod runtime; pub mod websocket_store; diff --git a/rust/src/store/request_store.rs b/rust/src/store/request_store.rs new file mode 100644 index 0000000..e10f49b --- /dev/null +++ b/rust/src/store/request_store.rs @@ -0,0 +1,42 @@ +use std::collections::HashMap; +use std::sync::{ + atomic::{AtomicU64, Ordering}, + Mutex, OnceLock, +}; + +static NEXT_REQUEST_HANDLE: AtomicU64 = AtomicU64::new(1); +static REQUEST_STORE: OnceLock>>> = + OnceLock::new(); + +fn request_store() -> &'static Mutex>> { + REQUEST_STORE.get_or_init(|| Mutex::new(HashMap::new())) +} + +pub fn insert_request(cancel: tokio::sync::oneshot::Sender<()>) -> u64 { + let handle = NEXT_REQUEST_HANDLE.fetch_add(1, Ordering::Relaxed); + + request_store() + .lock() + .expect("request store poisoned") + .insert(handle, cancel); + + handle +} + +pub fn remove_request(handle: u64) { + request_store() + .lock() + .expect("request store poisoned") + .remove(&handle); +} + +pub fn cancel_request(handle: u64) -> bool { + let cancel = request_store() + .lock() + .expect("request store poisoned") + .remove(&handle); + + cancel + .map(|cancel| cancel.send(()).is_ok()) + .unwrap_or(false) +} diff --git a/rust/src/transport/dns.rs b/rust/src/transport/dns.rs index b84be93..145df8a 100644 --- a/rust/src/transport/dns.rs +++ b/rust/src/transport/dns.rs @@ -1,11 +1,11 @@ use crate::transport::types::DnsOptions; use anyhow::{Context, Result}; use hickory_resolver::{ - TokioResolver, config::{LookupIpStrategy, NameServerConfig, NameServerConfigGroup, ResolverConfig}, lookup_ip::LookupIpIntoIter, name_server::TokioConnectionProvider, proto::xfer::Protocol, + TokioResolver, }; use std::net::{IpAddr, SocketAddr}; use wreq::dns::{Addrs, Name, Resolve, Resolving}; diff --git a/rust/src/transport/mod.rs b/rust/src/transport/mod.rs index dbb932b..09e8395 100644 --- a/rust/src/transport/mod.rs +++ b/rust/src/transport/mod.rs @@ -6,5 +6,5 @@ mod tls; pub mod types; mod websocket; -pub use request::execute_request; +pub use request::make_request; pub use websocket::connect_websocket; diff --git a/rust/src/transport/request.rs b/rust/src/transport/request.rs index a3d73b1..7f06c48 100644 --- a/rust/src/transport/request.rs +++ b/rust/src/transport/request.rs @@ -1,5 +1,4 @@ use crate::store::body_store::store_body; -use crate::store::runtime::runtime; use crate::transport::cookies::parse_cookie_pair; use crate::transport::dns::configure_client_builder as configure_dns; use crate::transport::headers::build_orig_header_map; @@ -10,10 +9,6 @@ use std::collections::HashMap; use std::time::Duration; use wreq::redirect; -pub fn execute_request(options: RequestOptions) -> Result { - runtime().block_on(make_request(options)) -} - pub async fn make_request(options: RequestOptions) -> Result { let RequestOptions { url, @@ -51,11 +46,7 @@ pub async fn make_request(options: RequestOptions) -> Result { .build() .context("Failed to build HTTP client")?; - let method = if method.is_empty() { - "GET" - } else { - &method - }; + let method = if method.is_empty() { "GET" } else { &method }; let mut request = match method.to_uppercase().as_str() { "GET" => client.get(&url), @@ -79,7 +70,9 @@ pub async fn make_request(options: RequestOptions) -> Result { request = request.body(body); } - request = request.timeout(Duration::from_millis(timeout)); + if let Some(timeout) = timeout { + request = request.timeout(Duration::from_millis(timeout)); + } request = request.redirect(redirect::Policy::none()); request = request.default_headers(!disable_default_headers); request = request.gzip(compress); diff --git a/rust/src/transport/tls.rs b/rust/src/transport/tls.rs index 3dcad38..288a2b5 100644 --- a/rust/src/transport/tls.rs +++ b/rust/src/transport/tls.rs @@ -1,8 +1,8 @@ use crate::transport::types::{CertificateAuthorityOptions, TlsIdentityOptions}; use anyhow::{Context, Result}; use wreq::{ - ClientBuilder, tls::{CertStore, Identity}, + ClientBuilder, }; pub fn configure_client_builder( diff --git a/rust/src/transport/types.rs b/rust/src/transport/types.rs index 8687d52..4914b40 100644 --- a/rust/src/transport/types.rs +++ b/rust/src/transport/types.rs @@ -3,7 +3,10 @@ use wreq::Emulation; #[derive(Debug, Clone)] pub enum TlsIdentityOptions { - Pem { cert: Vec, key: Vec }, + Pem { + cert: Vec, + key: Vec, + }, Pfx { archive: Vec, passphrase: Option, @@ -33,7 +36,7 @@ pub struct RequestOptions { pub proxy: Option, pub disable_system_proxy: bool, pub dns: Option, - pub timeout: u64, + pub timeout: Option, pub disable_default_headers: bool, pub compress: bool, pub tls_identity: Option, @@ -59,7 +62,7 @@ pub struct WebSocketConnectOptions { pub proxy: Option, pub disable_system_proxy: bool, pub dns: Option, - pub timeout: u64, + pub timeout: Option, pub disable_default_headers: bool, pub protocols: Vec, pub tls_identity: Option, diff --git a/rust/src/transport/websocket.rs b/rust/src/transport/websocket.rs index 075de33..52443c3 100644 --- a/rust/src/transport/websocket.rs +++ b/rust/src/transport/websocket.rs @@ -150,8 +150,11 @@ async fn make_websocket(options: WebSocketConnectOptions) -> Result [packageName, version]), - ); + return Object.fromEntries(platformTargets.map(({ packageName }) => [packageName, version])); } diff --git a/scripts/postbuild.mjs b/scripts/postbuild.mjs index f8f094a..a8bf75b 100644 --- a/scripts/postbuild.mjs +++ b/scripts/postbuild.mjs @@ -1,47 +1,43 @@ -import { mkdir, writeFile } from "node:fs/promises"; -import { dirname, resolve } from "node:path"; -import { fileURLToPath } from "node:url"; +import { mkdir, writeFile } from 'node:fs/promises'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; const __dirname = dirname(fileURLToPath(import.meta.url)); -const distDir = resolve(__dirname, "../dist"); +const distDir = resolve(__dirname, '../dist'); const runtimeExports = [ - "fetch", - "createClient", - "getProfiles", - "BROWSER_PROFILES", - "Headers", - "Response", - "RequestError", - "HTTPError", - "TimeoutError", - "AbortError", - "WebSocket", - "CloseEvent", - "websocket", - "WebSocketError", + 'fetch', + 'createClient', + 'getProfiles', + 'BROWSER_PROFILES', + 'Headers', + 'Response', + 'RequestError', + 'HTTPError', + 'TimeoutError', + 'AbortError', + 'WebSocket', + 'CloseEvent', + 'websocket', + 'WebSocketError', ]; const esmLines = [ "import nodeWreq from './node-wreq.js';", - "", + '', ...runtimeExports.map((name) => `export const ${name} = nodeWreq.${name};`), - "", - "export default nodeWreq;", - "", + '', + 'export default nodeWreq;', + '', ]; const typeLines = [ "export * from './node-wreq';", "import nodeWreq from './node-wreq';", - "export default nodeWreq;", - "", + 'export default nodeWreq;', + '', ]; await mkdir(distDir, { recursive: true }); -await writeFile(resolve(distDir, "node-wreq.mjs"), esmLines.join("\n"), "utf8"); -await writeFile( - resolve(distDir, "node-wreq.d.mts"), - typeLines.join("\n"), - "utf8", -); +await writeFile(resolve(distDir, 'node-wreq.mjs'), esmLines.join('\n'), 'utf8'); +await writeFile(resolve(distDir, 'node-wreq.d.mts'), typeLines.join('\n'), 'utf8'); diff --git a/scripts/prepare-main-package.mjs b/scripts/prepare-main-package.mjs index bd8e258..46c5460 100644 --- a/scripts/prepare-main-package.mjs +++ b/scripts/prepare-main-package.mjs @@ -1,33 +1,31 @@ -import { cp, mkdir, readFile, rm, writeFile } from "node:fs/promises"; -import { dirname, resolve } from "node:path"; -import { fileURLToPath } from "node:url"; -import { getOptionalDependencyMap } from "./platform-targets.mjs"; -import { resolvePublishVersion } from "./publish-version.mjs"; +import { cp, mkdir, readFile, rm, writeFile } from 'node:fs/promises'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { getOptionalDependencyMap } from './platform-targets.mjs'; +import { resolvePublishVersion } from './publish-version.mjs'; const __dirname = dirname(fileURLToPath(import.meta.url)); -const repoRoot = resolve(__dirname, ".."); -const outDir = resolve(repoRoot, process.argv[2] ?? ".release/main-package"); +const repoRoot = resolve(__dirname, '..'); +const outDir = resolve(repoRoot, process.argv[2] ?? '.release/main-package'); -const rootPackage = JSON.parse( - await readFile(resolve(repoRoot, "package.json"), "utf8"), -); +const rootPackage = JSON.parse(await readFile(resolve(repoRoot, 'package.json'), 'utf8')); const publishVersion = resolvePublishVersion(rootPackage); await rm(outDir, { recursive: true, force: true }); await mkdir(outDir, { recursive: true }); -await cp(resolve(repoRoot, "dist"), resolve(outDir, "dist"), { +await cp(resolve(repoRoot, 'dist'), resolve(outDir, 'dist'), { recursive: true, }); -await rm(resolve(outDir, "dist/test"), { recursive: true, force: true }); +await rm(resolve(outDir, 'dist/test'), { recursive: true, force: true }); -await cp(resolve(repoRoot, "docs"), resolve(outDir, "docs"), { +await cp(resolve(repoRoot, 'docs'), resolve(outDir, 'docs'), { recursive: true, }); -await cp(resolve(repoRoot, "README.md"), resolve(outDir, "README.md")); +await cp(resolve(repoRoot, 'README.md'), resolve(outDir, 'README.md')); const publishPackage = { name: rootPackage.name, @@ -47,11 +45,11 @@ const publishPackage = { os: rootPackage.os, cpu: rootPackage.cpu, optionalDependencies: getOptionalDependencyMap(publishVersion), - files: ["dist", "docs", "README.md"], + files: ['dist', 'docs', 'README.md'], }; await writeFile( - resolve(outDir, "package.json"), + resolve(outDir, 'package.json'), `${JSON.stringify(publishPackage, null, 2)}\n`, - "utf8", + 'utf8' ); diff --git a/scripts/prepare-platform-package.mjs b/scripts/prepare-platform-package.mjs index dc91b59..3d7c5f6 100644 --- a/scripts/prepare-platform-package.mjs +++ b/scripts/prepare-platform-package.mjs @@ -1,11 +1,11 @@ -import { cp, mkdir, readFile, rm, writeFile } from "node:fs/promises"; -import { basename, dirname, resolve } from "node:path"; -import { fileURLToPath } from "node:url"; -import { getPlatformTargetByTriple } from "./platform-targets.mjs"; -import { resolvePublishVersion } from "./publish-version.mjs"; +import { cp, mkdir, readFile, rm, writeFile } from 'node:fs/promises'; +import { basename, dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { getPlatformTargetByTriple } from './platform-targets.mjs'; +import { resolvePublishVersion } from './publish-version.mjs'; const __dirname = dirname(fileURLToPath(import.meta.url)); -const repoRoot = resolve(__dirname, ".."); +const repoRoot = resolve(__dirname, '..'); function parseArgs(argv) { const args = {}; @@ -14,7 +14,7 @@ function parseArgs(argv) { const key = argv[index]; const value = argv[index + 1]; - if (!key.startsWith("--")) { + if (!key.startsWith('--')) { continue; } @@ -33,16 +33,14 @@ if (!target) { } if (!args.binary) { - throw new Error("Missing required --binary argument"); + throw new Error('Missing required --binary argument'); } if (!args.outDir) { - throw new Error("Missing required --outDir argument"); + throw new Error('Missing required --outDir argument'); } -const rootPackage = JSON.parse( - await readFile(resolve(repoRoot, "package.json"), "utf8"), -); +const rootPackage = JSON.parse(await readFile(resolve(repoRoot, 'package.json'), 'utf8')); const publishVersion = resolvePublishVersion(rootPackage); const outDir = resolve(repoRoot, args.outDir); const binarySource = resolve(repoRoot, args.binary); @@ -64,15 +62,15 @@ const packageJson = { cpu: target.cpu, ...(target.libc ? { libc: target.libc } : {}), main: `./${target.binaryName}`, - files: [target.binaryName, "README.md"], + files: [target.binaryName, 'README.md'], publishConfig: { - access: "public", + access: 'public', }, }; const mainPackageUrl = - typeof rootPackage.homepage === "string" - ? rootPackage.homepage.replace(/#readme$/, "") + typeof rootPackage.homepage === 'string' + ? rootPackage.homepage.replace(/#readme$/, '') : `https://www.npmjs.com/package/${rootPackage.name}`; const readme = `# ${target.packageName} @@ -92,9 +90,9 @@ Binary: \`${basename(target.binaryName)}\` `; await writeFile( - resolve(outDir, "package.json"), + resolve(outDir, 'package.json'), `${JSON.stringify(packageJson, null, 2)}\n`, - "utf8", + 'utf8' ); -await writeFile(resolve(outDir, "README.md"), readme, "utf8"); +await writeFile(resolve(outDir, 'README.md'), readme, 'utf8'); diff --git a/scripts/update-wreq-upstream.mjs b/scripts/update-wreq-upstream.mjs index d369617..4a4c92e 100644 --- a/scripts/update-wreq-upstream.mjs +++ b/scripts/update-wreq-upstream.mjs @@ -24,7 +24,9 @@ async function fetchLatestVersion(crate) { }); if (!response.ok) { - throw new Error(`Failed to fetch ${crate} metadata from crates.io: ${response.status} ${response.statusText}`); + throw new Error( + `Failed to fetch ${crate} metadata from crates.io: ${response.status} ${response.statusText}` + ); } const payload = await response.json(); @@ -100,7 +102,9 @@ async function main() { await writeFile(manifestPath, manifest); for (const update of plannedUpdates) { - console.log(`Updating ${update.dependency}: ${update.currentVersion} -> ${update.latestVersion}`); + console.log( + `Updating ${update.dependency}: ${update.currentVersion} -> ${update.latestVersion}` + ); updateLockfile(update.dependency, update.latestVersion); } } diff --git a/src/http/fetch.ts b/src/http/fetch.ts index 59f1d4c..7585891 100644 --- a/src/http/fetch.ts +++ b/src/http/fetch.ts @@ -61,7 +61,11 @@ export async function fetch(input: RequestInput, init?: WreqInit) { let response = shortCircuit ?? - (await dispatchNativeRequest(await buildNativeRequest(request, options), startTime)); + (await dispatchNativeRequest( + await buildNativeRequest(request, options), + startTime, + options.signal + )); if (shortCircuit) { response.setTimings({ diff --git a/src/http/pipeline/dispatch.ts b/src/http/pipeline/dispatch.ts index 8945989..e09e945 100644 --- a/src/http/pipeline/dispatch.ts +++ b/src/http/pipeline/dispatch.ts @@ -1,4 +1,4 @@ -import { RequestError, TimeoutError } from '../../errors'; +import { AbortError, RequestError, TimeoutError } from '../../errors'; import { nativeRequest } from '../../native'; import type { NativeRequestOptions, RequestStats, WreqInit } from '../../types'; import { Response } from '../response'; @@ -16,9 +16,14 @@ export async function reportStats( export async function dispatchNativeRequest( options: NativeRequestOptions, - startTime: number + startTime: number, + signal?: AbortSignal | null ): Promise { - const nativeResponse = await nativeRequest(options).catch((error: unknown) => { + const nativeResponse = await nativeRequest(options, signal).catch((error: unknown) => { + if (error instanceof AbortError) { + throw error; + } + const message = String(error); const lowered = message.toLowerCase(); @@ -32,7 +37,14 @@ export async function dispatchNativeRequest( const responseStart = Date.now(); return new Response({ - ...nativeResponse, + status: nativeResponse.status, + statusText: nativeResponse.statusText, + headers: nativeResponse.headers, + body: nativeResponse.body, + bodyHandle: nativeResponse.bodyHandle, + cookies: nativeResponse.cookies, + setCookies: nativeResponse.setCookies, + url: nativeResponse.url, timings: { startTime, responseStart, diff --git a/src/http/pipeline/options.ts b/src/http/pipeline/options.ts index dc9ead6..c4bd23c 100644 --- a/src/http/pipeline/options.ts +++ b/src/http/pipeline/options.ts @@ -59,6 +59,18 @@ export function resolveRetryOptions(retry?: WreqInit['retry']): ResolvedRetryOpt }; } +function resolveNativeTimeout(timeout: number | undefined): Pick { + if (timeout === undefined) { + return {}; + } + + if (!Number.isFinite(timeout) || timeout < 0) { + throw new TypeError('timeout must be a finite non-negative number'); + } + + return { timeout: timeout === 0 ? 0 : Math.max(1, Math.ceil(timeout)) }; +} + export function resolveOptions(init: WreqInit): ResolvedOptions { return { ...init, @@ -89,6 +101,7 @@ export async function buildNativeRequest( ): Promise { const { proxy, disableSystemProxy } = normalizeProxyOptions(options.proxy); const body = await request._getBodyBytesForDispatch(); + const timeout = resolveNativeTimeout(options.timeout); return { url: request.url, @@ -101,7 +114,7 @@ export async function buildNativeRequest( proxy, disableSystemProxy, dns: normalizeDnsOptions(options.dns), - timeout: options.timeout, + ...timeout, disableDefaultHeaders: options.disableDefaultHeaders, compress: options.compress, tlsIdentity: normalizeTlsIdentity(options.tlsIdentity), diff --git a/src/http/request.ts b/src/http/request.ts index 87b0b89..d75d0b5 100644 --- a/src/http/request.ts +++ b/src/http/request.ts @@ -84,6 +84,7 @@ export class Request { }); cloned.#bodyBytes = cloneBytes(this.#bodyBytes); + cloned.#multipartBody = this.#multipartBody?.clone() ?? null; return cloned; } diff --git a/src/native.ts b/src/native.ts new file mode 100644 index 0000000..2f80256 --- /dev/null +++ b/src/native.ts @@ -0,0 +1,219 @@ +import { AbortError } from './errors'; +import type { + BrowserProfile, + HttpMethod, + NativeResponse, + NativeRequestOptions, + NativeWebSocketConnectOptions, + NativeWebSocketConnection, + NativeWebSocketReadResult, +} from './types'; + +type NativeBinding = { + request: (options: NativeRequestOptions) => { + handle: number; + promise: Promise; + }; + cancelRequest: (handle: number) => boolean; + websocketConnect: (options: NativeWebSocketConnectOptions) => Promise; + websocketRead: (handle: number) => Promise; + websocketSendText: (handle: number, text: string) => Promise; + websocketSendBinary: (handle: number, data: Buffer) => Promise; + websocketClose: (handle: number, code?: number, reason?: string) => Promise; + readBodyChunk: ( + handle: number, + size?: number + ) => Promise<{ + chunk: Buffer; + done: boolean; + }>; + readBodyAll: (handle: number) => Promise; + cancelBody: (handle: number) => boolean; + getProfiles: () => string[]; +}; + +let nativeBinding: NativeBinding | undefined; + +function loadNativeBinding(): NativeBinding { + const platform = process.platform; + const arch = process.arch; + + const platformArchMap: Record> = { + darwin: { + x64: 'darwin-x64', + arm64: 'darwin-arm64', + }, + linux: { + x64: 'linux-x64-gnu', + }, + win32: { + x64: 'win32-x64-msvc', + }, + }; + + const platformArch = platformArchMap[platform]?.[arch]; + + if (!platformArch) { + throw new Error( + `Unsupported platform: ${platform}-${arch}. ` + + `Supported platforms: darwin-x64, darwin-arm64, linux-x64, win32-x64` + ); + } + + const binaryName = `node-wreq.${platformArch}.node`; + + try { + return require(`../rust/${binaryName}`) as NativeBinding; + } catch { + try { + return require('../rust/node-wreq.node') as NativeBinding; + } catch { + throw new Error( + `Failed to load native module for ${platform}-${arch}. ` + + `Tried: ../rust/${binaryName} and ../rust/node-wreq.node. ` + + `Make sure the package is installed correctly and the native module is built for your platform.` + ); + } + } +} + +function getBinding(): NativeBinding { + nativeBinding ??= loadNativeBinding(); + + return nativeBinding; +} + +let cachedProfiles: BrowserProfile[] | undefined; + +export function getProfiles(): BrowserProfile[] { + cachedProfiles ??= getBinding().getProfiles() as BrowserProfile[]; + + return cachedProfiles; +} + +export async function nativeRequest( + options: NativeRequestOptions, + signal?: AbortSignal | null +): Promise { + if (signal?.aborted) { + throw new AbortError(undefined, { cause: signal.reason }); + } + + const task = getBinding().request(options); + + if (!signal) { + return task.promise; + } + + return new Promise((resolve, reject) => { + let settled = false; + + const cleanup = () => { + signal.removeEventListener('abort', onAbort); + }; + + const onAbort = () => { + if (settled) { + return; + } + + settled = true; + cleanup(); + getBinding().cancelRequest(task.handle); + reject(new AbortError(undefined, { cause: signal.reason })); + }; + + signal.addEventListener('abort', onAbort, { once: true }); + + task.promise.then( + (response) => { + if (settled) { + return; + } + + settled = true; + cleanup(); + resolve(response); + }, + (error) => { + if (settled) { + return; + } + + settled = true; + cleanup(); + reject(error); + } + ); + }); +} + +export async function nativeReadBodyChunk( + handle: number, + size?: number +): Promise<{ + chunk: Uint8Array; + done: boolean; +}> { + return getBinding().readBodyChunk(handle, size); +} + +export async function nativeReadBodyAll(handle: number): Promise { + return getBinding().readBodyAll(handle); +} + +export function nativeCancelBody(handle: number): boolean { + return getBinding().cancelBody(handle); +} + +export async function nativeWebSocketConnect( + options: NativeWebSocketConnectOptions +): Promise { + return getBinding().websocketConnect(options); +} + +export async function nativeWebSocketRead(handle: number): Promise { + return getBinding().websocketRead(handle); +} + +export async function nativeWebSocketSendText(handle: number, text: string): Promise { + return getBinding().websocketSendText(handle, text); +} + +export async function nativeWebSocketSendBinary(handle: number, data: Uint8Array): Promise { + return getBinding().websocketSendBinary(handle, Buffer.from(data)); +} + +export async function nativeWebSocketClose( + handle: number, + code?: number, + reason?: string +): Promise { + return getBinding().websocketClose(handle, code, reason); +} + +export function validateBrowserProfile(browser?: BrowserProfile): void { + if (!browser) { + return; + } + + if (!getProfiles().includes(browser)) { + throw new Error(`Invalid browser profile: ${browser}`); + } +} + +export function normalizeMethod(method?: string): HttpMethod { + const normalized = (method ?? 'GET').toUpperCase(); + + switch (normalized) { + case 'GET': + case 'POST': + case 'PUT': + case 'DELETE': + case 'PATCH': + case 'HEAD': + return normalized; + default: + throw new Error(`Unsupported HTTP method: ${method}`); + } +} diff --git a/src/native/binding.ts b/src/native/binding.ts index e213c85..ebf1081 100644 --- a/src/native/binding.ts +++ b/src/native/binding.ts @@ -9,7 +9,11 @@ import type { } from '../types'; export type NativeBinding = { - request: (options: NativeRequestOptions) => Promise; + request: (options: NativeRequestOptions) => { + handle: number; + promise: Promise; + }; + cancelRequest: (handle: number) => boolean; websocketConnect: (options: NativeWebSocketConnectOptions) => Promise; websocketRead: (handle: number) => Promise; websocketSendText: (handle: number, text: string) => Promise; diff --git a/src/native/request.ts b/src/native/request.ts index b0c4b4d..572dcd6 100644 --- a/src/native/request.ts +++ b/src/native/request.ts @@ -1,8 +1,62 @@ +import { AbortError } from '../errors'; import type { NativeRequestOptions, NativeResponse } from '../types'; import { getBinding } from './binding'; -export async function nativeRequest(options: NativeRequestOptions): Promise { - return getBinding().request(options); +export async function nativeRequest( + options: NativeRequestOptions, + signal?: AbortSignal | null +): Promise { + if (signal?.aborted) { + throw new AbortError(undefined, { cause: signal.reason }); + } + + const task = getBinding().request(options); + + if (!signal) { + return task.promise; + } + + return new Promise((resolve, reject) => { + let settled = false; + + const cleanup = () => { + signal.removeEventListener('abort', onAbort); + }; + + const onAbort = () => { + if (settled) { + return; + } + + settled = true; + cleanup(); + getBinding().cancelRequest(task.handle); + reject(new AbortError(undefined, { cause: signal.reason })); + }; + + signal.addEventListener('abort', onAbort, { once: true }); + + task.promise.then( + (response) => { + if (settled) { + return; + } + + settled = true; + cleanup(); + resolve(response); + }, + (error) => { + if (settled) { + return; + } + + settled = true; + cleanup(); + reject(error); + } + ); + }); } export async function nativeReadBodyChunk( diff --git a/src/test/helpers/local-server.ts b/src/test/helpers/local-server.ts index 2fe355f..08c0894 100644 --- a/src/test/helpers/local-server.ts +++ b/src/test/helpers/local-server.ts @@ -113,9 +113,11 @@ export function setupLocalTestServer() { } if (url.pathname === '/timings/delay') { + const delayMs = Number(url.searchParams.get('ms') ?? '50'); + setTimeout(() => { sendJson(response, 200, { delayed: true }); - }, 50); + }, delayMs); return; } diff --git a/src/test/http-client.spec.ts b/src/test/http-client.spec.ts index a2d9ad3..a140c9c 100644 --- a/src/test/http-client.spec.ts +++ b/src/test/http-client.spec.ts @@ -66,6 +66,75 @@ describe('http client', () => { ); }); + test('should disable request timeout when timeout is set to 0', async () => { + const response = await fetch(`${getBaseUrl()}/timings/delay`, { + browser: 'chrome_137', + timeout: 0, + }); + + assert.strictEqual(response.status, 200); + assert.deepStrictEqual(await response.json(), { delayed: true }); + }); + + test('should abort requests after dispatch has started', async () => { + const controller = new AbortController(); + const responsePromise = fetch(`${getBaseUrl()}/timings/delay?ms=250`, { + browser: 'chrome_137', + timeout: 0, + signal: controller.signal, + }); + + setTimeout(() => { + controller.abort(new Error('stop')); + }, 20); + + await assert.rejects(responsePromise, { + name: 'AbortError', + code: 'ERR_ABORTED', + }); + }); + + test('should reject invalid timeout values', async () => { + await assert.rejects( + async () => { + await fetch(`${getBaseUrl()}/headers/raw`, { + browser: 'chrome_137', + timeout: Number.NaN, + }); + }, + (error: unknown) => { + assert.ok(error instanceof Error); + assert.strictEqual(error.name, 'RequestError'); + const cause = (error as { cause?: unknown }).cause; + + assert.ok( + cause instanceof TypeError, + 'invalid timeout should be surfaced with the original TypeError as cause' + ); + + return true; + } + ); + + await assert.rejects( + async () => { + await fetch(`${getBaseUrl()}/headers/raw`, { + browser: 'chrome_137', + timeout: -1, + }); + }, + (error: unknown) => { + assert.ok(error instanceof Error); + assert.strictEqual(error.name, 'RequestError'); + const cause = (error as { cause?: unknown }).cause; + + assert.ok(cause instanceof TypeError); + + return true; + } + ); + }); + test('should support fetch-style requests', async () => { const response = await fetch('https://httpbin.org/get', { browser: 'chrome_137', diff --git a/src/test/transport-features.spec.ts b/src/test/transport-features.spec.ts index 0c197bf..a9284d0 100644 --- a/src/test/transport-features.spec.ts +++ b/src/test/transport-features.spec.ts @@ -1,7 +1,7 @@ import assert from 'node:assert'; import { Buffer } from 'node:buffer'; import { describe, test } from 'node:test'; -import { fetch } from '../node-wreq'; +import { fetch, Request } from '../node-wreq'; import { setupLocalTestServer } from './helpers/local-server'; import { setupProxyTestServer } from './helpers/proxy-server'; @@ -42,6 +42,29 @@ describe('transport features', () => { ); }); + test('should preserve multipart request bodies when cloning requests', async () => { + const formData = new FormData(); + + formData.append('alpha', '1'); + formData.append( + 'upload', + new File([Buffer.from('hello multipart')], 'hello.txt', { type: 'text/plain' }) + ); + + const request = new Request(`${getBaseUrl()}/body/echo`, { + method: 'POST', + body: formData, + }); + + const cloned = request.clone(); + const response = await fetch(cloned); + const body = await response.json<{ body: string }>(); + + assert.ok(body.body.includes('name="alpha"')); + assert.ok(body.body.includes('name="upload"')); + assert.ok(body.body.includes('filename="hello.txt"')); + }); + test('should decode response.text() using the declared charset', async () => { const response = await fetch(`${getBaseUrl()}/charset/windows-1251`); diff --git a/src/websocket/index.ts b/src/websocket/index.ts index 113419c..05e272a 100644 --- a/src/websocket/index.ts +++ b/src/websocket/index.ts @@ -24,6 +24,20 @@ import { const DEFAULT_TIMEOUT = 30_000; +function resolveNativeTimeout( + timeout: number | undefined +): Pick { + if (timeout === undefined) { + return { timeout: DEFAULT_TIMEOUT }; + } + + if (!Number.isFinite(timeout) || timeout < 0) { + throw new TypeError('timeout must be a finite non-negative number'); + } + + return { timeout: timeout === 0 ? 0 : Math.max(1, Math.ceil(timeout)) }; +} + type OpenHandler = ((event: Event) => void) | null; type MessageHandler = ((event: MessageEvent) => void) | null; type CloseHandler = ((event: CloseEvent) => void) | null; @@ -232,7 +246,7 @@ export class WebSocket extends EventTarget { proxy, disableSystemProxy, dns: normalizeDnsOptions(init.dns), - timeout: init.timeout ?? DEFAULT_TIMEOUT, + ...resolveNativeTimeout(init.timeout), disableDefaultHeaders: init.disableDefaultHeaders ?? false, tlsIdentity: normalizeTlsIdentity(init.tlsIdentity), ca: normalizeCertificateAuthority(init.ca), From 166b231c82f0fdc9539555bfefd063145204b13e Mon Sep 17 00:00:00 2001 From: ruby Date: Thu, 16 Apr 2026 00:04:53 +0400 Subject: [PATCH 2/3] refactor: move native runtime bindings into src/native --- src/http/fetch.ts | 2 +- src/http/pipeline/dispatch.ts | 2 +- src/http/pipeline/options.ts | 2 +- src/http/pipeline/redirects.ts | 2 +- src/http/pipeline/retries.ts | 2 +- src/http/response.ts | 2 +- src/index.ts | 2 +- src/native.ts | 219 --------------------------------- src/websocket/index.ts | 2 +- 9 files changed, 8 insertions(+), 227 deletions(-) delete mode 100644 src/native.ts diff --git a/src/http/fetch.ts b/src/http/fetch.ts index 7585891..9f301ab 100644 --- a/src/http/fetch.ts +++ b/src/http/fetch.ts @@ -7,7 +7,7 @@ import { runBeforeRetryHooks, runInitHooks, } from '../hooks'; -import { normalizeMethod } from '../native'; +import { normalizeMethod } from '../native/index'; import type { RedirectEntry, RequestInput, RetryDecisionContext, WreqInit } from '../types'; import { loadCookiesIntoRequest, persistResponseCookies } from './pipeline/cookies'; import { dispatchNativeRequest, reportStats } from './pipeline/dispatch'; diff --git a/src/http/pipeline/dispatch.ts b/src/http/pipeline/dispatch.ts index e09e945..7cc3e28 100644 --- a/src/http/pipeline/dispatch.ts +++ b/src/http/pipeline/dispatch.ts @@ -1,5 +1,5 @@ import { AbortError, RequestError, TimeoutError } from '../../errors'; -import { nativeRequest } from '../../native'; +import { nativeRequest } from '../../native/index'; import type { NativeRequestOptions, RequestStats, WreqInit } from '../../types'; import { Response } from '../response'; diff --git a/src/http/pipeline/options.ts b/src/http/pipeline/options.ts index c4bd23c..b26a0ca 100644 --- a/src/http/pipeline/options.ts +++ b/src/http/pipeline/options.ts @@ -3,7 +3,7 @@ import { serializeEmulationOptions } from '../../config/emulation'; import { normalizeDnsOptions, normalizeProxyOptions } from '../../config/network'; import { normalizeCertificateAuthority, normalizeTlsIdentity } from '../../config/tls'; import { Headers } from '../../headers'; -import { normalizeMethod, validateBrowserProfile } from '../../native'; +import { normalizeMethod, validateBrowserProfile } from '../../native/index'; import type { NativeRequestOptions, ResolvedOptions, diff --git a/src/http/pipeline/redirects.ts b/src/http/pipeline/redirects.ts index 12bc939..321bdfb 100644 --- a/src/http/pipeline/redirects.ts +++ b/src/http/pipeline/redirects.ts @@ -1,6 +1,6 @@ import { RequestError } from '../../errors'; import { Headers } from '../../headers'; -import { normalizeMethod } from '../../native'; +import { normalizeMethod } from '../../native/index'; import type { BodyInit, HttpMethod, RedirectEntry } from '../../types'; import { Response } from '../response'; diff --git a/src/http/pipeline/retries.ts b/src/http/pipeline/retries.ts index feb9c2b..17b0052 100644 --- a/src/http/pipeline/retries.ts +++ b/src/http/pipeline/retries.ts @@ -1,4 +1,4 @@ -import { normalizeMethod } from '../../native'; +import { normalizeMethod } from '../../native/index'; import type { ResolvedRetryOptions, RetryDecisionContext } from '../../types'; import { inferErrorCode } from './errors'; diff --git a/src/http/response.ts b/src/http/response.ts index 70d9088..fc6a60b 100644 --- a/src/http/response.ts +++ b/src/http/response.ts @@ -3,7 +3,7 @@ import { STATUS_CODES } from 'node:http'; import { ReadableStream } from 'node:stream/web'; import { TextDecoder } from 'node:util'; import { Headers } from '../headers'; -import { nativeCancelBody, nativeReadBodyChunk } from '../native'; +import { nativeCancelBody, nativeReadBodyChunk } from '../native/index'; import type { BodyInit, HeadersInit, diff --git a/src/index.ts b/src/index.ts index f47e21a..ce0baff 100644 --- a/src/index.ts +++ b/src/index.ts @@ -5,7 +5,7 @@ import { Headers } from './headers'; import { fetch } from './http/fetch'; import { Request } from './http/request'; import { Response } from './http/response'; -import { getProfiles } from './native'; +import { getProfiles } from './native/index'; import type { AfterResponseContext, AlpnProtocol, diff --git a/src/native.ts b/src/native.ts deleted file mode 100644 index 2f80256..0000000 --- a/src/native.ts +++ /dev/null @@ -1,219 +0,0 @@ -import { AbortError } from './errors'; -import type { - BrowserProfile, - HttpMethod, - NativeResponse, - NativeRequestOptions, - NativeWebSocketConnectOptions, - NativeWebSocketConnection, - NativeWebSocketReadResult, -} from './types'; - -type NativeBinding = { - request: (options: NativeRequestOptions) => { - handle: number; - promise: Promise; - }; - cancelRequest: (handle: number) => boolean; - websocketConnect: (options: NativeWebSocketConnectOptions) => Promise; - websocketRead: (handle: number) => Promise; - websocketSendText: (handle: number, text: string) => Promise; - websocketSendBinary: (handle: number, data: Buffer) => Promise; - websocketClose: (handle: number, code?: number, reason?: string) => Promise; - readBodyChunk: ( - handle: number, - size?: number - ) => Promise<{ - chunk: Buffer; - done: boolean; - }>; - readBodyAll: (handle: number) => Promise; - cancelBody: (handle: number) => boolean; - getProfiles: () => string[]; -}; - -let nativeBinding: NativeBinding | undefined; - -function loadNativeBinding(): NativeBinding { - const platform = process.platform; - const arch = process.arch; - - const platformArchMap: Record> = { - darwin: { - x64: 'darwin-x64', - arm64: 'darwin-arm64', - }, - linux: { - x64: 'linux-x64-gnu', - }, - win32: { - x64: 'win32-x64-msvc', - }, - }; - - const platformArch = platformArchMap[platform]?.[arch]; - - if (!platformArch) { - throw new Error( - `Unsupported platform: ${platform}-${arch}. ` + - `Supported platforms: darwin-x64, darwin-arm64, linux-x64, win32-x64` - ); - } - - const binaryName = `node-wreq.${platformArch}.node`; - - try { - return require(`../rust/${binaryName}`) as NativeBinding; - } catch { - try { - return require('../rust/node-wreq.node') as NativeBinding; - } catch { - throw new Error( - `Failed to load native module for ${platform}-${arch}. ` + - `Tried: ../rust/${binaryName} and ../rust/node-wreq.node. ` + - `Make sure the package is installed correctly and the native module is built for your platform.` - ); - } - } -} - -function getBinding(): NativeBinding { - nativeBinding ??= loadNativeBinding(); - - return nativeBinding; -} - -let cachedProfiles: BrowserProfile[] | undefined; - -export function getProfiles(): BrowserProfile[] { - cachedProfiles ??= getBinding().getProfiles() as BrowserProfile[]; - - return cachedProfiles; -} - -export async function nativeRequest( - options: NativeRequestOptions, - signal?: AbortSignal | null -): Promise { - if (signal?.aborted) { - throw new AbortError(undefined, { cause: signal.reason }); - } - - const task = getBinding().request(options); - - if (!signal) { - return task.promise; - } - - return new Promise((resolve, reject) => { - let settled = false; - - const cleanup = () => { - signal.removeEventListener('abort', onAbort); - }; - - const onAbort = () => { - if (settled) { - return; - } - - settled = true; - cleanup(); - getBinding().cancelRequest(task.handle); - reject(new AbortError(undefined, { cause: signal.reason })); - }; - - signal.addEventListener('abort', onAbort, { once: true }); - - task.promise.then( - (response) => { - if (settled) { - return; - } - - settled = true; - cleanup(); - resolve(response); - }, - (error) => { - if (settled) { - return; - } - - settled = true; - cleanup(); - reject(error); - } - ); - }); -} - -export async function nativeReadBodyChunk( - handle: number, - size?: number -): Promise<{ - chunk: Uint8Array; - done: boolean; -}> { - return getBinding().readBodyChunk(handle, size); -} - -export async function nativeReadBodyAll(handle: number): Promise { - return getBinding().readBodyAll(handle); -} - -export function nativeCancelBody(handle: number): boolean { - return getBinding().cancelBody(handle); -} - -export async function nativeWebSocketConnect( - options: NativeWebSocketConnectOptions -): Promise { - return getBinding().websocketConnect(options); -} - -export async function nativeWebSocketRead(handle: number): Promise { - return getBinding().websocketRead(handle); -} - -export async function nativeWebSocketSendText(handle: number, text: string): Promise { - return getBinding().websocketSendText(handle, text); -} - -export async function nativeWebSocketSendBinary(handle: number, data: Uint8Array): Promise { - return getBinding().websocketSendBinary(handle, Buffer.from(data)); -} - -export async function nativeWebSocketClose( - handle: number, - code?: number, - reason?: string -): Promise { - return getBinding().websocketClose(handle, code, reason); -} - -export function validateBrowserProfile(browser?: BrowserProfile): void { - if (!browser) { - return; - } - - if (!getProfiles().includes(browser)) { - throw new Error(`Invalid browser profile: ${browser}`); - } -} - -export function normalizeMethod(method?: string): HttpMethod { - const normalized = (method ?? 'GET').toUpperCase(); - - switch (normalized) { - case 'GET': - case 'POST': - case 'PUT': - case 'DELETE': - case 'PATCH': - case 'HEAD': - return normalized; - default: - throw new Error(`Unsupported HTTP method: ${method}`); - } -} diff --git a/src/websocket/index.ts b/src/websocket/index.ts index 05e272a..6d10cff 100644 --- a/src/websocket/index.ts +++ b/src/websocket/index.ts @@ -10,7 +10,7 @@ import { nativeWebSocketSendBinary, nativeWebSocketSendText, validateBrowserProfile, -} from '../native'; +} from '../native/index'; import type { WebSocketBinaryType, WebSocketInit } from '../types'; import { CloseEvent } from './close-event'; import { getSendByteLength, normalizeSendData, toMessageEventData } from './send-data'; From ae8d1f27975eb4ef735afe1d97bebeb6a86f7865 Mon Sep 17 00:00:00 2001 From: ruby Date: Thu, 16 Apr 2026 00:11:39 +0400 Subject: [PATCH 3/3] refactor: remove dead code --- rust/src/napi/body.rs | 21 +-------------------- rust/src/store/body_store.rs | 22 ---------------------- src/http/request.ts | 4 ---- src/native/binding.ts | 1 - src/native/index.ts | 2 +- src/native/request.ts | 4 ---- 6 files changed, 2 insertions(+), 52 deletions(-) diff --git a/rust/src/napi/body.rs b/rust/src/napi/body.rs index db986ae..ddc5ae7 100644 --- a/rust/src/napi/body.rs +++ b/rust/src/napi/body.rs @@ -1,4 +1,4 @@ -use crate::store::body_store::{cancel_body, read_body_all, read_body_chunk}; +use crate::store::body_store::{cancel_body, read_body_chunk}; use neon::prelude::*; use neon::types::JsBuffer; @@ -32,24 +32,6 @@ fn read_body_chunk_js(mut cx: FunctionContext) -> JsResult { Ok(promise) } -fn read_body_all_js(mut cx: FunctionContext) -> JsResult { - let handle = cx.argument::(0)?.value(&mut cx) as u64; - - let channel = cx.channel(); - let (deferred, promise) = cx.promise(); - - std::thread::spawn(move || { - let result = read_body_all(handle); - - deferred.settle_with(&channel, move |mut cx| match result { - Ok(bytes) => JsBuffer::from_slice(&mut cx, &bytes), - Err(error) => cx.throw_error(format!("{:#}", error)), - }); - }); - - Ok(promise) -} - fn cancel_body_js(mut cx: FunctionContext) -> JsResult { let handle = cx.argument::(0)?.value(&mut cx) as u64; Ok(cx.boolean(cancel_body(handle))) @@ -57,7 +39,6 @@ fn cancel_body_js(mut cx: FunctionContext) -> JsResult { pub fn register(cx: &mut ModuleContext) -> NeonResult<()> { cx.export_function("readBodyChunk", read_body_chunk_js)?; - cx.export_function("readBodyAll", read_body_all_js)?; cx.export_function("cancelBody", cancel_body_js)?; Ok(()) } diff --git a/rust/src/store/body_store.rs b/rust/src/store/body_store.rs index c26cb30..fb643f3 100644 --- a/rust/src/store/body_store.rs +++ b/rust/src/store/body_store.rs @@ -65,28 +65,6 @@ pub fn read_body_chunk(handle: u64, _size: usize) -> Result<(Vec, bool)> { Ok((chunk.to_vec(), false)) } -pub fn read_body_all(handle: u64) -> Result> { - let Some(body) = remove_body(handle) else { - return Err(anyhow::anyhow!("Unknown body handle: {}", handle)); - }; - - runtime().block_on(async move { - let mut body = body.lock().await; - let mut bytes = Vec::new(); - - while let Some(chunk) = body - .response - .chunk() - .await - .context("Failed to read response body chunk")? - { - bytes.extend_from_slice(&chunk); - } - - Ok::, anyhow::Error>(bytes) - }) -} - pub fn cancel_body(handle: u64) -> bool { remove_body(handle).is_some() } diff --git a/src/http/request.ts b/src/http/request.ts index d75d0b5..1336225 100644 --- a/src/http/request.ts +++ b/src/http/request.ts @@ -228,7 +228,3 @@ export class Request { return this.#readBodyBytes(); } } - -export function isWreqRequest(value: unknown): value is Request { - return value instanceof Request; -} diff --git a/src/native/binding.ts b/src/native/binding.ts index ebf1081..d9ec7dc 100644 --- a/src/native/binding.ts +++ b/src/native/binding.ts @@ -26,7 +26,6 @@ export type NativeBinding = { chunk: Buffer; done: boolean; }>; - readBodyAll: (handle: number) => Promise; cancelBody: (handle: number) => boolean; getProfiles: () => string[]; }; diff --git a/src/native/index.ts b/src/native/index.ts index 7a9be2a..dbbc954 100644 --- a/src/native/index.ts +++ b/src/native/index.ts @@ -1,6 +1,6 @@ export { normalizeMethod } from './methods'; export { getProfiles, validateBrowserProfile } from './profiles'; -export { nativeCancelBody, nativeReadBodyAll, nativeReadBodyChunk, nativeRequest } from './request'; +export { nativeCancelBody, nativeReadBodyChunk, nativeRequest } from './request'; export { nativeWebSocketClose, nativeWebSocketConnect, diff --git a/src/native/request.ts b/src/native/request.ts index 572dcd6..36e2078 100644 --- a/src/native/request.ts +++ b/src/native/request.ts @@ -69,10 +69,6 @@ export async function nativeReadBodyChunk( return getBinding().readBodyChunk(handle, size); } -export async function nativeReadBodyAll(handle: number): Promise { - return getBinding().readBodyAll(handle); -} - export function nativeCancelBody(handle: number): boolean { return getBinding().cancelBody(handle); }