From e5b15d33e320fc24aa8e2025add8a746073bf950 Mon Sep 17 00:00:00 2001 From: "waldemort-auto[bot]" <223556219+Copilot@users.noreply.github.com> Date: Fri, 28 Aug 2026 16:16:50 +0000 Subject: [PATCH] Require HTTPS in restricted HTTP builds Prevent credentials and request bodies from being transmitted over plaintext HTTP when a restricted allow-list feature is enabled. Keep plaintext support for development-only http-allow-all builds and cover DSL and raw-node execution paths. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CHANGELOG.md | 4 + docs/http-security.md | 19 +++-- src/ssrf.rs | 63 ++++++++++++++-- tests/e2e/sql/06_http_and_ssrf.sql | 113 +++++++++++++++++++++++++++- tests/e2e/sql/48_http_allow_all.sql | 6 +- 5 files changed, 188 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cc7358b8..b28c14a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,10 @@ Pre-1.0 note: while `pg_durable` is in major version `0`, minor releases may inc - **`pg_durable.host` (#360):** a postmaster GUC that selects the PostgreSQL host used by every connection pg_durable creates. When empty or unset, `PGHOST` is used, falling back to `127.0.0.1`. +### Fixed + +- **Restricted HTTP transport (#342):** restricted allow-list builds now require HTTPS so credentials and request bodies cannot be sent over plaintext HTTP; development-only `http-allow-all` builds continue to permit HTTP. + ## [0.2.6] - 2026-08-23 ### Added diff --git a/docs/http-security.md b/docs/http-security.md index 860d3f8b..d57c7252 100644 --- a/docs/http-security.md +++ b/docs/http-security.md @@ -29,9 +29,9 @@ or SQL. | Feature | What is allowed | Use case | |---------|-----------------|----------| | *(none)* | Nothing — `df.http()` errors immediately at DSL time **and** at execution time | Deployments that don't need HTTP | -| `http-allow-azure-domains` | Subdomains of the Azure allow-list plus `api.github.com`; bare IPs blocked; redirects blocked | Production | -| `http-allow-test-domains` | Everything in `http-allow-azure-domains` **plus** `httpbingo.org` | E2E testing; implies `http-allow-azure-domains` | -| `http-allow-all` | All URLs; SSRF IP blocklist and allow-list are both disabled | Local development only | +| `http-allow-azure-domains` | HTTPS to subdomains of the Azure allow-list plus `api.github.com`; bare IPs blocked; redirects blocked | Production | +| `http-allow-test-domains` | HTTPS to everything in `http-allow-azure-domains` **plus** `httpbingo.org` | E2E testing; implies `http-allow-azure-domains` | +| `http-allow-all` | HTTP and HTTPS to all URLs; SSRF IP blocklist and allow-list are both disabled | Local development only | The scripts and CI use `http-allow-test-domains` so that the HTTP E2E tests pass — this includes the source-built `Dockerfile` used for local dev and CI. @@ -296,9 +296,13 @@ check; the allowlist is the definitive gate for IP-literal URLs. ### 6.1 Scheme restriction -Only `http://` and `https://` are accepted. All other schemes (`file://`, -`ftp://`, `gopher://`, etc.) are rejected before any DNS resolution or -connection attempt. +Restricted builds accept only `https://`. This prevents credentials and request +bodies from being transmitted over plaintext connections. Plaintext `http://` +is available only with the development-only `http-allow-all` feature. + +All other schemes (`file://`, `ftp://`, `gopher://`, etc.) are rejected before +any DNS resolution or connection attempt. Scheme validation runs both when the +DSL node is created and when the canonical parsed URL is executed. ### 6.2 Redirect blocking @@ -329,7 +333,8 @@ leaking internal network topology to potentially malicious users. |----------|---------| | No EXECUTE privilege on df.http() | `Blocked: role '{role}' does not have EXECUTE privilege on df.http(). Grant EXECUTE ON FUNCTION df.http(text,text,text,jsonb,integer) TO {role} to allow HTTP requests.` | | HTTP disabled (no feature) | `Blocked: outbound HTTP requests are disabled. Rebuild with the 'http-allow-azure-domains' Cargo feature to enable them.` | -| Unsupported scheme | `Blocked: unsupported URL scheme '{scheme}'. Only http and https are allowed.` | +| Plaintext HTTP in a restricted build | `Blocked: plaintext HTTP is not permitted in restricted builds. HTTPS is required.` | +| Unsupported scheme | `Blocked: unsupported URL scheme '{scheme}'. Only {allowed} is allowed.` where `{allowed}` is `https` in restricted builds or `http and https` with `http-allow-all` | | Bare IP address | `Blocked: requests to bare IP addresses are not permitted. Use an approved Azure service hostname instead.` | | Non-allowed domain | `Blocked: '{host}' is not in the allowed endpoint list. Only requests to approved Azure service domains are permitted.` | | Blocked IP (literal or DNS) | `Blocked: the resolved IP address for '{host}' is in a restricted range. df.http() cannot access private or internal network addresses.` | diff --git a/src/ssrf.rs b/src/ssrf.rs index cdd5de64..904d393b 100644 --- a/src/ssrf.rs +++ b/src/ssrf.rs @@ -152,15 +152,35 @@ fn check_blocked_ipv6(ip: Ipv6Addr) -> Option<&'static str> { } } -/// Validate a URL scheme. Only `http` and `https` are permitted. -/// Returns `Err` with a user-facing message if the scheme is disallowed. +/// Validate a URL scheme against the build's outbound HTTP policy. +/// Restricted allow-list builds require HTTPS; development-only `http-allow-all` +/// builds retain plaintext HTTP support. Builds without HTTP support accept both +/// schemes here so the feature-specific disabled error remains authoritative. pub fn validate_url_scheme(url: &str) -> Result<(), String> { let scheme = url.split("://").next().unwrap_or("").to_ascii_lowercase(); + let allows_plaintext = cfg!(feature = "http-allow-all") + || !cfg!(any( + feature = "http-allow-azure-domains", + feature = "http-allow-test-domains" + )); + match scheme.as_str() { - "http" | "https" => Ok(()), - other => Err(format!( - "Blocked: unsupported URL scheme '{other}'. Only http and https are allowed." - )), + "https" => Ok(()), + "http" if allows_plaintext => Ok(()), + "http" => Err( + "Blocked: plaintext HTTP is not permitted in restricted builds. HTTPS is required." + .to_string(), + ), + other => { + let allowed = if allows_plaintext { + "http and https" + } else { + "https" + }; + Err(format!( + "Blocked: unsupported URL scheme '{other}'. Only {allowed} is allowed." + )) + } } } @@ -546,14 +566,43 @@ mod tests { // --- URL scheme validation --- + #[cfg(any( + feature = "http-allow-azure-domains", + feature = "http-allow-test-domains" + ))] + #[cfg(not(feature = "http-allow-all"))] + #[test] + fn restricted_builds_require_https() { + assert!(validate_url_scheme("https://example.com").is_ok()); + assert!(validate_url_scheme("HTTPS://example.com").is_ok()); + assert!(validate_url_scheme("http://example.com") + .unwrap_err() + .contains("HTTPS is required")); + assert!(validate_url_scheme("HTTP://EXAMPLE.COM") + .unwrap_err() + .contains("HTTPS is required")); + } + + #[cfg(feature = "http-allow-all")] #[test] - fn allows_http_https() { + fn allow_all_builds_accept_http_and_https() { assert!(validate_url_scheme("http://example.com").is_ok()); assert!(validate_url_scheme("https://example.com").is_ok()); assert!(validate_url_scheme("HTTP://EXAMPLE.COM").is_ok()); assert!(validate_url_scheme("HTTPS://example.com").is_ok()); } + #[cfg(not(any( + feature = "http-allow-all", + feature = "http-allow-azure-domains", + feature = "http-allow-test-domains" + )))] + #[test] + fn disabled_builds_defer_http_rejection_to_feature_policy() { + assert!(validate_url_scheme("http://example.com").is_ok()); + assert!(validate_url_scheme("https://example.com").is_ok()); + } + #[test] fn blocks_file_scheme() { assert!(validate_url_scheme("file:///etc/passwd").is_err()); diff --git a/tests/e2e/sql/06_http_and_ssrf.sql b/tests/e2e/sql/06_http_and_ssrf.sql index b525b901..561ad6e1 100644 --- a/tests/e2e/sql/06_http_and_ssrf.sql +++ b/tests/e2e/sql/06_http_and_ssrf.sql @@ -718,7 +718,7 @@ SELECT df.clearvars(); CREATE TEMP TABLE _test_ssrf1 (instance_id TEXT); INSERT INTO _test_ssrf1 SELECT df.start( - df.http('http://169.254.169.254/latest/meta-data/', 'GET'), + df.http('https://169.254.169.254/latest/meta-data/', 'GET'), 'test-ssrf-metadata' ); @@ -754,7 +754,7 @@ DROP TABLE _test_ssrf1; CREATE TEMP TABLE _test_ssrf2 (instance_id TEXT); INSERT INTO _test_ssrf2 SELECT df.start( - df.http('http://127.0.0.1:9999/probe', 'GET'), + df.http('https://127.0.0.1:9999/probe', 'GET'), 'test-ssrf-localhost' ); @@ -786,6 +786,41 @@ END $$; DROP TABLE _test_ssrf2; +-- Test 2c: Restricted builds reject plaintext HTTP at DSL time for both APIs +DO $$ +DECLARE + caught_http BOOLEAN := false; + caught_multipart BOOLEAN := false; +BEGIN + BEGIN + PERFORM df.http('http://api.github.com/repos', 'GET'); + EXCEPTION WHEN OTHERS THEN + caught_http := SQLERRM ILIKE '%HTTPS is required%'; + END; + + BEGIN + PERFORM df.http_multipart( + 'http://api.github.com/repos', + 'POST', + '[{"name":"field","value":"value"}]'::jsonb + ); + EXCEPTION WHEN OTHERS THEN + caught_multipart := SQLERRM ILIKE '%HTTPS is required%'; + END; + + IF NOT caught_http THEN + RAISE EXCEPTION + 'TEST FAILED: df.http() should require HTTPS in restricted builds'; + END IF; + + IF NOT caught_multipart THEN + RAISE EXCEPTION + 'TEST FAILED: df.http_multipart() should require HTTPS in restricted builds'; + END IF; + + RAISE NOTICE 'TEST PASSED: restricted_http_requires_https_at_dsl_time'; +END $$; + -- Test 3: Block unsupported URL scheme (file://) — DSL time and execution time DO $$ DECLARE @@ -1109,6 +1144,80 @@ END $$; DROP TABLE _test_ssrf11; +-- Test 11b: Raw HTTP nodes cannot bypass restricted-build HTTPS enforcement +CREATE TEMP TABLE _test_ssrf11b (instance_id TEXT); + +INSERT INTO _test_ssrf11b +SELECT df.start( + '{"node_type":"HTTP","query":"{\"url\":\"http://api.github.com/repos\",\"method\":\"GET\",\"body\":null,\"headers\":null,\"timeout_seconds\":5}"}', + 'test-ssrf-plaintext-bypass' +); + +DO $$ +DECLARE + inst_id TEXT; + status TEXT; + node_result TEXT; +BEGIN + SELECT instance_id INTO inst_id FROM _test_ssrf11b; + SELECT df.await_instance(inst_id) INTO status; + + IF status != 'failed' THEN + RAISE EXCEPTION 'TEST FAILED: expected status = failed, got %', status; + END IF; + + SELECT result::text INTO node_result + FROM df.nodes + WHERE instance_id = inst_id AND node_type = 'HTTP'; + + IF node_result IS NULL OR node_result NOT ILIKE '%HTTPS is required%' THEN + RAISE EXCEPTION + 'TEST FAILED: expected HTTPS-required error in node result, got: %', + node_result; + END IF; + + RAISE NOTICE 'TEST PASSED: ssrf_plaintext_execution_time_rejection'; +END $$; + +DROP TABLE _test_ssrf11b; + +-- Test 11c: Raw multipart nodes use the same execution-time scheme policy +CREATE TEMP TABLE _test_ssrf11c (instance_id TEXT); + +INSERT INTO _test_ssrf11c +SELECT df.start( + '{"node_type":"HTTP_MULTIPART","query":"{\"url\":\"http://api.github.com/repos\",\"method\":\"POST\",\"parts\":[{\"name\":\"field\",\"data_b64\":\"dmFsdWU=\"}],\"headers\":null,\"timeout_seconds\":5}"}', + 'test-ssrf-multipart-plaintext-bypass' +); + +DO $$ +DECLARE + inst_id TEXT; + status TEXT; + node_result TEXT; +BEGIN + SELECT instance_id INTO inst_id FROM _test_ssrf11c; + SELECT df.await_instance(inst_id) INTO status; + + IF status != 'failed' THEN + RAISE EXCEPTION 'TEST FAILED: expected status = failed, got %', status; + END IF; + + SELECT result::text INTO node_result + FROM df.nodes + WHERE instance_id = inst_id AND node_type = 'HTTP_MULTIPART'; + + IF node_result IS NULL OR node_result NOT ILIKE '%HTTPS is required%' THEN + RAISE EXCEPTION + 'TEST FAILED: expected HTTPS-required multipart error, got: %', + node_result; + END IF; + + RAISE NOTICE 'TEST PASSED: ssrf_multipart_plaintext_execution_time_rejection'; +END $$; + +DROP TABLE _test_ssrf11c; + RESET SESSION AUTHORIZATION; -- ============================================================================ diff --git a/tests/e2e/sql/48_http_allow_all.sql b/tests/e2e/sql/48_http_allow_all.sql index 4ced0e0b..64e7290e 100644 --- a/tests/e2e/sql/48_http_allow_all.sql +++ b/tests/e2e/sql/48_http_allow_all.sql @@ -15,7 +15,7 @@ CREATE TEMP TABLE _test_allowall1 (instance_id TEXT); INSERT INTO _test_allowall1 SELECT df.start( - df.http('https://example.com/', 'GET'), + df.http('http://example.com/', 'GET'), 'test-http-allow-all-non-azure' ); @@ -38,6 +38,10 @@ BEGIN RAISE EXCEPTION 'TEST FAILED: allow-list should be bypassed under http-allow-all, got: %', node_result; END IF; + IF node_result ILIKE '%HTTPS is required%' THEN + RAISE EXCEPTION 'TEST FAILED: plaintext HTTP should be allowed under http-allow-all, got: %', node_result; + END IF; + IF node_result ILIKE '%bare IP%' THEN RAISE EXCEPTION 'TEST FAILED: IP check should be bypassed under http-allow-all, got: %', node_result; END IF;