diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1a4c99135..d117cc5d0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -308,6 +308,14 @@ jobs: - name: Run CPython tests run: cargo test -p bashkit --features cpython,http_client --lib --test integration -- cpython + # bashkit's own requests/httpx (guest modules) against the upstream + # packages: same scenarios, same output. + - name: CPython requests/httpx differential + run: | + python3 -m venv /tmp/http-diff + /tmp/http-diff/bin/pip install --quiet requests==2.34.2 httpx==0.28.1 + /tmp/http-diff/bin/python crates/bashkit-cpython-wasm/guest/tests/differential.py + # Same suites on the opt-in native-code build (`cpython-native`). - name: Run CPython tests (native code) run: cargo test -p bashkit --features cpython-native,http_client --lib --test integration -- cpython diff --git a/crates/bashkit-cpython-wasm/artifacts/MANIFEST b/crates/bashkit-cpython-wasm/artifacts/MANIFEST index 8978e92fd..6e6c86e5b 100644 --- a/crates/bashkit-cpython-wasm/artifacts/MANIFEST +++ b/crates/bashkit-cpython-wasm/artifacts/MANIFEST @@ -3,5 +3,5 @@ wasi_sdk=24 wasmtime_wizer=49.0.1 zlib=1.3.1 sqlite=sqlite-amalgamation-3500400 -8cbd501c5b6e0f73d986be9b5bf48f4f0f28d40ef83774e666ffbb854758b0a8 python.wasm.gz -abf5df19ec01bb180672e94b816711fc03dfd92e109b94fc10e5399523bb464c python314.zip +31c43d757812151582ff3e274f0d9c65cf210064a46646a5a73a68704b83329b python.wasm.gz +7c1fec908a3c250e0591a4892f3dfbaa73e5c81798dfa33b1dc80e18931dbb74 python314.zip diff --git a/crates/bashkit-cpython-wasm/artifacts/python.wasm.gz b/crates/bashkit-cpython-wasm/artifacts/python.wasm.gz index 74258d477..58fdb7823 100644 Binary files a/crates/bashkit-cpython-wasm/artifacts/python.wasm.gz and b/crates/bashkit-cpython-wasm/artifacts/python.wasm.gz differ diff --git a/crates/bashkit-cpython-wasm/artifacts/python314.zip b/crates/bashkit-cpython-wasm/artifacts/python314.zip index eed7c046a..81abd3bb9 100644 Binary files a/crates/bashkit-cpython-wasm/artifacts/python314.zip and b/crates/bashkit-cpython-wasm/artifacts/python314.zip differ diff --git a/crates/bashkit-cpython-wasm/guest/_bashkit_boot.py b/crates/bashkit-cpython-wasm/guest/_bashkit_boot.py index 3200f13b9..484b30ec6 100644 --- a/crates/bashkit-cpython-wasm/guest/_bashkit_boot.py +++ b/crates/bashkit-cpython-wasm/guest/_bashkit_boot.py @@ -44,6 +44,9 @@ "urllib.parse", "uuid", "warnings", "weakref", "zlib", "runpy", "_bashkit", "atexit", "builtins", "importlib", "sqlite3", "zipfile", "asyncio", + # bashkit's own HTTP clients over the host bridge: preloaded so + # `import requests` / `import httpx` cost nothing per call. + "requests", "requests.utils", "httpx", "httpx2", ) _USAGE = """\ diff --git a/crates/bashkit-cpython-wasm/guest/_bashkit_http.py b/crates/bashkit-cpython-wasm/guest/_bashkit_http.py index 2057d3955..b13d0ef6a 100644 --- a/crates/bashkit-cpython-wasm/guest/_bashkit_http.py +++ b/crates/bashkit-cpython-wasm/guest/_bashkit_http.py @@ -11,8 +11,11 @@ http.client writes, sends the request through the host once it is complete, and serves the reply as an HTTP/1.1 byte stream that the real `HTTPResponse` parses. Status line, headers, chunked uploads, keep-alive - and HEAD behave as in CPython, and host errors surface from `request()` - so urllib wraps them in `URLError` like a failed connect. + and HEAD behave as in CPython. Where errors surface matches a real + socket: a refused or denied request fails in `request()` (urllib wraps it + in `URLError`, like a failed connect), while a timeout waiting for the + reply surfaces from `getresponse()` (a read timeout, which urllib3 and + requests report as `ReadTimeout`). - `HTTPSConnection` is defined although `ssl` is missing (context arguments are accepted and ignored), so urllib.request registers its https handler. - Proxies and CONNECT tunnels are not used: the bridge sends to the tunnel @@ -123,7 +126,11 @@ def sendall(self, data): return method, target, headers, body, used = parsed del self._out[:used] - self._replies.append(self._exchange(method, target, headers, body)) + try: + reply = self._exchange(method, target, headers, body) + except TimeoutError as e: + reply = e # raised when the response is read + self._replies.append(reply) def send(self, data): self.sendall(data) @@ -158,7 +165,10 @@ def makefile(self, mode="r", *args, **kwargs): raise OSError("bridge socket is read-only") if not self._replies: raise ConnectionError("no HTTP request was sent") - return io.BytesIO(self._replies.pop(0)) + reply = self._replies.pop(0) + if isinstance(reply, BaseException): + raise reply + return io.BytesIO(reply) def settimeout(self, timeout): self._timeout = timeout diff --git a/crates/bashkit-cpython-wasm/guest/_bashkit_webcore.py b/crates/bashkit-cpython-wasm/guest/_bashkit_webcore.py new file mode 100644 index 000000000..199b2b440 --- /dev/null +++ b/crates/bashkit-cpython-wasm/guest/_bashkit_webcore.py @@ -0,0 +1,330 @@ +"""Shared core of bashkit's `requests` and `httpx` modules. + +Both talk to `_bashkit.http` directly (the host runs every request through +bashkit's egress pipeline), not through http.client/urllib3. + +Decisions: +- Small, API-compatible subsets written for bashkit instead of the upstream + packages: upstream `requests` pulls in 155 modules (urllib3, email, + http.cookiejar, charset_normalizer) and costs ~3 s per call to import on + Pulley. These modules use only what the snapshot already preloads, and + are preloaded themselves, so importing them is free. +- Redirects are followed here, so every hop is a new request the host checks + against the allowlist again. `Authorization` is dropped when the host + changes. +- Responses are buffered by the host (its response cap applies); streaming + APIs iterate over the buffered body. +- The host does not decompress (TM-NET-013); gzip/deflate bodies are + decompressed here, inside the guest's memory limit. +- Cookies: a minimal per-host jar (name=value, no path/expiry handling). +""" + +import base64 +import json +import os +import zlib +from collections.abc import Mapping, MutableMapping +from urllib.parse import urlencode, urljoin, urlsplit, urlunsplit + +import _bashkit + +REDIRECT_CODES = (301, 302, 303, 307, 308) +MAX_REDIRECTS = 30 + + +def reason_phrase(status): + from http import HTTPStatus + + try: + return HTTPStatus(status).phrase + except ValueError: + return "" + + +class CaseInsensitiveDict(MutableMapping): + """Header mapping: case-insensitive keys, original case preserved.""" + + def __init__(self, data=None, **kwargs): + self._store = {} + self.update(data or {}, **kwargs) + + def __setitem__(self, key, value): + self._store[key.lower()] = (key, value) + + def __getitem__(self, key): + return self._store[key.lower()][1] + + def __delitem__(self, key): + del self._store[key.lower()] + + def __iter__(self): + return (k for k, _ in self._store.values()) + + def __len__(self): + return len(self._store) + + def lower_items(self): + return ((k, v[1]) for k, v in self._store.items()) + + def __eq__(self, other): + if isinstance(other, Mapping): + other = CaseInsensitiveDict(other) + else: + return NotImplemented + return dict(self.lower_items()) == dict(other.lower_items()) + + def copy(self): + return CaseInsensitiveDict(self._store.values()) + + def __repr__(self): + return str(dict(self.items())) + + +def header_items(headers): + """Normalize a mapping or a list of pairs to a list of (str, str).""" + if headers is None: + return [] + items = headers.items() if hasattr(headers, "items") else headers + out = [] + for k, v in items: + if v is None: + continue + if isinstance(k, bytes): + k = k.decode("latin-1") + if isinstance(v, bytes): + v = v.decode("latin-1") + out.append((str(k), str(v))) + return out + + +def merge_headers(base, extra): + """Merge header lists; later names replace earlier ones (case-insensitive). + A None value in `extra` removes the header.""" + merged = CaseInsensitiveDict() + for k, v in base: + merged[k] = v + if extra is not None: + items = extra.items() if hasattr(extra, "items") else extra + for k, v in items: + if v is None: + merged.pop(k, None) + else: + merged[k] = v + return header_items(merged) + + +def add_params(url, params): + if not params: + return url + if isinstance(params, (str, bytes)): + query = params.decode() if isinstance(params, bytes) else params + else: + items = params.items() if hasattr(params, "items") else params + pairs = [] + for k, v in items: + if v is None: + continue + if isinstance(v, (list, tuple)): + pairs.extend((k, x) for x in v) + else: + pairs.append((k, v)) + query = urlencode(pairs) + if not query: + return url + parts = urlsplit(url) + joined = f"{parts.query}&{query}" if parts.query else query + return urlunsplit(parts._replace(query=joined)) + + +def encode_form(data): + if isinstance(data, (str, bytes)): + return data.encode() if isinstance(data, str) else data + items = data.items() if hasattr(data, "items") else data + pairs = [] + for k, v in items: + if isinstance(v, (list, tuple)): + pairs.extend((k, x) for x in v) + elif v is not None: + pairs.append((k, v)) + return urlencode(pairs).encode() + + +def encode_multipart(data, files): + """Return (body, content_type) for multipart/form-data.""" + boundary = os.urandom(16).hex() + out = bytearray() + + def part(disposition, content, ctype=None): + out.extend(f"--{boundary}\r\nContent-Disposition: {disposition}\r\n".encode()) + if ctype: + out.extend(f"Content-Type: {ctype}\r\n".encode()) + out.extend(b"\r\n") + out.extend(content) + out.extend(b"\r\n") + + fields = data.items() if hasattr(data, "items") else (data or []) + for name, value in fields: + values = value if isinstance(value, (list, tuple)) else [value] + for v in values: + if v is None: + continue + if not isinstance(v, bytes): + v = str(v).encode() + part(f'form-data; name="{_quote(name)}"', v) + items = files.items() if hasattr(files, "items") else files + for name, spec in items: + filename, content, ctype = None, spec, None + if isinstance(spec, (tuple, list)): + filename = spec[0] + content = spec[1] + ctype = spec[2] if len(spec) > 2 else None + if hasattr(content, "read"): + if filename is None: + filename = os.path.basename(getattr(content, "name", "") or "") or name + content = content.read() + if isinstance(content, str): + content = content.encode() + if filename is None: + filename = name + disposition = f'form-data; name="{_quote(name)}"' + if filename: + disposition += f'; filename="{_quote(filename)}"' + part(disposition, content, ctype or "application/octet-stream") + out.extend(f"--{boundary}--\r\n".encode()) + return bytes(out), f"multipart/form-data; boundary={boundary}" + + +def _quote(s): + return str(s).replace("\\", "\\\\").replace('"', '\\"').replace("\r", " ").replace("\n", " ") + + +def basic_auth(user, password): + if isinstance(user, str): + user = user.encode("latin-1") + if isinstance(password, str): + password = password.encode("latin-1") + return "Basic " + base64.b64encode(user + b":" + password).decode() + + +def json_body(obj, compact=True): + """httpx sends compact UTF-8 JSON; requests uses json.dumps defaults.""" + if compact: + return json.dumps(obj, allow_nan=False, separators=(",", ":"), ensure_ascii=False).encode() + return json.dumps(obj, allow_nan=False).encode() + + +def decode_body(headers, body): + """Undo Content-Encoding (gzip, deflate); unknown encodings pass through.""" + enc = "" + for k, v in headers: + if k.lower() == "content-encoding": + enc = v.strip().lower() + if not body or enc in ("", "identity"): + return body + if enc in ("gzip", "x-gzip"): + return zlib.decompress(body, 16 + zlib.MAX_WBITS) + if enc == "deflate": + try: + return zlib.decompress(body) + except zlib.error: + return zlib.decompress(body, -zlib.MAX_WBITS) + return body + + +def charset_of(headers): + for k, v in headers: + if k.lower() == "content-type": + for param in v.split(";")[1:]: + name, _, value = param.partition("=") + if name.strip().lower() == "charset": + return value.strip().strip("\"'") or None + return None + + +def content_type_of(headers): + for k, v in headers: + if k.lower() == "content-type": + return v.split(";", 1)[0].strip().lower() + return "" + + +def parse_set_cookies(headers): + """name -> value for every Set-Cookie header (attributes ignored).""" + out = {} + for k, v in headers: + if k.lower() == "set-cookie": + name, sep, value = v.split(";", 1)[0].partition("=") + if sep and name.strip(): + out[name.strip()] = value.strip() + return out + + +class CookieJar: + """Per-host name=value cookies.""" + + def __init__(self): + self._hosts = {} + + def update_from(self, host, headers): + cookies = parse_set_cookies(headers) + if cookies: + self._hosts.setdefault(host, {}).update(cookies) + + def set(self, name, value, host=""): + self._hosts.setdefault(host, {})[name] = value + + def header_for(self, host, extra=None): + merged = dict(self._hosts.get("", {})) + merged.update(self._hosts.get(host, {})) + if extra: + merged.update(extra) + if not merged: + return None + return "; ".join(f"{k}={v}" for k, v in merged.items()) + + def as_dict(self): + out = {} + for cookies in self._hosts.values(): + out.update(cookies) + return out + + +def split_url(url): + """Validate an absolute http(s) URL. Returns (scheme, host) or raises + ValueError with a short reason ('scheme', 'missing', 'host').""" + parts = urlsplit(url) + if not parts.scheme: + raise ValueError("missing") + if parts.scheme.lower() not in ("http", "https"): + raise ValueError("scheme") + if not parts.hostname: + raise ValueError("host") + return parts.scheme.lower(), parts.hostname.lower() + + +def send(method, url, headers, body, timeout): + """One exchange through the host. Raises ConnectionError, TimeoutError or + ValueError (from `_bashkit.http`).""" + return _bashkit.http(method, url, headers, body or b"", timeout) + + +def redirect_target(method, status, url, headers): + """Next (method, url, keep_body) for a redirect response, or None.""" + if status not in REDIRECT_CODES: + return None + location = None + for k, v in headers: + if k.lower() == "location": + location = v + if not location: + return None + new_url = urljoin(url, location.strip()) + if status == 303 and method != "HEAD": + return "GET", new_url, False + if status in (301, 302) and method == "POST": + return "GET", new_url, False + return method, new_url, status in (307, 308) + + +def same_host(a, b): + return urlsplit(a).hostname == urlsplit(b).hostname diff --git a/crates/bashkit-cpython-wasm/guest/build.sh b/crates/bashkit-cpython-wasm/guest/build.sh index 2e1acb94d..ba1d9aab2 100755 --- a/crates/bashkit-cpython-wasm/guest/build.sh +++ b/crates/bashkit-cpython-wasm/guest/build.sh @@ -122,7 +122,10 @@ rm -rf "$ROOT" "$WORK/stdlib" mkdir -p "$ROOT/usr/local/lib" "$WORK/stdlib" cp -r "$SRC/Lib/." "$WORK/stdlib/" cp "$HOST_BUILD"/build/lib.wasi-wasm32-3.14/_sysconfigdata_*.py "$WORK/stdlib/" -cp "$HERE/_bashkit_boot.py" "$HERE/_bashkit_http.py" "$WORK/stdlib/" +cp "$HERE/_bashkit_boot.py" "$HERE/_bashkit_http.py" "$HERE/_bashkit_webcore.py" \ + "$HERE/httpx.py" "$HERE/httpx2.py" "$WORK/stdlib/" +# bashkit's own requests/httpx over the host bridge (see _bashkit_webcore.py). +cp -r "$HERE/requests" "$WORK/stdlib/requests" ( cd "$WORK/stdlib" # Not usable or not useful in a sandboxed, single-threaded, headless guest. diff --git a/crates/bashkit-cpython-wasm/guest/httpx.py b/crates/bashkit-cpython-wasm/guest/httpx.py new file mode 100644 index 000000000..7dca693f8 --- /dev/null +++ b/crates/bashkit-cpython-wasm/guest/httpx.py @@ -0,0 +1,1372 @@ +"""httpx for bashkit's CPython: the common `httpx` API over the host's HTTP +egress pipeline (see `_bashkit_webcore`). `httpx2` is the same module. + +Supported: the verb helpers and `stream()`, `Client` and `AsyncClient` +(base_url, headers, params, cookies, auth, timeout, follow_redirects, +event_hooks, transport=MockTransport), `Request`, `Response` (status_code, +reason_phrase, headers, content, text, json(), url, history, next_request, +is_success/..., raise_for_status, iter_*/aiter_*), `URL`, `Headers`, +`QueryParams`, `Cookies`, `Timeout`, `BasicAuth`, `codes`, and the upstream +exception hierarchy. +Accepted and ignored: verify, cert, http2, proxy, limits, mounts, trust_env. +Bodies are buffered by the host, so streaming iterates a buffered body; +async requests run one at a time. +""" + +import builtins +import datetime +import json as _json +import time +from collections.abc import Mapping, MutableMapping +from contextlib import asynccontextmanager, contextmanager +from http import HTTPStatus +from urllib.parse import parse_qsl, urlencode, urljoin, urlsplit, urlunsplit + +import _bashkit_webcore as core + +__version__ = "0.28.1+bashkit" +__title__ = "httpx" + +# --- exceptions ----------------------------------------------------------- + + +class HTTPError(Exception): + def __init__(self, message): + super().__init__(message) + self._request = None + + @property + def request(self): + if self._request is None: + raise RuntimeError("The .request property has not been set.") + return self._request + + @request.setter + def request(self, request): + self._request = request + + +class RequestError(HTTPError): + def __init__(self, message, *, request=None): + super().__init__(message) + self._request = request + + +class TransportError(RequestError): + pass + + +class TimeoutException(TransportError): + pass + + +class ConnectTimeout(TimeoutException): + pass + + +class ReadTimeout(TimeoutException): + pass + + +class WriteTimeout(TimeoutException): + pass + + +class PoolTimeout(TimeoutException): + pass + + +class NetworkError(TransportError): + pass + + +class ReadError(NetworkError): + pass + + +class WriteError(NetworkError): + pass + + +class ConnectError(NetworkError): + pass + + +class CloseError(NetworkError): + pass + + +class ProxyError(TransportError): + pass + + +class UnsupportedProtocol(TransportError): + pass + + +class ProtocolError(TransportError): + pass + + +class LocalProtocolError(ProtocolError): + pass + + +class RemoteProtocolError(ProtocolError): + pass + + +class DecodingError(RequestError): + pass + + +class TooManyRedirects(RequestError): + pass + + +class HTTPStatusError(HTTPError): + def __init__(self, message, *, request, response): + super().__init__(message) + self.request = request + self.response = response + + +class InvalidURL(Exception): + pass + + +class CookieConflict(Exception): + pass + + +class StreamError(RuntimeError): + pass + + +class StreamConsumed(StreamError): + pass + + +class StreamClosed(StreamError): + pass + + +class ResponseNotRead(StreamError): + pass + + +class RequestNotRead(StreamError): + pass + + +# --- status codes --------------------------------------------------------- + + +class _Codes: + def __getattr__(self, name): + try: + return HTTPStatus[name.upper()].value + except KeyError: + raise AttributeError(name) from None + + def __getitem__(self, name): + return getattr(self, name) + + @staticmethod + def get_reason_phrase(value): + return core.reason_phrase(value) + + @staticmethod + def is_informational(value): + return 100 <= value <= 199 + + @staticmethod + def is_success(value): + return 200 <= value <= 299 + + @staticmethod + def is_redirect(value): + return 300 <= value <= 399 + + @staticmethod + def is_client_error(value): + return 400 <= value <= 499 + + @staticmethod + def is_server_error(value): + return 500 <= value <= 599 + + @staticmethod + def is_error(value): + return 400 <= value <= 599 + + +codes = _Codes() + +# --- config --------------------------------------------------------------- + + +class Timeout: + def __init__(self, timeout=5.0, *, connect=..., read=..., write=..., pool=...): + if isinstance(timeout, Timeout): + connect, read, write, pool = timeout.connect, timeout.read, timeout.write, timeout.pool + elif isinstance(timeout, tuple): + connect, read, write, pool = (tuple(timeout) + (None,) * 4)[:4] + else: + connect = timeout if connect is ... else connect + read = timeout if read is ... else read + write = timeout if write is ... else write + pool = timeout if pool is ... else pool + self.connect, self.read, self.write, self.pool = connect, read, write, pool + + def seconds(self): + parts = [t for t in (self.connect, self.read) if t is not None] + return sum(parts) if parts else None + + def __eq__(self, other): + return isinstance(other, Timeout) and ( + self.connect, self.read, self.write, self.pool + ) == (other.connect, other.read, other.write, other.pool) + + def __repr__(self): + if len({self.connect, self.read, self.write, self.pool}) == 1: + return f"Timeout(timeout={self.connect})" + return (f"Timeout(connect={self.connect}, read={self.read}, " + f"write={self.write}, pool={self.pool})") + + +DEFAULT_TIMEOUT_CONFIG = Timeout(timeout=5.0) +DEFAULT_MAX_REDIRECTS = 20 +USE_CLIENT_DEFAULT = object() + + +class Limits: + def __init__(self, *, max_connections=None, max_keepalive_connections=None, + keepalive_expiry=5.0): + self.max_connections = max_connections + self.max_keepalive_connections = max_keepalive_connections + self.keepalive_expiry = keepalive_expiry + + +DEFAULT_LIMITS = Limits(max_connections=100, max_keepalive_connections=20) + +# --- URL, params, headers, cookies --------------------------------------- + + +class QueryParams(Mapping): + def __init__(self, *args, **kwargs): + value = args[0] if args else kwargs + items = [] + if value is None: + pass + elif isinstance(value, QueryParams): + items = value.multi_items() + elif isinstance(value, (str, bytes)): + if isinstance(value, bytes): + value = value.decode("ascii") + items = parse_qsl(value, keep_blank_values=True) + else: + pairs = value.items() if hasattr(value, "items") else value + for k, v in pairs: + values = v if isinstance(v, (list, tuple)) else [v] + items.extend((str(k), _primitive(x)) for x in values) + self._items = items + + def keys(self): + return list(dict.fromkeys(k for k, _ in self._items)) + + def __getitem__(self, key): + for k, v in self._items: + if k == key: + return v + raise KeyError(key) + + def __iter__(self): + return iter(self.keys()) + + def __len__(self): + return len(self.keys()) + + def multi_items(self): + return list(self._items) + + def get_list(self, key): + return [v for k, v in self._items if k == key] + + def set(self, key, value): + q = QueryParams(self) + q._items = [(k, v) for k, v in self._items if k != key] + [(key, _primitive(value))] + return q + + def add(self, key, value): + q = QueryParams(self) + q._items = self._items + [(key, _primitive(value))] + return q + + def remove(self, key): + q = QueryParams(self) + q._items = [(k, v) for k, v in self._items if k != key] + return q + + def merge(self, params=None): + q = QueryParams(self) + other = QueryParams(params) + keys = set(other.keys()) + q._items = [(k, v) for k, v in self._items if k not in keys] + other._items + return q + + def __str__(self): + return urlencode(self._items) + + def __repr__(self): + return f"QueryParams({str(self)!r})" + + def __eq__(self, other): + if not isinstance(other, QueryParams): + try: + other = QueryParams(other) + except Exception: + return NotImplemented + return sorted(self._items) == sorted(other._items) + + __hash__ = None + + +def _primitive(v): + if v is True: + return "true" + if v is False: + return "false" + if v is None: + return "" + return str(v) + + +class URL: + def __init__(self, url="", **kwargs): + if isinstance(url, URL): + url = url._url + elif isinstance(url, bytes): + url = url.decode("ascii") + elif not isinstance(url, str): + raise TypeError(f"Invalid type for url. Expected str or httpx.URL, got {type(url)}") + params = kwargs.pop("params", None) + parts = urlsplit(url) + for key, value in kwargs.items(): + if key == "host": + netloc = value if parts.port is None else f"{value}:{parts.port}" + parts = parts._replace(netloc=netloc) + elif key in ("scheme", "path", "fragment"): + parts = parts._replace(**{key: value}) + elif key == "query": + parts = parts._replace(query=value.decode() if isinstance(value, bytes) else value) + else: + raise TypeError(f"{key!r} is an invalid keyword argument for URL()") + if params is not None: + parts = parts._replace(query=str(QueryParams(params))) + if parts.scheme in ("http", "https") and parts.netloc and not parts.path: + parts = parts._replace(path="/") + self._url = urlunsplit(parts) + self._parts = urlsplit(self._url) + + @property + def scheme(self): + return self._parts.scheme + + @property + def host(self): + return self._parts.hostname or "" + + @property + def port(self): + return self._parts.port + + @property + def netloc(self): + return self._parts.netloc.encode("ascii") + + @property + def path(self): + return self._parts.path or "/" + + @property + def query(self): + return self._parts.query.encode("ascii") + + @property + def params(self): + return QueryParams(self._parts.query) + + @property + def raw_path(self): + q = self._parts.query + return (self.path + (f"?{q}" if q else "")).encode("ascii") + + @property + def fragment(self): + return self._parts.fragment + + @property + def userinfo(self): + netloc = self._parts.netloc + return netloc.rpartition("@")[0].encode() if "@" in netloc else b"" + + @property + def username(self): + return self._parts.username or "" + + @property + def password(self): + return self._parts.password or "" + + @property + def is_absolute_url(self): + return bool(self._parts.scheme and self._parts.netloc) + + @property + def is_relative_url(self): + return not self.is_absolute_url + + def copy_with(self, **kwargs): + return URL(self, **kwargs) + + def copy_set_param(self, key, value): + return self.copy_with(params=self.params.set(key, value)) + + def copy_add_param(self, key, value): + return self.copy_with(params=self.params.add(key, value)) + + def copy_remove_param(self, key): + return self.copy_with(params=self.params.remove(key)) + + def copy_merge_params(self, params): + return self.copy_with(params=self.params.merge(params)) + + def join(self, url): + return URL(urljoin(self._url, str(url))) + + def __str__(self): + return self._url + + def __repr__(self): + return f"URL({self._url!r})" + + def __eq__(self, other): + return isinstance(other, (URL, str)) and str(self) == str(URL(other)) + + def __hash__(self): + return hash(self._url) + + +class Headers(MutableMapping): + def __init__(self, headers=None, encoding=None): + self._list = [] + if isinstance(headers, Headers): + self._list = list(headers._list) + elif headers is not None: + self._list = core.header_items(headers) + + def __getitem__(self, key): + values = self.get_list(key) + if not values: + raise KeyError(key) + return ", ".join(values) + + def __setitem__(self, key, value): + low = key.lower() + self._list = [(k, v) for k, v in self._list if k.lower() != low] + self._list.append((key, str(value))) + + def __delitem__(self, key): + low = key.lower() + if not any(k.lower() == low for k, _ in self._list): + raise KeyError(key) + self._list = [(k, v) for k, v in self._list if k.lower() != low] + + def __iter__(self): + return iter(dict.fromkeys(k.lower() for k, _ in self._list)) + + def __len__(self): + return len(set(k.lower() for k, _ in self._list)) + + def __contains__(self, key): + low = key.lower() + return any(k.lower() == low for k, _ in self._list) + + def get_list(self, key, split_commas=False): + low = key.lower() + values = [v for k, v in self._list if k.lower() == low] + if split_commas: + values = [x.strip() for v in values for x in v.split(",")] + return values + + def multi_items(self): + return [(k.lower(), v) for k, v in self._list] + + @property + def raw(self): + return [(k.encode("latin-1"), v.encode("latin-1")) for k, v in self._list] + + @property + def encoding(self): + return "ascii" + + def update(self, headers=None, **kwargs): + for k, v in core.header_items(Headers(headers)._list if headers is not None else []): + self[k] = v + for k, v in kwargs.items(): + self[k] = v + + def copy(self): + return Headers(self) + + def __eq__(self, other): + try: + other = Headers(other) + except Exception: + return False + return sorted(self.multi_items()) == sorted(other.multi_items()) + + __hash__ = None + + def __repr__(self): + return f"Headers({dict(self.items())!r})" + + +class Cookies(MutableMapping): + def __init__(self, cookies=None): + self.jar = core.CookieJar() + if isinstance(cookies, Cookies): + for host, values in cookies.jar._hosts.items(): + self.jar._hosts[host] = dict(values) + elif cookies: + items = cookies.items() if hasattr(cookies, "items") else cookies + for k, v in items: + self.jar.set(k, v) + + def set(self, name, value, domain="", path="/"): + self.jar.set(name, value, domain.lstrip(".").lower()) + + def get(self, name, default=None, domain=None, path=None): + if domain: + return self.jar._hosts.get(domain.lstrip(".").lower(), {}).get(name, default) + return self.jar.as_dict().get(name, default) + + def delete(self, name, domain=None, path=None): + for host, values in self.jar._hosts.items(): + if domain is None or host == domain: + values.pop(name, None) + + def clear(self, domain=None, path=None): + if domain is None: + self.jar._hosts.clear() + else: + self.jar._hosts.pop(domain, None) + + def update(self, cookies=None, **kwargs): + for k, v in Cookies(cookies).items(): + self[k] = v + for k, v in kwargs.items(): + self[k] = v + + def __getitem__(self, name): + return self.jar.as_dict()[name] + + def __setitem__(self, name, value): + self.set(name, value) + + def __delitem__(self, name): + if name not in self.jar.as_dict(): + raise KeyError(name) + self.delete(name) + + def __iter__(self): + return iter(self.jar.as_dict()) + + def __len__(self): + return len(self.jar.as_dict()) + + def __bool__(self): + return bool(self.jar.as_dict()) + + def __repr__(self): + return f"" + + +# --- auth ----------------------------------------------------------------- + + +class Auth: + requires_request_body = False + requires_response_body = False + + def auth_flow(self, request): + yield request + + +class BasicAuth(Auth): + def __init__(self, username, password): + self._header = core.basic_auth(username, password) + + def auth_flow(self, request): + request.headers["Authorization"] = self._header + yield request + + +class FunctionAuth(Auth): + def __init__(self, func): + self._func = func + + def auth_flow(self, request): + yield self._func(request) + + +def _build_auth(auth): + if auth is None: + return None + if isinstance(auth, tuple): + return BasicAuth(*auth) + if isinstance(auth, Auth): + return auth + if callable(auth): + return FunctionAuth(auth) + raise TypeError(f'Invalid "auth" argument: {auth!r}') + + +def _apply_auth(auth, request): + if auth is None: + return request + flow = auth.auth_flow(request) + try: + return next(flow) + except StopIteration: + return request + + +# --- models --------------------------------------------------------------- + + +class Request: + def __init__(self, method, url, *, params=None, headers=None, cookies=None, + content=None, data=None, files=None, json=None, stream=None, + extensions=None): + self.method = method.upper() if isinstance(method, str) else method.decode().upper() + self.url = URL(url) + if params is not None: + self.url = self.url.copy_merge_params(params) + self.headers = Headers(headers) + if "host" not in self.headers and self.url._parts.netloc: + # httpx sets Host first; the bashkit host replaces it anyway. + self.headers._list.insert(0, ("Host", self.url._parts.netloc.rpartition("@")[2])) + self.extensions = extensions or {} + body, ctype = b"", None + if content is not None: + if isinstance(content, str): + body = content.encode() + elif isinstance(content, (bytes, bytearray)): + body = bytes(content) + else: + body = b"".join(c.encode() if isinstance(c, str) else c for c in content) + elif files: + body, ctype = core.encode_multipart(data or {}, files) + elif data is not None: + if isinstance(data, (str, bytes)): + body = data.encode() if isinstance(data, str) else data + else: + body, ctype = core.encode_form(data), "application/x-www-form-urlencoded" + elif json is not None: + body, ctype = core.json_body(json), "application/json" + if ctype and "content-type" not in self.headers: + self.headers["Content-Type"] = ctype + if body and "content-length" not in self.headers: + self.headers["Content-Length"] = str(len(body)) + if cookies: + c = Cookies(cookies) + header = c.jar.header_for(self.url.host) + if header: + self.headers["Cookie"] = header + self._content = body + + @property + def content(self): + return self._content + + def read(self): + return self._content + + async def aread(self): + return self._content + + def __repr__(self): + return f"" + + +class Response: + def __init__(self, status_code, *, headers=None, content=None, text=None, + html=None, json=None, stream=None, request=None, extensions=None, + history=None, default_encoding="utf-8"): + self.status_code = status_code + self.headers = Headers(headers) + self._request = request + self.history = list(history or []) + self.next_request = None + self.extensions = extensions or {} + self.is_closed = True + self.is_stream_consumed = True + self.default_encoding = default_encoding + self._encoding = None + self._elapsed = datetime.timedelta(0) + self.cookies = Cookies() + if content is not None: + body = content.encode() if isinstance(content, str) else bytes(content) + elif text is not None: + body = text.encode() + if "content-type" not in self.headers: + self.headers["Content-Type"] = "text/plain; charset=utf-8" + elif html is not None: + body = html.encode() + if "content-type" not in self.headers: + self.headers["Content-Type"] = "text/html; charset=utf-8" + elif json is not None: + body = core.json_body(json) + if "content-type" not in self.headers: + self.headers["Content-Type"] = "application/json" + else: + body = b"" + self._content = body + + @property + def request(self): + if self._request is None: + raise RuntimeError("The request instance has not been set on this response.") + return self._request + + @request.setter + def request(self, value): + self._request = value + + @property + def url(self): + return self.request.url + + @property + def elapsed(self): + return self._elapsed + + @property + def http_version(self): + return "HTTP/1.1" + + @property + def reason_phrase(self): + return core.reason_phrase(self.status_code) + + @property + def content(self): + return self._content + + @property + def charset_encoding(self): + return core.charset_of(self.headers.multi_items()) + + @property + def encoding(self): + if self._encoding is None: + enc = self.charset_encoding + if enc: + try: + "".encode(enc) + except LookupError: + enc = None + self._encoding = enc or (self.default_encoding if isinstance(self.default_encoding, str) + else self.default_encoding(self._content)) + return self._encoding + + @encoding.setter + def encoding(self, value): + self._encoding = value + + @property + def text(self): + return self._content.decode(self.encoding, errors="replace") + + def json(self, **kwargs): + return _json.loads(self._content, **kwargs) + + @property + def num_bytes_downloaded(self): + return len(self._content) + + @property + def is_informational(self): + return codes.is_informational(self.status_code) + + @property + def is_success(self): + return codes.is_success(self.status_code) + + @property + def is_redirect(self): + return codes.is_redirect(self.status_code) + + @property + def is_client_error(self): + return codes.is_client_error(self.status_code) + + @property + def is_server_error(self): + return codes.is_server_error(self.status_code) + + @property + def is_error(self): + return codes.is_error(self.status_code) + + @property + def has_redirect_location(self): + return self.status_code in core.REDIRECT_CODES and "location" in self.headers + + @property + def links(self): + header = self.headers.get("link") + links = {} + if not header: + return links + for val in header.split(","): + url, _, params = val.partition(";") + link = {"url": url.strip(" <>'\"")} + for param in params.split(";"): + key, _, value = param.partition("=") + if key.strip(): + link[key.strip(" '\"")] = value.strip(" '\"") + links[link.get("rel") or link["url"]] = link + return links + + def raise_for_status(self): + if self._request is None: + raise RuntimeError( + "Cannot call `raise_for_status` as the request instance has not been set on this response." + ) + if self.is_success: + return self + if self.has_redirect_location: + message = ("{error_type} '{0.status_code} {0.reason_phrase}' for url '{0.url}'\n" + "Redirect location: '{0.headers[location]}'\n" + "For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/{0.status_code}") + else: + message = ("{error_type} '{0.status_code} {0.reason_phrase}' for url '{0.url}'\n" + "For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/{0.status_code}") + error_type = {1: "Informational response", 3: "Redirect response", 4: "Client error", + 5: "Server error"}.get(self.status_code // 100, "Invalid status code") + raise HTTPStatusError(message.format(self, error_type=error_type), + request=self._request, response=self) + + def read(self): + return self._content + + async def aread(self): + return self._content + + def iter_bytes(self, chunk_size=None): + data = self._content + size = chunk_size or len(data) or 1 + for i in range(0, len(data), size): + yield data[i : i + size] + + iter_raw = iter_bytes + + def iter_text(self, chunk_size=None): + text = self.text + size = chunk_size or len(text) or 1 + for i in range(0, len(text), size): + yield text[i : i + size] + + def iter_lines(self): + yield from self.text.splitlines() + + async def aiter_bytes(self, chunk_size=None): + for chunk in self.iter_bytes(chunk_size): + yield chunk + + aiter_raw = aiter_bytes + + async def aiter_text(self, chunk_size=None): + for chunk in self.iter_text(chunk_size): + yield chunk + + async def aiter_lines(self): + for line in self.iter_lines(): + yield line + + def close(self): + pass + + async def aclose(self): + pass + + def __enter__(self): + return self + + def __exit__(self, *args): + pass + + async def __aenter__(self): + return self + + async def __aexit__(self, *args): + pass + + def __repr__(self): + return f"" + + +# --- transports ----------------------------------------------------------- + + +class BaseTransport: + def handle_request(self, request): + raise NotImplementedError() + + def close(self): + pass + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + + +class AsyncBaseTransport: + async def handle_async_request(self, request): + raise NotImplementedError() + + async def aclose(self): + pass + + +class HTTPTransport(BaseTransport): + """Sends through the bashkit host. Connection options are accepted and + ignored: the host owns connections, TLS and proxies.""" + + def __init__(self, *args, **kwargs): + pass + + def handle_request(self, request): + url = str(request.url) + try: + core.split_url(url) + except ValueError as e: + if str(e) == "missing": + raise UnsupportedProtocol( + "Request URL is missing an 'http://' or 'https://' protocol.", request=request + ) from None + if str(e) == "scheme": + raise UnsupportedProtocol( + f"Request URL has an unsupported protocol '{request.url.scheme}://'.", + request=request, + ) from None + raise LocalProtocolError("Request URL has no host.", request=request) from None + timeout = request.extensions.get("timeout") + headers = [(k, v) for k, v in request.headers._list if k.lower() != "content-length"] + try: + status, rheaders, body = core.send(request.method, url, headers, request.content, timeout) + except TimeoutError as e: + raise ReadTimeout(str(e), request=request) from None + except builtins.ConnectionError as e: + raise ConnectError(str(e), request=request) from None + except ValueError as e: + raise LocalProtocolError(str(e), request=request) from None + if request.method != "HEAD": + try: + body = core.decode_body(rheaders, body) + except Exception as e: + raise DecodingError(str(e), request=request) from None + rheaders = [(k, v) for k, v in rheaders if k.lower() != "content-encoding"] + return Response(status, headers=rheaders, content=body, request=request) + + +class AsyncHTTPTransport(AsyncBaseTransport): + def __init__(self, *args, **kwargs): + self._sync = HTTPTransport() + + async def handle_async_request(self, request): + return self._sync.handle_request(request) + + +class MockTransport(BaseTransport, AsyncBaseTransport): + def __init__(self, handler): + self.handler = handler + + def handle_request(self, request): + return self.handler(request) + + async def handle_async_request(self, request): + response = self.handler(request) + if not isinstance(response, Response): + response = await response + return response + + +# --- clients -------------------------------------------------------------- + + +class _BaseClient: + def __init__(self, *, auth=None, params=None, headers=None, cookies=None, verify=True, + cert=None, http1=True, http2=False, proxy=None, mounts=None, + timeout=DEFAULT_TIMEOUT_CONFIG, follow_redirects=False, limits=DEFAULT_LIMITS, + max_redirects=DEFAULT_MAX_REDIRECTS, event_hooks=None, base_url="", + transport=None, trust_env=True, default_encoding="utf-8", **kwargs): + self._base_url = self._enforce_trailing_slash(URL(base_url)) + self.auth = auth + self._params = QueryParams(params) + self.headers = Headers({ + "Accept": "*/*", + "Accept-Encoding": "gzip, deflate", + "Connection": "keep-alive", + "User-Agent": f"python-httpx/{__version__}", + }) + self.headers.update(headers) + self._cookies = Cookies(cookies) + self._timeout = Timeout(timeout) + self.follow_redirects = follow_redirects + self.max_redirects = max_redirects + self._event_hooks = {"request": list((event_hooks or {}).get("request", [])), + "response": list((event_hooks or {}).get("response", []))} + self._transport = transport + self._default_encoding = default_encoding + self.is_closed = False + + @staticmethod + def _enforce_trailing_slash(url): + if url._url and not url._parts.path.endswith("/"): + return url.copy_with(path=url._parts.path + "/") + return url + + @property + def base_url(self): + return self._base_url + + @base_url.setter + def base_url(self, value): + self._base_url = self._enforce_trailing_slash(URL(value)) + + @property + def params(self): + return self._params + + @params.setter + def params(self, value): + self._params = QueryParams(value) + + @property + def cookies(self): + return self._cookies + + @cookies.setter + def cookies(self, value): + self._cookies = Cookies(value) + + @property + def timeout(self): + return self._timeout + + @timeout.setter + def timeout(self, value): + self._timeout = Timeout(value) + + @property + def event_hooks(self): + return self._event_hooks + + @event_hooks.setter + def event_hooks(self, value): + self._event_hooks = {"request": list(value.get("request", [])), + "response": list(value.get("response", []))} + + def _merge_url(self, url): + url = URL(url) + if url.is_relative_url and self._base_url._url: + rel = str(url).lstrip("/") + return URL(self._base_url._url + rel) + return url + + def build_request(self, method, url, *, content=None, data=None, files=None, json=None, + params=None, headers=None, cookies=None, timeout=USE_CLIENT_DEFAULT, + extensions=None): + url = self._merge_url(url) + merged_params = self._params.merge(params) if params is not None else self._params + if merged_params: + url = url.copy_merge_params(merged_params) + merged = Headers(self.headers) + if headers is not None: + for k, v in Headers(headers)._list: + merged[k] = v + t = self._timeout if timeout is USE_CLIENT_DEFAULT else Timeout(timeout) + ext = dict(extensions or {}) + ext["timeout"] = t.seconds() + request = Request(method, url, headers=merged, content=content, data=data, files=files, + json=json, extensions=ext) + jar = Cookies(self._cookies) + if cookies: + for k, v in Cookies(cookies).items(): + jar.set(k, v) + header = jar.jar.header_for(request.url.host) + if header and "cookie" not in request.headers: + request.headers["Cookie"] = header + return request + + def _redirect_request(self, request, response): + target = core.redirect_target(request.method, response.status_code, str(request.url), + response.headers.multi_items()) + if target is None: + return None + method, new_url, keep_body = target + headers = Headers(request.headers) + headers.pop("host", None) + if not core.same_host(str(request.url), new_url): + headers.pop("authorization", None) + headers.pop("cookie", None) + new = Request(method, new_url, headers=headers, + content=request.content if keep_body else None, + extensions=dict(request.extensions)) + if not keep_body: + for h in ("content-type", "content-length", "transfer-encoding"): + new.headers.pop(h, None) + header = self._cookies.jar.header_for(new.url.host) + if header: + new.headers["Cookie"] = header + return new + + def _after_response(self, request, response, started): + response.request = request + response._elapsed = datetime.timedelta(seconds=time.monotonic() - started) + response.default_encoding = self._default_encoding + multi = response.headers.multi_items() + self._cookies.jar.update_from(request.url.host, multi) + response.cookies = Cookies(core.parse_set_cookies(multi)) + + +class Client(_BaseClient): + def __init__(self, **kwargs): + super().__init__(**kwargs) + if self._transport is None: + self._transport = HTTPTransport() + + def request(self, method, url, *, content=None, data=None, files=None, json=None, + params=None, headers=None, cookies=None, auth=USE_CLIENT_DEFAULT, + follow_redirects=USE_CLIENT_DEFAULT, timeout=USE_CLIENT_DEFAULT, + extensions=None): + request = self.build_request(method, url, content=content, data=data, files=files, + json=json, params=params, headers=headers, cookies=cookies, + timeout=timeout, extensions=extensions) + return self.send(request, auth=auth, follow_redirects=follow_redirects) + + @contextmanager + def stream(self, method, url, **kwargs): + response = self.request(method, url, **kwargs) + try: + yield response + finally: + response.close() + + def send(self, request, *, stream=False, auth=USE_CLIENT_DEFAULT, + follow_redirects=USE_CLIENT_DEFAULT): + if self.is_closed: + raise RuntimeError("Cannot send a request, as the client has been closed.") + follow = self.follow_redirects if follow_redirects is USE_CLIENT_DEFAULT else follow_redirects + auth = _build_auth(self.auth if auth is USE_CLIENT_DEFAULT else auth) + request = _apply_auth(auth, request) + history = [] + while True: + for hook in self._event_hooks["request"]: + hook(request) + started = time.monotonic() + response = self._transport.handle_request(request) + self._after_response(request, response, started) + response.history = list(history) + for hook in self._event_hooks["response"]: + hook(response) + nxt = self._redirect_request(request, response) + if nxt is None: + return response + if not follow: + response.next_request = nxt + return response + if len(history) >= self.max_redirects: + raise TooManyRedirects("Exceeded maximum allowed redirects.", request=request) + history.append(response) + request = nxt + + def get(self, url, **kwargs): + return self.request("GET", url, **kwargs) + + def options(self, url, **kwargs): + return self.request("OPTIONS", url, **kwargs) + + def head(self, url, **kwargs): + return self.request("HEAD", url, **kwargs) + + def post(self, url, **kwargs): + return self.request("POST", url, **kwargs) + + def put(self, url, **kwargs): + return self.request("PUT", url, **kwargs) + + def patch(self, url, **kwargs): + return self.request("PATCH", url, **kwargs) + + def delete(self, url, **kwargs): + return self.request("DELETE", url, **kwargs) + + def close(self): + self.is_closed = True + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + + +class AsyncClient(_BaseClient): + """Same as `Client` with async methods. Requests run one at a time: + the host call blocks the guest while it waits.""" + + def __init__(self, **kwargs): + super().__init__(**kwargs) + if self._transport is None: + self._transport = AsyncHTTPTransport() + + async def request(self, method, url, *, content=None, data=None, files=None, json=None, + params=None, headers=None, cookies=None, auth=USE_CLIENT_DEFAULT, + follow_redirects=USE_CLIENT_DEFAULT, timeout=USE_CLIENT_DEFAULT, + extensions=None): + request = self.build_request(method, url, content=content, data=data, files=files, + json=json, params=params, headers=headers, cookies=cookies, + timeout=timeout, extensions=extensions) + return await self.send(request, auth=auth, follow_redirects=follow_redirects) + + @asynccontextmanager + async def stream(self, method, url, **kwargs): + response = await self.request(method, url, **kwargs) + try: + yield response + finally: + await response.aclose() + + async def send(self, request, *, stream=False, auth=USE_CLIENT_DEFAULT, + follow_redirects=USE_CLIENT_DEFAULT): + if self.is_closed: + raise RuntimeError("Cannot send a request, as the client has been closed.") + follow = self.follow_redirects if follow_redirects is USE_CLIENT_DEFAULT else follow_redirects + auth = _build_auth(self.auth if auth is USE_CLIENT_DEFAULT else auth) + request = _apply_auth(auth, request) + history = [] + while True: + for hook in self._event_hooks["request"]: + await hook(request) + started = time.monotonic() + response = await self._transport.handle_async_request(request) + self._after_response(request, response, started) + response.history = list(history) + for hook in self._event_hooks["response"]: + await hook(response) + nxt = self._redirect_request(request, response) + if nxt is None: + return response + if not follow: + response.next_request = nxt + return response + if len(history) >= self.max_redirects: + raise TooManyRedirects("Exceeded maximum allowed redirects.", request=request) + history.append(response) + request = nxt + + async def get(self, url, **kwargs): + return await self.request("GET", url, **kwargs) + + async def options(self, url, **kwargs): + return await self.request("OPTIONS", url, **kwargs) + + async def head(self, url, **kwargs): + return await self.request("HEAD", url, **kwargs) + + async def post(self, url, **kwargs): + return await self.request("POST", url, **kwargs) + + async def put(self, url, **kwargs): + return await self.request("PUT", url, **kwargs) + + async def patch(self, url, **kwargs): + return await self.request("PATCH", url, **kwargs) + + async def delete(self, url, **kwargs): + return await self.request("DELETE", url, **kwargs) + + async def aclose(self): + self.is_closed = True + + async def __aenter__(self): + return self + + async def __aexit__(self, *args): + await self.aclose() + + +# --- top-level API -------------------------------------------------------- + + +def request(method, url, *, params=None, content=None, data=None, files=None, json=None, + headers=None, cookies=None, auth=None, proxy=None, timeout=DEFAULT_TIMEOUT_CONFIG, + follow_redirects=False, verify=True, trust_env=True, **kwargs): + with Client(cookies=cookies, timeout=timeout) as client: + return client.request(method, url, content=content, data=data, files=files, json=json, + params=params, headers=headers, auth=auth, + follow_redirects=follow_redirects) + + +@contextmanager +def stream(method, url, **kwargs): + yield request(method, url, **kwargs) + + +def get(url, **kwargs): + return request("GET", url, **kwargs) + + +def options(url, **kwargs): + return request("OPTIONS", url, **kwargs) + + +def head(url, **kwargs): + return request("HEAD", url, **kwargs) + + +def post(url, **kwargs): + return request("POST", url, **kwargs) + + +def put(url, **kwargs): + return request("PUT", url, **kwargs) + + +def patch(url, **kwargs): + return request("PATCH", url, **kwargs) + + +def delete(url, **kwargs): + return request("DELETE", url, **kwargs) + + +__all__ = [ + "AsyncBaseTransport", "AsyncClient", "AsyncHTTPTransport", "Auth", "BaseTransport", + "BasicAuth", "Client", "CloseError", "ConnectError", "ConnectTimeout", "CookieConflict", + "Cookies", "DecodingError", "FunctionAuth", "HTTPError", "HTTPStatusError", + "HTTPTransport", "Headers", "InvalidURL", "Limits", "LocalProtocolError", "MockTransport", + "NetworkError", "PoolTimeout", "ProtocolError", "ProxyError", "QueryParams", "ReadError", + "ReadTimeout", "RemoteProtocolError", "Request", "RequestError", "RequestNotRead", + "Response", "ResponseNotRead", "StreamClosed", "StreamConsumed", "StreamError", + "Timeout", "TimeoutException", "TooManyRedirects", "TransportError", "URL", + "UnsupportedProtocol", "WriteError", "WriteTimeout", "codes", "delete", "get", "head", + "options", "patch", "post", "put", "request", "stream", +] diff --git a/crates/bashkit-cpython-wasm/guest/httpx2.py b/crates/bashkit-cpython-wasm/guest/httpx2.py new file mode 100644 index 000000000..504e1380c --- /dev/null +++ b/crates/bashkit-cpython-wasm/guest/httpx2.py @@ -0,0 +1,8 @@ +"""httpx2 (pydantic/httpx2) for bashkit's CPython: the same module as +`httpx` (see httpx.py), registered under both names.""" + +import sys + +import httpx + +sys.modules[__name__] = httpx diff --git a/crates/bashkit-cpython-wasm/guest/requests/__init__.py b/crates/bashkit-cpython-wasm/guest/requests/__init__.py new file mode 100644 index 000000000..14b6ec127 --- /dev/null +++ b/crates/bashkit-cpython-wasm/guest/requests/__init__.py @@ -0,0 +1,40 @@ +"""requests for bashkit's CPython: the common `requests` API over the host's +HTTP egress pipeline (see `_bashkit_webcore`). + +Supported: the verb helpers, `Session` (headers, params, auth, cookies, +hooks, mount), params/data/json/files/headers/cookies/auth/timeout/ +allow_redirects, `Response` (status_code, ok, reason, headers, content, +text, json(), encoding, url, history, links, iter_content/iter_lines, +raise_for_status), and the upstream exception hierarchy. +Accepted and ignored: verify, cert, proxies, stream (bodies are buffered), +adapters and their retries. +""" + +__title__ = "requests" +__version__ = "2.32.0+bashkit" + +from . import exceptions, status_codes, structures # noqa: E402 +from .api import delete, get, head, options, patch, post, put, request # noqa: E402 +from .exceptions import ( # noqa: E402 + ConnectionError, + ConnectTimeout, + FileModeWarning, + HTTPError, + JSONDecodeError, + ReadTimeout, + RequestException, + Timeout, + TooManyRedirects, + URLRequired, +) +from .models import PreparedRequest, Request, Response # noqa: E402 +from .sessions import Session, session # noqa: E402 +from .status_codes import codes # noqa: E402 + + +def __getattr__(name): + if name == "utils": + from . import utils as mod + + return mod + raise AttributeError(name) diff --git a/crates/bashkit-cpython-wasm/guest/requests/adapters.py b/crates/bashkit-cpython-wasm/guest/requests/adapters.py new file mode 100644 index 000000000..ce2598cda --- /dev/null +++ b/crates/bashkit-cpython-wasm/guest/requests/adapters.py @@ -0,0 +1,24 @@ +"""requests.adapters (bashkit). + +Every request goes through the bashkit host, so adapters only exist for +compatibility: `Session.mount()` accepts them and they change nothing. +Retries (`max_retries`) are not performed. +""" + + +class BaseAdapter: + def close(self): + pass + + +class HTTPAdapter(BaseAdapter): + def __init__(self, pool_connections=10, pool_maxsize=10, max_retries=0, pool_block=False): + self.max_retries = max_retries + self.config = {} + + def init_poolmanager(self, *args, **kwargs): + pass + + +DEFAULT_POOLSIZE = 10 +DEFAULT_RETRIES = 0 diff --git a/crates/bashkit-cpython-wasm/guest/requests/api.py b/crates/bashkit-cpython-wasm/guest/requests/api.py new file mode 100644 index 000000000..d568dabf2 --- /dev/null +++ b/crates/bashkit-cpython-wasm/guest/requests/api.py @@ -0,0 +1,37 @@ +"""requests.api (bashkit): module-level helpers, one Session per call.""" + +from . import sessions + + +def request(method, url, **kwargs): + with sessions.Session() as session: + return session.request(method=method, url=url, **kwargs) + + +def get(url, params=None, **kwargs): + return request("get", url, params=params, **kwargs) + + +def options(url, **kwargs): + return request("options", url, **kwargs) + + +def head(url, **kwargs): + kwargs.setdefault("allow_redirects", False) + return request("head", url, **kwargs) + + +def post(url, data=None, json=None, **kwargs): + return request("post", url, data=data, json=json, **kwargs) + + +def put(url, data=None, **kwargs): + return request("put", url, data=data, **kwargs) + + +def patch(url, data=None, **kwargs): + return request("patch", url, data=data, **kwargs) + + +def delete(url, **kwargs): + return request("delete", url, **kwargs) diff --git a/crates/bashkit-cpython-wasm/guest/requests/auth.py b/crates/bashkit-cpython-wasm/guest/requests/auth.py new file mode 100644 index 000000000..8244fb19e --- /dev/null +++ b/crates/bashkit-cpython-wasm/guest/requests/auth.py @@ -0,0 +1,33 @@ +"""requests.auth (bashkit): basic auth and the AuthBase hook.""" + +from _bashkit_webcore import basic_auth + + +class AuthBase: + def __call__(self, r): + raise NotImplementedError("Auth hooks must be callable.") + + +class HTTPBasicAuth(AuthBase): + def __init__(self, username, password): + self.username = username + self.password = password + + def __eq__(self, other): + return (self.username, self.password) == ( + getattr(other, "username", None), + getattr(other, "password", None), + ) + + def __call__(self, r): + r.headers["Authorization"] = basic_auth(self.username, self.password) + return r + + +class HTTPProxyAuth(HTTPBasicAuth): + def __call__(self, r): + return r # proxies are not used in bashkit + + +def _basic_auth_str(username, password): + return basic_auth(username, password) diff --git a/crates/bashkit-cpython-wasm/guest/requests/exceptions.py b/crates/bashkit-cpython-wasm/guest/requests/exceptions.py new file mode 100644 index 000000000..a0fc29bde --- /dev/null +++ b/crates/bashkit-cpython-wasm/guest/requests/exceptions.py @@ -0,0 +1,113 @@ +"""requests.exceptions (bashkit): same hierarchy as upstream requests.""" + +import json as _json + + +class RequestException(IOError): + def __init__(self, *args, **kwargs): + self.response = kwargs.pop("response", None) + self.request = kwargs.pop("request", None) + if self.response is not None and self.request is None: + self.request = getattr(self.response, "request", None) + super().__init__(*args, **kwargs) + + +class InvalidJSONError(RequestException): + pass + + +class JSONDecodeError(InvalidJSONError, _json.JSONDecodeError): + def __init__(self, *args, **kwargs): + _json.JSONDecodeError.__init__(self, *args) + InvalidJSONError.__init__(self, *self.args, **kwargs) + + def __reduce__(self): + return _json.JSONDecodeError.__reduce__(self) + + +class HTTPError(RequestException): + pass + + +class ConnectionError(RequestException): + pass + + +class ProxyError(ConnectionError): + pass + + +class SSLError(ConnectionError): + pass + + +class Timeout(RequestException): + pass + + +class ConnectTimeout(ConnectionError, Timeout): + pass + + +class ReadTimeout(Timeout): + pass + + +class URLRequired(RequestException): + pass + + +class TooManyRedirects(RequestException): + pass + + +class MissingSchema(RequestException, ValueError): + pass + + +class InvalidSchema(RequestException, ValueError): + pass + + +class InvalidURL(RequestException, ValueError): + pass + + +class InvalidHeader(RequestException, ValueError): + pass + + +class InvalidProxyURL(InvalidURL): + pass + + +class ChunkedEncodingError(RequestException): + pass + + +class ContentDecodingError(RequestException): + pass + + +class StreamConsumedError(RequestException, TypeError): + pass + + +class RetryError(RequestException): + pass + + +class UnrewindableBodyError(RequestException): + pass + + +class RequestsWarning(Warning): + pass + + +class FileModeWarning(RequestsWarning, DeprecationWarning): + pass + + +class RequestsDependencyWarning(RequestsWarning): + pass diff --git a/crates/bashkit-cpython-wasm/guest/requests/models.py b/crates/bashkit-cpython-wasm/guest/requests/models.py new file mode 100644 index 000000000..ac91c6ef8 --- /dev/null +++ b/crates/bashkit-cpython-wasm/guest/requests/models.py @@ -0,0 +1,218 @@ +"""requests.models (bashkit): Request, PreparedRequest, Response.""" + +import datetime +import json as _json + +import _bashkit_webcore as core + +from .exceptions import HTTPError, JSONDecodeError, StreamConsumedError +from .structures import CaseInsensitiveDict + + +class Request: + def __init__(self, method=None, url=None, headers=None, files=None, data=None, + params=None, auth=None, cookies=None, hooks=None, json=None): + self.method = method + self.url = url + self.headers = headers or {} + self.files = files or [] + self.data = data or [] + self.json = json + self.params = params or {} + self.auth = auth + self.cookies = cookies + self.hooks = hooks or {} + + def __repr__(self): + return f"" + + def prepare(self): + p = PreparedRequest() + p.prepare(method=self.method, url=self.url, headers=self.headers, + files=self.files, data=self.data, params=self.params, + auth=self.auth, json=self.json) + return p + + +class PreparedRequest: + def __init__(self): + self.method = None + self.url = None + self.headers = CaseInsensitiveDict() + self.body = None + self.hooks = {"response": []} + + def prepare(self, method=None, url=None, headers=None, files=None, data=None, + params=None, auth=None, cookies=None, hooks=None, json=None): + self.method = (method or "GET").upper() + self.url = core.add_params(str(url), params) + self.headers = CaseInsensitiveDict(core.header_items(headers)) + body, ctype = None, None + if files: + body, ctype = core.encode_multipart(data or {}, files) + elif data: + if hasattr(data, "read"): + body = data.read() + body = body.encode() if isinstance(body, str) else body + elif isinstance(data, (str, bytes)): + body = data.encode() if isinstance(data, str) else data + else: + body = core.encode_form(data) + ctype = "application/x-www-form-urlencoded" + elif json is not None: + body = core.json_body(json, compact=False) + ctype = "application/json" + if ctype and "content-type" not in {k.lower() for k in self.headers}: + self.headers["Content-Type"] = ctype + self.body = body + if auth is not None: + if isinstance(auth, tuple) and len(auth) == 2: + self.headers["Authorization"] = core.basic_auth(*auth) + else: + r = auth(self) + if r is not None and r is not self: + self.__dict__.update(r.__dict__) + return self + + def copy(self): + p = PreparedRequest() + p.method, p.url, p.body = self.method, self.url, self.body + p.headers = self.headers.copy() + p.hooks = dict(self.hooks) + return p + + @property + def path_url(self): + parts = core.urlsplit(self.url) + return (parts.path or "/") + (f"?{parts.query}" if parts.query else "") + + def __repr__(self): + return f"" + + +class Response: + __attrs__ = ["_content", "status_code", "headers", "url", "history", + "encoding", "reason", "cookies", "elapsed", "request"] + + def __init__(self): + self._content = b"" + self._consumed = False + self.status_code = None + self.headers = CaseInsensitiveDict() + self.raw = None + self.url = None + self.encoding = None + self.history = [] + self.reason = None + self.cookies = {} + self.elapsed = datetime.timedelta(0) + self.request = None + self.connection = None + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + + def __repr__(self): + return f"" + + def __bool__(self): + return self.ok + + def __iter__(self): + return self.iter_content(128) + + @property + def ok(self): + try: + self.raise_for_status() + except HTTPError: + return False + return True + + @property + def is_redirect(self): + return "location" in self.headers and self.status_code in core.REDIRECT_CODES + + @property + def is_permanent_redirect(self): + return "location" in self.headers and self.status_code in (301, 308) + + @property + def content(self): + return self._content + + @property + def apparent_encoding(self): + try: + self._content.decode("utf-8") + return "utf-8" + except UnicodeDecodeError: + return "ISO-8859-1" + + @property + def text(self): + if not self._content: + return "" + encoding = self.encoding or self.apparent_encoding + try: + return str(self._content, encoding, errors="replace") + except (LookupError, TypeError): + return str(self._content, errors="replace") + + def json(self, **kwargs): + try: + if self.encoding is None and self._content[:3] == b"\xef\xbb\xbf": + return _json.loads(self._content.decode("utf-8-sig"), **kwargs) + return _json.loads(self.text if self.encoding else self._content, **kwargs) + except _json.JSONDecodeError as e: + raise JSONDecodeError(e.msg, e.doc, e.pos) from None + + @property + def links(self): + header = self.headers.get("link") + links = {} + if not header: + return links + for val in header.split(","): + url, _, params = val.partition(";") + link = {"url": url.strip(" <>'\"")} + for param in params.split(";"): + key, _, value = param.partition("=") + if key.strip(): + link[key.strip(" '\"")] = value.strip(" '\"") + links[link.get("rel") or link["url"]] = link + return links + + def iter_content(self, chunk_size=1, decode_unicode=False): + if self._consumed and not self._content: + raise StreamConsumedError() + data = self._content + size = chunk_size or len(data) or 1 + if decode_unicode: + text = self.text + for i in range(0, len(text), size): + yield text[i : i + size] + return + for i in range(0, len(data), size): + yield data[i : i + size] + + def iter_lines(self, chunk_size=512, decode_unicode=False, delimiter=None): + body = self.text if decode_unicode else self._content + lines = body.split(delimiter) if delimiter else body.splitlines() + yield from lines + + def raise_for_status(self): + reason = self.reason or "" + if 400 <= self.status_code < 500: + msg = f"{self.status_code} Client Error: {reason} for url: {self.url}" + elif 500 <= self.status_code < 600: + msg = f"{self.status_code} Server Error: {reason} for url: {self.url}" + else: + return + raise HTTPError(msg, response=self) + + def close(self): + self._consumed = True diff --git a/crates/bashkit-cpython-wasm/guest/requests/sessions.py b/crates/bashkit-cpython-wasm/guest/requests/sessions.py new file mode 100644 index 000000000..09979baba --- /dev/null +++ b/crates/bashkit-cpython-wasm/guest/requests/sessions.py @@ -0,0 +1,285 @@ +"""requests.sessions (bashkit): Session and the request loop. + +Every exchange goes through `_bashkit.http`, so the host's egress policy +applies to each request and each redirect hop. +""" + +import builtins +import datetime +import time +from collections.abc import MutableMapping + +import _bashkit_webcore as core + +from . import __version__ +from .adapters import HTTPAdapter +from .exceptions import ( + ConnectionError, + InvalidHeader, + InvalidSchema, + InvalidURL, + MissingSchema, + ReadTimeout, + TooManyRedirects, +) +from .models import PreparedRequest, Request, Response +from .structures import CaseInsensitiveDict + + +def default_user_agent(name="python-requests"): + return f"{name}/{__version__}" + + +def default_headers(): + return CaseInsensitiveDict({ + "User-Agent": default_user_agent(), + "Accept-Encoding": "gzip, deflate", + "Accept": "*/*", + }) + + +class RequestsCookieJar(MutableMapping): + """Dict-like cookie jar. Cookies from responses are kept per host and + sent back only to that host; cookies set by hand go to every host.""" + + def __init__(self): + self._jar = core.CookieJar() + + def __getitem__(self, name): + return self._jar.as_dict()[name] + + def __setitem__(self, name, value): + self._jar.set(name, value) + + def __delitem__(self, name): + found = False + for cookies in self._jar._hosts.values(): + if name in cookies: + del cookies[name] + found = True + if not found: + raise KeyError(name) + + def __iter__(self): + return iter(self._jar.as_dict()) + + def __len__(self): + return len(self._jar.as_dict()) + + def set(self, name, value, domain=None, **kwargs): + if value is None: + self.pop(name, None) + else: + self._jar.set(name, value, (domain or "").lstrip(".").lower()) + + def get_dict(self, domain=None, path=None): + if domain: + return dict(self._jar._hosts.get(domain.lstrip(".").lower(), {})) + return self._jar.as_dict() + + def __repr__(self): + return f"" + + +def _timeout_seconds(timeout): + if isinstance(timeout, tuple): + parts = [t for t in timeout if t is not None] + return sum(parts) if parts else None + return timeout + + +def _check_url(url): + try: + core.split_url(url) + except ValueError as e: + if str(e) == "missing": + raise MissingSchema( + f"Invalid URL {url!r}: No scheme supplied. Perhaps you meant https://{url}?" + ) from None + if str(e) == "scheme": + raise InvalidSchema(f"No connection adapters were found for {url!r}") from None + raise InvalidURL(f"Invalid URL {url!r}: No host supplied") from None + + +class Session: + __attrs__ = ["headers", "cookies", "auth", "proxies", "hooks", "params", + "verify", "cert", "adapters", "stream", "trust_env", "max_redirects"] + + def __init__(self): + self.headers = default_headers() + self.auth = None + self.proxies = {} + self.hooks = {"response": []} + self.params = {} + self.stream = False + self.verify = True + self.cert = None + self.max_redirects = core.MAX_REDIRECTS + self.trust_env = True + self.cookies = RequestsCookieJar() + self.adapters = {"https://": HTTPAdapter(), "http://": HTTPAdapter()} + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + + def close(self): + pass + + def mount(self, prefix, adapter): + self.adapters[prefix] = adapter + + def get_adapter(self, url): + for prefix, adapter in self.adapters.items(): + if url.lower().startswith(prefix.lower()): + return adapter + raise InvalidSchema(f"No connection adapters were found for {url!r}") + + def prepare_request(self, request): + headers = CaseInsensitiveDict(self.headers) + for k, v in core.header_items(request.headers): + headers[k] = v + for k, v in (request.headers or {}).items() if hasattr(request.headers, "items") else []: + if v is None: + headers.pop(k, None) + params = dict(self.params) + if request.params and hasattr(request.params, "items"): + params.update(request.params) + params = {k: v for k, v in params.items() if v is not None} + elif request.params: + params = request.params + auth = request.auth if request.auth is not None else self.auth + p = PreparedRequest() + p.prepare(method=request.method, url=request.url, headers=headers, + files=request.files, data=request.data, params=params, + auth=auth, json=request.json) + p._cookies = dict(request.cookies or {}) + hooks = request.hooks.get("response", []) if request.hooks else [] + p.hooks = {"response": list(self.hooks.get("response", [])) + ( + list(hooks) if isinstance(hooks, (list, tuple)) else [hooks])} + return p + + def request(self, method, url, params=None, data=None, headers=None, cookies=None, + files=None, auth=None, timeout=None, allow_redirects=True, proxies=None, + hooks=None, stream=None, verify=None, cert=None, json=None): + req = Request(method=method.upper(), url=url, headers=headers, files=files, + data=data or {}, json=json, params=params or {}, auth=auth, + cookies=cookies, hooks=hooks) + prep = self.prepare_request(req) + return self.send(prep, timeout=timeout, allow_redirects=allow_redirects) + + def send(self, request, timeout=None, allow_redirects=True, **kwargs): + timeout = _timeout_seconds(timeout) + extra_cookies = getattr(request, "_cookies", None) + history = [] + prep = request + while True: + _check_url(prep.url) + resp = self._exchange(prep, timeout, extra_cookies) + for hook in prep.hooks.get("response", []): + r = hook(resp) + if r is not None: + resp = r + target = core.redirect_target(prep.method, resp.status_code, prep.url, + list(resp.headers.items())) + if not allow_redirects or target is None: + resp.history = history + if not allow_redirects and target is not None: + nxt = prep.copy() + nxt.method, nxt.url = target[0], target[1] + resp._next = nxt + return resp + history.append(resp) + if len(history) > self.max_redirects: + raise TooManyRedirects(f"Exceeded {self.max_redirects} redirects.", response=resp) + method, new_url, keep_body = target + nxt = prep.copy() + nxt.method, nxt.url = method, new_url + if not keep_body: + nxt.body = None + for h in ("Content-Type", "Content-Length", "Transfer-Encoding"): + nxt.headers.pop(h, None) + if not core.same_host(prep.url, new_url): + nxt.headers.pop("Authorization", None) + nxt.headers.pop("Cookie", None) + prep = nxt + + def _exchange(self, prep, timeout, extra_cookies): + _, host = core.split_url(prep.url) + headers = [(k, v) for k, v in prep.headers.items() if k.lower() != "cookie"] + explicit = prep.headers.get("Cookie") + cookie = explicit or self.cookies._jar.header_for(host, extra_cookies) + if cookie: + headers.append(("Cookie", cookie)) + started = time.monotonic() + try: + status, rheaders, body = core.send(prep.method, prep.url, headers, prep.body, timeout) + except TimeoutError as e: + raise ReadTimeout(str(e), request=prep) from None + except builtins.ConnectionError as e: + raise ConnectionError(str(e), request=prep) from None + except ValueError as e: + msg = str(e) + if "header" in msg.lower(): + raise InvalidHeader(msg, request=prep) from None + raise InvalidURL(msg, request=prep) from None + self.cookies._jar.update_from(host, rheaders) + resp = Response() + resp.status_code = status + resp.reason = core.reason_phrase(status) + resp.headers = CaseInsensitiveDict() + for k, v in rheaders: + if k in resp.headers and k.lower() != "set-cookie": + resp.headers[k] = f"{resp.headers[k]}, {v}" + else: + resp.headers[k] = v + try: + resp._content = core.decode_body(rheaders, body) if prep.method != "HEAD" else b"" + except Exception as e: + from .exceptions import ContentDecodingError + + raise ContentDecodingError(f"Received response with content-encoding, but failed to decode it: {e}", + response=resp) from None + charset = core.charset_of(rheaders) + ctype = core.content_type_of(rheaders) + if charset: + resp.encoding = charset + elif ctype.startswith("text/"): + resp.encoding = "ISO-8859-1" + elif ctype == "application/json": + resp.encoding = "utf-8" + resp.url = prep.url + resp.request = prep + resp.cookies = core.parse_set_cookies(rheaders) + resp.elapsed = datetime.timedelta(seconds=time.monotonic() - started) + return resp + + def get(self, url, **kwargs): + kwargs.setdefault("allow_redirects", True) + return self.request("GET", url, **kwargs) + + def options(self, url, **kwargs): + kwargs.setdefault("allow_redirects", True) + return self.request("OPTIONS", url, **kwargs) + + def head(self, url, **kwargs): + kwargs.setdefault("allow_redirects", False) + return self.request("HEAD", url, **kwargs) + + def post(self, url, data=None, json=None, **kwargs): + return self.request("POST", url, data=data, json=json, **kwargs) + + def put(self, url, data=None, **kwargs): + return self.request("PUT", url, data=data, **kwargs) + + def patch(self, url, data=None, **kwargs): + return self.request("PATCH", url, data=data, **kwargs) + + def delete(self, url, **kwargs): + return self.request("DELETE", url, **kwargs) + + +def session(): + return Session() diff --git a/crates/bashkit-cpython-wasm/guest/requests/status_codes.py b/crates/bashkit-cpython-wasm/guest/requests/status_codes.py new file mode 100644 index 000000000..f4482be39 --- /dev/null +++ b/crates/bashkit-cpython-wasm/guest/requests/status_codes.py @@ -0,0 +1,20 @@ +"""requests.status_codes (bashkit): `codes.ok`, `codes.not_found`, ...""" + +from http import HTTPStatus + +from .structures import LookupDict + +codes = LookupDict(name="status_codes") +for _s in HTTPStatus: + _name = _s.name.lower() + setattr(codes, _name, _s.value) + setattr(codes, _s.name, _s.value) +for _alias, _value in { + "ok": 200, "okay": 200, "all_ok": 200, "all_good": 200, "\\o/": 200, + "found": 302, "moved": 301, "temporary_redirect": 307, "permanent_redirect": 308, + "bad": 400, "unauthorized": 401, "forbidden": 403, "not_found": 404, + "not_allowed": 405, "conflict": 409, "teapot": 418, "too_many_requests": 429, + "server_error": 500, "internal_server_error": 500, "unavailable": 503, +}.items(): + setattr(codes, _alias, _value) +del _s, _name, _alias, _value diff --git a/crates/bashkit-cpython-wasm/guest/requests/structures.py b/crates/bashkit-cpython-wasm/guest/requests/structures.py new file mode 100644 index 000000000..67cc959f5 --- /dev/null +++ b/crates/bashkit-cpython-wasm/guest/requests/structures.py @@ -0,0 +1,18 @@ +"""requests.structures (bashkit).""" + +from _bashkit_webcore import CaseInsensitiveDict + + +class LookupDict(dict): + def __init__(self, name=None): + self.name = name + super().__init__() + + def __getitem__(self, key): + return self.__dict__.get(key, None) + + def get(self, key, default=None): + return self.__dict__.get(key, default) + + +__all__ = ["CaseInsensitiveDict", "LookupDict"] diff --git a/crates/bashkit-cpython-wasm/guest/requests/utils.py b/crates/bashkit-cpython-wasm/guest/requests/utils.py new file mode 100644 index 000000000..109eff419 --- /dev/null +++ b/crates/bashkit-cpython-wasm/guest/requests/utils.py @@ -0,0 +1,19 @@ +"""requests.utils (bashkit): the helpers scripts reach for.""" + +from urllib.parse import quote, unquote, urlparse # noqa: F401 + +from .sessions import default_headers, default_user_agent # noqa: F401 + + +def dict_from_cookiejar(cj): + return dict(cj) + + +def add_dict_to_cookiejar(cj, cookie_dict): + for k, v in cookie_dict.items(): + cj[k] = v + return cj + + +def requote_uri(uri): + return quote(unquote(uri), safe="!#$%&'()*+,/:;=?@[]~") diff --git a/crates/bashkit-cpython-wasm/guest/tests/differential.py b/crates/bashkit-cpython-wasm/guest/tests/differential.py new file mode 100644 index 000000000..6aa1bebb5 --- /dev/null +++ b/crates/bashkit-cpython-wasm/guest/tests/differential.py @@ -0,0 +1,379 @@ +"""Differential test: bashkit's requests/httpx modules vs upstream. + +Runs every scenario twice on the host CPython, once with bashkit's modules +(guest/requests, guest/httpx.py) over a fake `_bashkit.http`, once with the +real `requests` and `httpx` packages whose transport is patched to call the +same fake server. Both sides must print the same output. This catches API +drift; the bridge itself is covered by +crates/bashkit/tests/integration/cpython_http_tests.rs. + +Usage: python3 differential.py (needs `pip install requests httpx`) +""" + +import json +import os +import subprocess +import sys + +GUEST = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +# --- the fake server, shared by both sides --------------------------------- + +SERVER = r''' +import gzip, json +from urllib.parse import urlsplit + +HOST_OWNED = {"host", "content-length", "transfer-encoding", "connection", + "keep-alive", "upgrade", "te", "trailer", "proxy-connection", + "proxy-authorization"} + +class Timeout(Exception): + pass + +class Denied(Exception): + pass + +def route(method, url, headers, body): + """What the bashkit host + a test server would answer.""" + headers = [(k, v) for k, v in headers if k.lower() not in HOST_OWNED] + p = urlsplit(url) + if p.hostname == "denied.test": + raise Denied("access denied: URL not in allowlist") + path = p.path + status, rheaders, rbody = _route(method, p, path, headers, body) + return status, rheaders, (b"" if method == "HEAD" else rbody) + +def _route(method, p, path, headers, body): + if path == "/json": + return 200, [("Content-Type", "application/json")], json.dumps({"ok": True, "q": p.query}).encode() + if path == "/echo": + return 200, [("Content-Type", "text/plain; charset=utf-8")], method.encode() + b" " + (body or b"") + if path == "/headers": + seen = {k.lower(): v for k, v in headers if k.lower() not in ("user-agent", "accept-encoding", "accept")} + return 200, [("Content-Type", "application/json")], json.dumps(seen, sort_keys=True).encode() + if path == "/gzip": + return 200, [("Content-Encoding", "gzip"), ("Content-Type", "text/plain")], gzip.compress(b"zipped") + if path.startswith("/status/"): + return int(path.split("/")[2]), [("Content-Type", "text/plain")], b"status body" + if path == "/redirect": + return 302, [("Location", "/json")], b"" + if path == "/see-other": + return 303, [("Location", "/echo")], b"" + if path == "/temp": + return 307, [("Location", "/echo")], b"" + if path == "/loop": + return 302, [("Location", "/loop")], b"" + if path == "/cross": + return 302, [("Location", "http://other.test/headers")], b"" + if path == "/redirect-out": + return 302, [("Location", "http://denied.test/x")], b"" + if path == "/cookie": + return 200, [("Set-Cookie", "sid=abc; Path=/"), ("Set-Cookie", "t=1")], b"" + if path == "/latin": + return 200, [("Content-Type", "text/plain")], b"caf\xe9" + if path == "/lines": + return 200, [("Content-Type", "text/plain; charset=utf-8")], b"a\nb\nc" + if path == "/link": + return 200, [("Link", '; rel="next"')], b"" + if path == "/slow": + raise Timeout("timed out") + return 200, [("Content-Type", "text/plain")], b"hello" +''' + +BASHKIT_SIDE = r''' +import sys, types +sys.path.insert(0, GUEST) +server = types.ModuleType("server"); exec(SERVER, server.__dict__) +fake = types.ModuleType("_bashkit") +def http(method, url, headers, body, timeout): + try: + return server.route(method, url, headers, body) + except server.Timeout as e: + raise TimeoutError(str(e)) + except server.Denied as e: + raise ConnectionError(str(e)) +fake.http = http +sys.modules["_bashkit"] = fake +''' + +UPSTREAM_SIDE = r''' +import http.client, io, sys, types +from http import HTTPStatus +server = types.ModuleType("server"); exec(SERVER, server.__dict__) +import requests, requests.adapters, urllib3 +import httpx + +def _requests_send(self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None): + headers = list(request.headers.items()) + body = request.body.encode() if isinstance(request.body, str) else request.body + try: + status, rheaders, rbody = server.route(request.method, request.url, headers, body) + except server.Timeout as e: + raise requests.ReadTimeout(str(e), request=request) + except server.Denied as e: + raise requests.ConnectionError(str(e), request=request) + hd = urllib3.HTTPHeaderDict() + for k, v in rheaders: + hd.add(k, v) + resp = urllib3.HTTPResponse(body=io.BytesIO(rbody), headers=hd, status=status, + reason=HTTPStatus(status).phrase, preload_content=False, + decode_content=True, request_method=request.method) + # What http.client would expose, so requests can read Set-Cookie. + msg = http.client.HTTPMessage() + for k, v in rheaders: + msg[k] = v + resp._original_response = types.SimpleNamespace(msg=msg, info=lambda: msg, isclosed=lambda: True, close=lambda: None) + return self.build_response(request, resp) + +requests.adapters.HTTPAdapter.send = _requests_send + +def _httpx_handle(self, request): + # httpcore's check, which this patch bypasses. + if request.url.scheme not in ("http", "https"): + raise httpx.UnsupportedProtocol("unsupported protocol", request=request) + try: + status, rheaders, rbody = server.route(request.method, str(request.url), + list(request.headers.multi_items()), request.read()) + except server.Timeout as e: + raise httpx.ReadTimeout(str(e), request=request) + except server.Denied as e: + raise httpx.ConnectError(str(e), request=request) + return httpx.Response(status, headers=rheaders, stream=httpx.ByteStream(rbody), request=request) + +async def _httpx_handle_async(self, request): + return _httpx_handle(self, request) + +httpx.HTTPTransport.handle_request = _httpx_handle +httpx.AsyncHTTPTransport.handle_async_request = _httpx_handle_async +''' + +# --- scenarios ------------------------------------------------------------ + +SCENARIOS = { + "requests_basics": r''' +import requests +r = requests.get("https://api.test/json", params={"q": "a b", "n": [1, 2]}) +print(r.status_code, r.ok, r.reason, r.headers["content-type"], r.encoding, r.json(), r.url) +print(requests.post("http://api.test/echo", json={"k": 1, "u": "é"}).text) +print(requests.post("http://api.test/echo", data={"a": "1 2", "b": ["x", "y"]}).text) +print(requests.put("http://api.test/echo", data=b"raw").text, requests.patch("http://api.test/echo", data="s").text) +print(requests.delete("http://api.test/echo").text) +r = requests.head("http://api.test/json"); print(r.status_code, repr(r.text)) +print(requests.get("http://api.test/headers", headers={"X-A": "1"}, auth=("u", "p")).json()) +print(requests.get("http://api.test/headers", cookies={"c": "1"}).json()) +''', + "requests_errors": r''' +import requests +for code in (404, 500): + r = requests.get(f"http://api.test/status/{code}") + print(r.status_code, r.ok, bool(r)) + try: + r.raise_for_status() + except requests.HTTPError as e: + print(type(e).__name__, e, e.response.status_code) +for url in ("api.test/x", "ftp://api.test/x", "http:///x"): + try: + requests.get(url) + except requests.RequestException as e: + print(type(e).__name__, isinstance(e, ValueError)) +try: + requests.get("http://api.test/slow", timeout=1) +except requests.Timeout as e: + print(type(e).__name__, isinstance(e, requests.ConnectionError)) +try: + requests.get("http://denied.test/") +except requests.ConnectionError as e: + print(type(e).__name__) +try: + requests.get("http://api.test/plain").json() +except requests.JSONDecodeError as e: + print(type(e).__name__, isinstance(e, ValueError)) +print(issubclass(requests.ReadTimeout, requests.Timeout), issubclass(requests.ConnectTimeout, requests.ConnectionError)) +''', + "requests_redirects": r''' +import requests +r = requests.get("http://api.test/redirect") +print(r.status_code, r.url, [h.status_code for h in r.history], r.history[0].is_redirect) +r = requests.get("http://api.test/redirect", allow_redirects=False) +print(r.status_code, r.is_redirect, r.headers["location"]) +print(repr(requests.post("http://api.test/see-other", data="x").text)) +print(requests.post("http://api.test/temp", data="x").text) +try: + requests.get("http://api.test/loop") +except requests.TooManyRedirects as e: + print(type(e).__name__) +print(requests.get("http://api.test/cross", auth=("u", "p")).json()) +try: + requests.get("http://api.test/redirect-out") +except requests.ConnectionError as e: + print(type(e).__name__) +''', + "requests_session_and_body": r''' +import requests +s = requests.Session() +s.headers.update({"X-S": "1"}) +s.params = {"p": "1"} +s.get("http://api.test/cookie") +print(sorted(s.cookies.items()), s.cookies.get("sid")) +print(s.get("http://api.test/headers").json()) +print(requests.get("http://api.test/json").json(), s.get("http://api.test/json").json()) +print(requests.get("http://api.test/gzip").text) +r = requests.get("http://api.test/latin"); print(r.encoding, r.text) +r = requests.get("http://api.test/lines") +print(list(r.iter_lines()), list(r.iter_lines(decode_unicode=True)), list(r.iter_content(2))) +print(requests.get("http://api.test/link").links) +print(requests.codes.ok, requests.codes.not_found, requests.codes["teapot"]) +seen = [] +requests.get("http://api.test/plain", hooks={"response": lambda r, *a, **k: seen.append(r.status_code)}) +print(seen) +with requests.Session() as s2: + print(s2.get("http://api.test/plain").text) +''', + "httpx_basics": r''' +import httpx +r = httpx.get("https://api.test/json", params={"q": "a b"}) +print(r, r.status_code, r.reason_phrase, r.http_version, r.headers["content-type"], r.json(), r.url, r.is_success) +print(httpx.post("http://api.test/echo", json={"k": 1, "u": "é"}).text) +print(httpx.post("http://api.test/echo", data={"a": "1 2"}).text) +print(httpx.post("http://api.test/echo", content=b"raw").text, httpx.put("http://api.test/echo", content="s").text) +print(httpx.get("http://api.test/headers", headers={"X-A": "1"}, auth=("u", "p")).json()) +print(httpx.get("http://api.test/gzip").text) +r = httpx.get("http://api.test/lines") +print(list(r.iter_lines()), list(r.iter_bytes(2)), r.encoding) +print(httpx.get("http://api.test/link").links) +print(httpx.codes.OK, httpx.codes.NOT_FOUND, httpx.Timeout(10.0), httpx.Timeout(5.0, connect=1.0)) +u = httpx.URL("https://a.test/p?x=1&y=2") +print(u.host, u.path, u.query, u.params["y"], u.scheme, u.join("/q"), u.copy_merge_params({"z": 3})) +h = httpx.Headers([("A", "1"), ("a", "2")]) +print(h["a"], h.get_list("A"), list(h.multi_items())) +q = httpx.QueryParams({"a": ["1", "2"], "b": "x"}) +print(str(q), q.get_list("a"), q["a"]) +''', + "httpx_errors_and_redirects": r''' +import httpx +r = httpx.get("http://api.test/redirect") +print(r.status_code, r.is_redirect, r.has_redirect_location, r.next_request.url) +for code in (404, 500, 302): + path = "redirect" if code == 302 else f"status/{code}" + try: + httpx.get(f"http://api.test/{path}").raise_for_status() + except httpx.HTTPStatusError as e: + print(type(e).__name__, e.response.status_code, str(e).splitlines()[0]) +with httpx.Client(follow_redirects=True) as c: + r = c.get("http://api.test/redirect") + print(r.status_code, r.url, [h.status_code for h in r.history]) + print(repr(c.post("http://api.test/see-other", content=b"x").text)) + print(c.post("http://api.test/temp", content=b"x").text) + print(c.get("http://api.test/cross", auth=("u", "p")).json()) + try: + c.get("http://api.test/loop") + except httpx.TooManyRedirects as e: + print(type(e).__name__) +for url in ("api.test/x", "ftp://api.test/x"): + try: + httpx.get(url) + except httpx.HTTPError as e: + print(type(e).__name__) +try: + httpx.get("http://api.test/slow") +except httpx.TimeoutException as e: + print(type(e).__name__, isinstance(e, httpx.TransportError)) +try: + httpx.get("http://denied.test/") +except httpx.ConnectError as e: + print(type(e).__name__, e.request.url) +''', + "httpx_client_and_async": r''' +import asyncio, httpx +with httpx.Client(base_url="http://api.test/", headers={"X-C": "1"}, params={"p": "1"}) as c: + c.get("/cookie") + print(dict(c.cookies), c.base_url) + print(c.get("headers").json()) + print(c.get("json").json()) + with c.stream("GET", "/lines") as r: + print(list(r.iter_lines())) +seen = [] +with httpx.Client(event_hooks={"request": [lambda r: seen.append(r.method)], + "response": [lambda r: seen.append(r.status_code)]}) as c: + c.get("http://api.test/plain") +print(seen) +def handler(request): + return httpx.Response(201, json={"path": request.url.path, "m": request.method}) +with httpx.Client(transport=httpx.MockTransport(handler)) as c: + r = c.post("https://api.test/items") + print(r.status_code, r.json(), r.headers["content-type"]) +async def main(): + async with httpx.AsyncClient() as c: + rs = await asyncio.gather(c.get("http://api.test/json"), c.post("http://api.test/echo", data={"a": "b"})) + print([r.status_code for r in rs], rs[1].text) + async with c.stream("GET", "http://api.test/lines") as r: + print([line async for line in r.aiter_lines()]) +asyncio.run(main()) +''', + "requests_misc": r''' +import io, datetime, requests +r = requests.get("http://api.test/json", timeout=(1, 2)) +print(repr(r), isinstance(r.elapsed, datetime.timedelta), r.request.method, r.request.url) +s = requests.Session() +p = s.prepare_request(requests.Request("POST", "http://api.test/echo", json={"a": 1}, headers={"X": "1"})) +print(p.method, p.url, p.body, p.headers["Content-Type"], p.headers["X"]) +print(s.send(p).text) +print(requests.post("http://api.test/echo", data="plain text").text) +r = requests.post("http://api.test/echo", files={"f": io.BytesIO(b"FILEDATA")}, data={"k": "v"}) +print("FILEDATA" in r.text, 'name="k"' in r.text, 'filename="f"' in r.text) +print(requests.get("http://api.test/headers", headers={"X-Drop": None, "X-Keep": "1"}).json()) +print(requests.get("http://api.test/json", params="raw=1").url) +print(requests.get("http://api.test/json", params={"none": None, "t": True}).url) +print(requests.structures.CaseInsensitiveDict({"A": 1})["a"], requests.exceptions.HTTPError.__mro__[1].__name__) +''', + "httpx_misc": r''' +import httpx +r = httpx.get("http://api.test/json") +print(r.request.method, r.request.url, r.request.headers["host"] if "host" in r.request.headers else "-", r.elapsed.total_seconds() >= 0) +r = httpx.Response(200, text="hi") +print(r.text, r.headers["content-type"], r.encoding) +req = httpx.Request("POST", "https://api.test/x", json={"a": 1}) +print(req.method, req.url, req.content, req.headers["content-type"], req.headers["content-length"]) +c = httpx.Client(timeout=httpx.Timeout(3.0)) +print(c.timeout, c.follow_redirects, c.headers["user-agent"].startswith("python-httpx/")) +print(c.build_request("GET", "http://api.test/p", params={"a": 1}).url) +print(httpx.post("http://api.test/echo", files={"f": ("n.txt", b"DATA", "text/plain")}).text.count("DATA")) +cookies = httpx.Cookies({"a": "1"}); cookies.set("b", "2", domain="api.test") +print(sorted(cookies.items()), cookies.get("b")) +print(httpx.get("http://api.test/headers", cookies={"c": "1"}).json()) +''', +} + + +def run(side, code): + prelude = (f"GUEST = {GUEST!r}\nSERVER = {SERVER!r}\n" + + (BASHKIT_SIDE if side == "bashkit" else UPSTREAM_SIDE)) + proc = subprocess.run([sys.executable, "-I", "-c", prelude + code], + capture_output=True, text=True, timeout=60) + out = proc.stdout + if proc.returncode: + out += "EXIT %d: %s" % (proc.returncode, proc.stderr.strip().splitlines()[-1:]) + return out + + +def main(): + failed = 0 + for name, code in SCENARIOS.items(): + ours, theirs = run("bashkit", code), run("upstream", code) + if ours == theirs: + print(f"ok {name}") + continue + failed += 1 + print(f"FAIL {name}") + a, b = ours.splitlines(), theirs.splitlines() + for i in range(max(len(a), len(b))): + x = a[i] if i < len(a) else "" + y = b[i] if i < len(b) else "" + if x != y: + print(f" line {i + 1}\n bashkit: {x}\n upstream: {y}") + print(json.dumps({"scenarios": len(SCENARIOS), "failed": failed})) + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/crates/bashkit/benches/python.rs b/crates/bashkit/benches/python.rs index 1b8b4416a..36a8dc97b 100644 --- a/crates/bashkit/benches/python.rs +++ b/crates/bashkit/benches/python.rs @@ -101,6 +101,8 @@ fn bench_call(c: &mut Criterion) { const IMPORTS: &[(&str, &str)] = &[ ("email_message", "python3 -c 'import email.message'"), ("http_client", "python3 -c 'import http.client'"), + // bashkit's own requests/httpx, preloaded in the snapshot. + ("requests_httpx", "python3 -c 'import requests, httpx'"), ]; fn bench_import(c: &mut Criterion) { diff --git a/crates/bashkit/docs/cpython.md b/crates/bashkit/docs/cpython.md index 29ef37cc3..8043c7d23 100644 --- a/crates/bashkit/docs/cpython.md +++ b/crates/bashkit/docs/cpython.md @@ -149,7 +149,44 @@ with urllib.request.urlopen("https://api.example.com/v1/items") as r: - `CPythonLimits::max_http_requests` caps requests per call (default 100); a request's timeout never outlasts the call's deadline. -`requests` and `httpx` are not bundled yet. +### `requests` and `httpx` + +`import requests`, `import httpx` and `import httpx2` work out of the box. +They are Bashkit's own compact implementations of the common API, written +on top of the same host bridge (not the upstream packages, which take +seconds to import in the sandbox). They are preloaded in the snapshot, so +importing them costs nothing. + +```python +import requests +r = requests.get("https://api.example.com/v1/items", params={"page": 2}, timeout=5) +r.raise_for_status() +print(r.json()) + +import httpx +with httpx.Client(base_url="https://api.example.com", headers={"X-Key": "..."}) as c: + print(c.post("/v1/items", json={"name": "a"}).status_code) +``` + +- **requests**: `get`/`post`/`put`/`patch`/`delete`/`head`/`request`, + `Session` (headers, params, auth, cookies, hooks), `params`, `data`, + `json`, `files` (multipart), `headers`, `cookies`, basic `auth`, + `timeout`, `allow_redirects`; `Response` with `status_code`, `ok`, + `reason`, `headers`, `content`, `text`, `json()`, `url`, `history`, + `links`, `iter_content`/`iter_lines`, `raise_for_status()`; the upstream + exception classes (`ConnectionError`, `ReadTimeout`, `HTTPError`, ...). +- **httpx** (and `httpx2`, the same module): the verb functions and + `stream()`, `Client` and `AsyncClient` (`base_url`, `headers`, `params`, + `cookies`, `auth`, `timeout`, `follow_redirects`, `event_hooks`, + `transport=httpx.MockTransport(...)`), `Response`, `URL`, `Headers`, + `QueryParams`, `Cookies`, `Timeout`, `BasicAuth`, `codes` and the + upstream exception classes. As in httpx, redirects are not followed + unless `follow_redirects=True`. +- Not supported: retries (`HTTPAdapter(max_retries=...)` is accepted and + ignored), proxies, client certificates, HTTP/2, `OPTIONS` (not an allowed + method), digest auth, streaming uploads. Bodies are buffered, so + `stream=True` and `iter_*` walk a body that is already complete. + `AsyncClient` requests run one at a time. ## Limitations @@ -160,8 +197,9 @@ with urllib.request.urlopen("https://api.example.com/v1/items") as r: - **No threads**: `threading.Thread.start()` raises `RuntimeError`; `multiprocessing` and `concurrent.futures.ProcessPoolExecutor` are absent. `asyncio` works. -- **No native extensions or pip**: only the bundled stdlib. `ctypes`, - `numpy`, `requests` and other third-party packages are unavailable; `ssl`, +- **No native extensions or pip**: only the bundled stdlib (plus Bashkit's + own `requests`/`httpx`, see [HTTP](#http)). `ctypes`, `numpy` and other + third-party packages are unavailable; `ssl`, `_hashlib` (OpenSSL), `tkinter`, `curses`, `readline`, `dbm.gnu` are not built. `hashlib` still provides md5, sha1, sha2, sha3 and blake2. - **No interactive mode**: `python3` with no program reads one from stdin; diff --git a/crates/bashkit/src/builtins/cpython/mod.rs b/crates/bashkit/src/builtins/cpython/mod.rs index 0555fa138..35e7f869f 100644 --- a/crates/bashkit/src/builtins/cpython/mod.rs +++ b/crates/bashkit/src/builtins/cpython/mod.rs @@ -245,9 +245,9 @@ impl Builtin for CPython { "python/python3: CPython 3.14 in a WebAssembly sandbox. Full pure-Python \ stdlib plus json, re, csv, sqlite3, zlib, hashlib, decimal, datetime. \ open()/pathlib/os work on the virtual filesystem. -c, -m, script files and \ - stdin work. HTTP only via urllib.request/http.client and only to hosts the \ - network allowlist permits; no raw sockets, no subprocess, no threads, no pip \ - or third-party packages.", + stdin work. HTTP via requests, httpx (common API subsets) or \ + urllib.request, only to hosts the network allowlist permits; no raw sockets, \ + no subprocess, no threads, no pip or other third-party packages.", ) } diff --git a/crates/bashkit/src/lib.rs b/crates/bashkit/src/lib.rs index 127cdf194..16e7d3328 100644 --- a/crates/bashkit/src/lib.rs +++ b/crates/bashkit/src/lib.rs @@ -4121,7 +4121,8 @@ pub mod python_guide {} /// - Why CPython instead of Monty /// - Supported command line, stdio, environment and VFS behavior /// - Resource limits via [`CPythonLimits`] -/// - Limitations (no subprocess, network, threads or third-party packages) +/// - HTTP through the egress pipeline, including `requests` and `httpx` +/// - Limitations (no subprocess, raw sockets, threads or other third-party packages) /// /// **Related:** [`BashBuilder::cpython`], [`CPythonLimits`], [`CPython`], [`threat_model`] #[cfg(feature = "cpython")] diff --git a/crates/bashkit/tests/integration/cpython_capability_tests.rs b/crates/bashkit/tests/integration/cpython_capability_tests.rs index ce22b0880..fe3d6fb48 100644 --- a/crates/bashkit/tests/integration/cpython_capability_tests.rs +++ b/crates/bashkit/tests/integration/cpython_capability_tests.rs @@ -457,8 +457,9 @@ const CASES: &[Case] = &[ ), sandbox_only( "third_party_unavailable", - "try:\n import requests\nexcept ImportError:\n print('no requests')", - "no requests\n", + // requests/httpx are bashkit's own modules; other packages are absent. + "try:\n import numpy\nexcept ImportError:\n print('no numpy')", + "no numpy\n", ), ]; diff --git a/crates/bashkit/tests/integration/cpython_http_tests.rs b/crates/bashkit/tests/integration/cpython_http_tests.rs index 1e0e0eaa1..48dbf83ec 100644 --- a/crates/bashkit/tests/integration/cpython_http_tests.rs +++ b/crates/bashkit/tests/integration/cpython_http_tests.rs @@ -61,6 +61,28 @@ impl HttpTransport for FakeServer { "/missing" => reply(404, &[("Content-Type", "text/plain")], b"nope"), "/redirect" => reply(302, &[("Location", "/json")], b""), "/redirect-out" => reply(302, &[("Location", "http://203.0.113.9/x")], b""), + "/see-other" => reply(303, &[("Location", "/echo")], b""), + "/temp" => reply(307, &[("Location", "/echo")], b""), + "/loop" => reply(302, &[("Location", "/loop")], b""), + "/cross" => reply(302, &[("Location", "http://93.184.216.35/plain")], b""), + "/gzip" => { + use std::io::Write; + let mut gz = + flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + gz.write_all(b"zipped").unwrap(); + reply(200, &[("Content-Encoding", "gzip")], &gz.finish().unwrap()) + } + "/latin" => reply(200, &[("Content-Type", "text/plain")], b"caf\xe9"), + "/lines" => reply( + 200, + &[("Content-Type", "text/plain; charset=utf-8")], + b"a\nb\nc", + ), + "/cookie" => reply( + 200, + &[("Set-Cookie", "sid=abc; Path=/"), ("Set-Cookie", "t=1")], + b"", + ), "/slow" => { tokio::time::sleep(Duration::from_secs(10)).await; reply(200, &[], b"late") @@ -413,11 +435,12 @@ async fn request_timeout_raises() { &mut bash, &format!( "python3 -c ' -import urllib.request, urllib.error +import urllib.request try: urllib.request.urlopen(\"http://{HOST}/slow\", timeout=0.3) -except urllib.error.URLError as e: - print(type(e.reason).__name__, e.reason) +except TimeoutError as e: + # Like CPython: a read timeout is not wrapped in URLError. + print(type(e).__name__, e) '" ), ) @@ -462,3 +485,427 @@ except urllib.error.URLError as e: .await; assert_eq!(r.stdout, "network access not configured\n", "{}", r.stderr); } + +/// Runs a Python script (HOST replaced) from a file, through `run`. +async fn py(bash: &mut Bash, script: &str) -> bashkit::ExecResult { + let script = script.replace("HOST", HOST); + run( + bash, + &format!("cat > /t.py <<'EOF'\n{script}\nEOF\npython3 /t.py"), + ) + .await +} + +/// bashkit's own `requests` (guest/requests/) over the bridge. +mod requests_lib { + use super::*; + + #[tokio::test] + async fn get_json_post_and_errors() { + let (builder, log) = server(allow_host()); + let mut bash = builder.build(); + let r = py( + &mut bash, + r#" +import requests +r = requests.get("https://HOST/json", params={"q": "a b"}, timeout=5) +print(r.status_code, r.headers["content-type"], r.json()) +print(requests.post("http://HOST/echo", json={"k": 1}).text) +r = requests.get("http://HOST/missing") +print(r.status_code, r.ok, r.reason, r.text) +try: + r.raise_for_status() +except requests.HTTPError as e: + print(e) +s = requests.Session() +print([s.get("http://HOST/plain").text for _ in range(2)]) +r = requests.get("http://HOST/redirect") +print(r.json(), len(r.history), r.url) +try: + requests.get("http://HOST/redirect-out") +except requests.ConnectionError as e: + print("denied", "access denied" in str(e)) +try: + requests.get("HOST/x") +except requests.exceptions.MissingSchema: + print("missing schema") +"#, + ) + .await; + assert_eq!(r.exit_code, 0, "{}", r.stderr); + assert_eq!( + r.stdout, + format!( + "200 application/json {{'ok': True}}\n\ + POST {{\"k\": 1}}\n\ + 404 False Not Found nope\n\ + 404 Client Error: Not Found for url: http://{HOST}/missing\n\ + ['hello', 'hello']\n\ + {{'ok': True}} 1 http://{HOST}/json\n\ + denied True\n\ + missing schema\n" + ) + ); + let log = log.lock().unwrap(); + assert_eq!(log[0].url, format!("https://{HOST}/json?q=a+b")); + assert!( + header(&log[0], "User-Agent") + .unwrap() + .starts_with("python-requests/") + ); + assert_eq!(header(&log[1], "Content-Type"), Some("application/json")); + assert!(log.iter().all(|r| !r.url.contains("203.0.113.9"))); + } + + #[tokio::test] + async fn form_multipart_put_auth_and_cookies() { + let (builder, log) = server(allow_host()); + let mut bash = builder.build(); + let r = py( + &mut bash, + r#" +import requests +print(requests.post("http://HOST/echo", data={"a": "1 2"}).text) +r = requests.post("http://HOST/echo", files={"f": ("x.txt", b"DATA")}) +print("DATA" in r.text, 'filename="x.txt"' in r.text) +print(requests.put("http://HOST/echo", data=b"raw", headers={"X-K": "v"}).text) +s = requests.Session() +s.get("http://HOST/cookie") +print(sorted(s.cookies.items())) +s.get("http://HOST/plain", auth=("u", "p")) +"#, + ) + .await; + assert_eq!(r.exit_code, 0, "{}", r.stderr); + assert_eq!( + r.stdout, + "POST a=1+2\nTrue True\nPUT raw\n[('sid', 'abc'), ('t', '1')]\n" + ); + let log = log.lock().unwrap(); + assert_eq!( + header(&log[0], "Content-Type"), + Some("application/x-www-form-urlencoded") + ); + assert!( + header(&log[1], "Content-Type") + .unwrap() + .starts_with("multipart/form-data; boundary=") + ); + assert_eq!(header(&log[2], "X-K"), Some("v")); + assert_eq!(header(&log[4], "Cookie"), Some("sid=abc; t=1")); + assert_eq!(header(&log[4], "Authorization"), Some("Basic dTpw")); + } + + /// Cookies a server sets go back only to that host, and credentials + /// are dropped when a redirect leaves the host. + #[tokio::test] + async fn cookies_and_auth_stay_on_their_host() { + let other = "93.184.216.35"; + let allow = allow_host() + .allow(format!("http://{other}")) + .allow(format!("http://{HOST}")); + let (builder, log) = server(allow); + let mut bash = builder.build(); + let r = py( + &mut bash, + &r#" +import requests +s = requests.Session() +s.get("http://HOST/cookie") +s.get("http://OTHER/plain") +"# + .replace("OTHER", other), + ) + .await; + assert_eq!(r.exit_code, 0, "{}", r.stderr); + let log = log.lock().unwrap(); + assert_eq!(header(&log[1], "Cookie"), None); + } + + /// 303 turns into GET without a body, 307 keeps method and body, + /// loops end in TooManyRedirects; gzip, charset fallback, line + /// iteration, response hooks and JSON errors. + #[tokio::test] + async fn redirect_rules_bodies_and_helpers() { + let (builder, _log) = server(allow_host()); + let mut bash = builder.build(); + let r = py( + &mut bash, + r#" +import requests +print(repr(requests.post("http://HOST/see-other", data="x").text)) +print(requests.post("http://HOST/temp", data="x").text) +try: + requests.get("http://HOST/loop") +except requests.TooManyRedirects: + print("too many") +print(requests.get("http://HOST/gzip").text) +r = requests.get("http://HOST/latin") +print(r.encoding, r.text) +r = requests.get("http://HOST/lines") +print(list(r.iter_lines(decode_unicode=True)), list(r.iter_content(2))) +requests.get("http://HOST/plain", hooks={"response": lambda r, *a, **k: print("hook", r.status_code)}) +try: + requests.get("http://HOST/plain").json() +except requests.JSONDecodeError as e: + print("json error", isinstance(e, ValueError)) +"#, + ) + .await; + assert_eq!(r.exit_code, 0, "{}", r.stderr); + assert_eq!( + r.stdout, + "'GET '\nPOST x\ntoo many\nzipped\nISO-8859-1 café\n\ + ['a', 'b', 'c'] [b'a\\n', b'b\\n', b'c']\nhook 200\njson error True\n" + ); + } + + /// Credentials never follow a redirect to another host; bad headers + /// and the per-call request cap surface as requests exceptions. + #[tokio::test] + async fn cross_host_auth_crlf_and_request_cap() { + let log = Log::default(); + let mut bash = Bash::builder() + .cpython_with_limits(CPythonLimits::default().max_http_requests(3)) + .network(allow_host().allow("http://93.184.216.35")) + .http_transport(Arc::new(FakeServer { log: log.clone() })) + .build(); + let r = py( + &mut bash, + r#" +import requests +print(requests.get("http://HOST/cross", auth=("u", "p")).status_code) +try: + requests.get("http://HOST/plain", headers={"X": "a\r\nInjected: 1"}) +except requests.exceptions.InvalidHeader as e: + print("invalid header:", e) +requests.get("http://HOST/plain") +try: + requests.get("http://HOST/plain") +except requests.ConnectionError as e: + print(e) +"#, + ) + .await; + assert_eq!(r.exit_code, 0, "{}", r.stderr); + assert_eq!( + r.stdout, + "200\ninvalid header: invalid header value\n\ + too many HTTP requests (limit 3 per call)\n" + ); + let log = log.lock().unwrap(); + assert_eq!(header(&log[0], "Authorization"), Some("Basic dTpw")); + assert_eq!(log[1].url, "http://93.184.216.35/plain"); + assert_eq!(header(&log[1], "Authorization"), None); + assert_eq!(log.len(), 3); + } + + #[tokio::test] + async fn timeout_and_no_network() { + let (builder, _log) = server(allow_host()); + let mut bash = builder.build(); + let r = py( + &mut bash, + r#" +import requests +try: + requests.get("http://HOST/slow", timeout=0.3) +except requests.ReadTimeout: + print("timeout") +"#, + ) + .await; + assert_eq!(r.stdout, "timeout\n", "{}", r.stderr); + + let mut bash = Bash::builder().cpython().build(); + let r = py( + &mut bash, + r#" +import requests +try: + requests.get("https://example.com/") +except requests.ConnectionError as e: + print("network access not configured" in str(e)) +"#, + ) + .await; + assert_eq!(r.stdout, "True\n", "{}", r.stderr); + } + + /// The point of bashkit's own module: importing it pulls in none of the + /// heavy stdlib HTTP stack, and it is already loaded in the snapshot. + #[tokio::test] + async fn import_is_light() { + let mut bash = Bash::builder().cpython().build(); + let r = py( + &mut bash, + r#" +import sys +before = set(sys.modules) +import requests, httpx, httpx2 +heavy = {"http.client", "email", "urllib3", "ssl", "http.cookiejar"} +print(sorted(set(sys.modules) - before), sorted(heavy & set(sys.modules))) +"#, + ) + .await; + assert_eq!(r.stdout, "[] []\n", "{}", r.stderr); + } +} + +/// bashkit's own `httpx` (guest/httpx.py; `httpx2` is the same module). +mod httpx_lib { + use super::*; + + #[tokio::test] + async fn client_basics_and_redirects() { + let (builder, log) = server(allow_host()); + let mut bash = builder.build(); + let r = py( + &mut bash, + r#" +import httpx, httpx2 +print(httpx2 is httpx) +r = httpx.get("https://HOST/json", params={"q": "a b"}) +print(r, r.json(), r.headers["content-type"], r.headers.get_list("x-multi")) +print(httpx.post("http://HOST/echo", json={"k": 1}).text) +print(httpx.post("http://HOST/echo", content=b"raw").text) +r = httpx.get("http://HOST/redirect") +print(r.status_code, r.next_request.url) +with httpx.Client(base_url="http://HOST", headers={"X-A": "1"}, auth=("u", "p"), + follow_redirects=True) as c: + r = c.get("/redirect") + print(r.url, len(r.history)) + c.get("/cookie") + c.get("/plain") + print(dict(c.cookies)) +try: + httpx.get("http://HOST/missing").raise_for_status() +except httpx.HTTPStatusError as e: + print(e.response.status_code, str(e).splitlines()[0]) +try: + httpx.get("http://HOST/redirect-out", follow_redirects=True) +except httpx.ConnectError as e: + print("denied", "access denied" in str(e)) +try: + httpx.get("ftp://HOST/") +except httpx.UnsupportedProtocol as e: + print(e) +"#, + ) + .await; + assert_eq!(r.exit_code, 0, "{}", r.stderr); + assert_eq!( + r.stdout, + format!( + "True\n\ + {{'ok': True}} application/json ['a', 'b']\n\ + POST {{\"k\":1}}\n\ + POST raw\n\ + 302 http://{HOST}/json\n\ + http://{HOST}/json 1\n\ + {{'sid': 'abc', 't': '1'}}\n\ + 404 Client error '404 Not Found' for url 'http://{HOST}/missing'\n\ + denied True\n\ + Request URL has an unsupported protocol 'ftp://'.\n" + ) + ); + let log = log.lock().unwrap(); + assert_eq!(log[0].url, format!("https://{HOST}/json?q=a+b")); + assert!( + header(&log[0], "User-Agent") + .unwrap() + .starts_with("python-httpx/") + ); + let plain = log.iter().rfind(|r| r.url.ends_with("/plain")).unwrap(); + assert_eq!(header(plain, "Cookie"), Some("sid=abc; t=1")); + assert_eq!(header(plain, "X-A"), Some("1")); + assert_eq!(header(plain, "Authorization"), Some("Basic dTpw")); + assert!(log.iter().all(|r| !r.url.contains("203.0.113.9"))); + } + + #[tokio::test] + async fn redirects_gzip_hooks_stream_and_errors() { + let (builder, log) = server(allow_host().allow("http://93.184.216.35")); + let mut bash = builder.build(); + let r = py( + &mut bash, + r#" +import httpx +seen = [] +with httpx.Client(follow_redirects=True, max_redirects=5, + event_hooks={"request": [lambda r: seen.append(r.method)], + "response": [lambda r: seen.append(r.status_code)]}) as c: + print(repr(c.post("http://HOST/see-other", content=b"x").text)) + print(c.post("http://HOST/temp", content=b"x").text) + try: + c.get("http://HOST/loop") + except httpx.TooManyRedirects: + print("too many") + print(c.get("http://HOST/cross", auth=("u", "p")).status_code) + print(c.get("http://HOST/gzip").text) + with c.stream("GET", "http://HOST/lines") as r: + print(list(r.iter_lines()), list(r.iter_bytes(2))) + try: + c.get("http://HOST/plain", headers={"X": "a\r\nInjected: 1"}) + except httpx.LocalProtocolError as e: + print("local:", e) +print(seen[:4]) +"#, + ) + .await; + assert_eq!(r.exit_code, 0, "{}", r.stderr); + assert_eq!( + r.stdout, + "'GET '\nPOST x\ntoo many\n200\nzipped\n\ + ['a', 'b', 'c'] [b'a\\n', b'b\\n', b'c']\nlocal: invalid header value\n\ + ['POST', 303, 'GET', 200]\n" + ); + let log = log.lock().unwrap(); + let cross = log.iter().position(|r| r.url.ends_with("/cross")).unwrap(); + assert_eq!(header(&log[cross], "Authorization"), Some("Basic dTpw")); + assert_eq!(log[cross + 1].url, "http://93.184.216.35/plain"); + assert_eq!(header(&log[cross + 1], "Authorization"), None); + } + + #[tokio::test] + async fn async_client_and_timeout() { + let (builder, _log) = server(allow_host()); + let mut bash = builder.build(); + let r = py( + &mut bash, + r#" +import asyncio, httpx +async def main(): + async with httpx.AsyncClient() as c: + rs = await asyncio.gather(c.get("http://HOST/json"), c.post("http://HOST/echo", data={"a": "b"})) + print([r.status_code for r in rs], rs[1].text) + try: + await c.get("http://HOST/slow", timeout=0.3) + except httpx.ReadTimeout: + print("timeout") +asyncio.run(main()) +"#, + ) + .await; + assert_eq!(r.exit_code, 0, "{}", r.stderr); + assert_eq!(r.stdout, "[200, 200] POST a=b\ntimeout\n"); + } + + #[tokio::test] + async fn mock_transport_needs_no_network() { + let mut bash = Bash::builder().cpython().build(); + let r = py( + &mut bash, + r#" +import httpx +def handler(request): + return httpx.Response(201, json={"path": request.url.path}) +with httpx.Client(transport=httpx.MockTransport(handler)) as c: + r = c.post("https://api.test/items") + print(r.status_code, r.json()) +"#, + ) + .await; + assert_eq!(r.stdout, "201 {'path': '/items'}\n", "{}", r.stderr); + } +} diff --git a/knowledge/log.md b/knowledge/log.md index 18e825251..7fb614d9d 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -3,6 +3,7 @@ ## 2026-10-07 * **Fix**: Quoted or escaped glob characters are literal everywhere. `echo a\*`, `a"*"` and `a'?'` no longer glob; `case`/`[[ == ]]` patterns match `"$x"`, `'a*'`, `a\*` and `\[s\]` literally; `"*"zz*` with no match prints `*zz*` (escapes no longer leak); `"$x"zz*` no longer reads `$xzz`; `'a'$x` expands and `'a'*` globs; `a{1,2}"*"` still brace-expands. The lexer escapes quoted glob characters in mixed words (the existing `QuotedGlobWord` convention, now with quote markers); `expand_word` returns unescaped text for non-glob uses; patterns use `expand_pattern_word`. Known gap: data from an unquoted expansion inside such a word loses `\` before glob characters when nothing matches. +* **Feature**: `import requests`, `import httpx` and `import httpx2` work in CPython. They are bashkit's own compact implementations of the common API over the HTTP bridge, preloaded in the snapshot (import cost ~0 vs ~3.1 s for vendored upstream requests). http.client read timeouts now surface from `getresponse()`. L-CPY-003 updated, L-CPY-010 added. See [CPython WebAssembly Runtime](runtimes/cpython-wasm.md#requests-and-httpx-bashkits-own-modules). * **Fix**: A backslash outside quotes keeps `$` and `` ` `` literal. `echo \$HOME` printed the home directory; it now prints `$HOME`, as do `a\$x`, `\${x}` and `x=\$y` (the lexer emits the same NUL literal marker as quoted `"\$"`). Found while working gap telemetry. * **Fix**: `$'\x1e'` and `$'\x1f'` bytes (record/unit separators) are kept as data. They collided with the lexer's quote-boundary markers and were silently dropped, which broke the common `SEP=$'\x1f'` composite-key idiom for associative arrays (`v["$section"$'\x1f'"$key"]`). Decoded ANSI-C content now NUL-escapes those bytes like `$`. Found by gap telemetry (`config_ini_merge` replay printed nothing). * **Fix**: A usage error in a bundled builtin no longer aborts the script. `grep -E '('`, `grep -P` with a bad pattern, `rg` with a bad regex, an awk syntax error, `grep` with no pattern and `grep -f missing` now fail that one command with exit 2 and the message on stderr, and the script continues (bash semantics; `set -e` still stops). `Err` from custom builtins (`BashBuilder::builtin`, registry, resolver) keeps aborting, per the public contract. See foundations/builtins "Errors: Failed Command vs Aborted Execution". diff --git a/knowledge/operations/limitations.md b/knowledge/operations/limitations.md index 9d57724ac..9dc6fcac8 100644 --- a/knowledge/operations/limitations.md +++ b/knowledge/operations/limitations.md @@ -126,13 +126,14 @@ Boundaries of the WebAssembly CPython guest; see | ID | Limitation | Why | Evidence | |----|------------|-----|----------| | L-CPY-001 | No subprocesses from Python (`subprocess`, `os.system`, `os.fork`, `os.popen` fail); Python cannot call back into the shell | The guest has no process API; a shell bridge is a deliberate follow-up, not an accident | `processes_unavailable` (cpython_security_tests) | -| L-CPY-002 | No raw network from Python: `socket` cannot connect and `ssl` is not built. HTTP works only through `urllib.request`/`http.client` via the host's egress pipeline (needs `http_client` + an allowlist); responses are buffered up to the response cap; proxy, `ssl` context and client-certificate settings are ignored | No socket imports in the WASI host; HTTP goes through `HttpClient` like `curl`, TLS on the host | TM-PY-CPY-003, `network_unavailable`, `cpython_http_tests` | -| L-CPY-003 | No threads, `multiprocessing`, `ctypes`, native extensions or third-party packages | Single-threaded wasm guest with only the bundled pure-Python stdlib | `threads_and_native_code_unavailable` | +| L-CPY-002 | No raw network from Python: `socket` cannot connect and `ssl` is not built. HTTP works only through `urllib.request`/`http.client` (and bashkit's `requests`/`httpx`) via the host's egress pipeline (needs `http_client` + an allowlist); responses are buffered up to the response cap; proxy, `ssl` context and client-certificate settings are ignored | No socket imports in the WASI host; HTTP goes through `HttpClient` like `curl`, TLS on the host | TM-PY-CPY-003, `network_unavailable`, `cpython_http_tests` | +| L-CPY-003 | No threads, `multiprocessing`, `ctypes`, native extensions or third-party packages (`requests`/`httpx`/`httpx2` are bashkit's own compact implementations of the common API, see L-CPY-010) | Single-threaded wasm guest with only the bundled pure-Python stdlib | `threads_and_native_code_unavailable` | | L-CPY-004 | `errno` values are WASI's (`ENOENT` is 44); exception types and the `errno` module agree | wasi-libc numbering; rewriting it would desync the guest's `errno` module | `cpython_integration_tests::missing_script_exits_2`, stance | | L-CPY-005 | `hash()` of `str`/`bytes` uses one seed baked into the snapshot | The seed is chosen during snapshot initialization; per-call re-seeding would require re-hashing every interned object | stance | | L-CPY-006 | Deep C-level recursion ends the call with `python3: fatal error: stack overflow` instead of `RecursionError` | The interpreter's wasm stack is bounded (4 MiB); the trap is contained | `deep_c_recursion_is_contained` | | L-CPY-007 | No interactive REPL; `python3` with no program reads one from stdin | No TTY inside the sandbox | stance | | L-CPY-008 | Guest memory per call is capped at 1 GiB even if `max_memory` is higher | Pooled instance slots have a fixed maximum size | [CPython WebAssembly Runtime](../runtimes/cpython-wasm.md) | +| L-CPY-010 | `requests` and `httpx` are subsets: no retries, proxies, client certificates, HTTP/2, digest auth, streaming uploads or OPTIONS; bodies are buffered; `AsyncClient` requests run sequentially | Upstream packages cost ~3 s per call to import on Pulley; the subset is preloaded in the snapshot and costs nothing | `requests_lib`, `httpx_lib` in `cpython_http_tests` | | L-CPY-009 | Stdlib ships as bytecode only: tracebacks show no source line for stdlib frames, `inspect.getsource()` fails on stdlib objects. Modules that cannot work in the guest are not shipped: `ctypes`, `ssl`, `ftplib`, `imaplib`, `poplib`, `smtplib`, `socketserver`, `http.server`, `wsgiref`, `xmlrpc`, `webbrowser`, `pdb`, `bdb`, `pydoc` (so `help()`), `_pyrepl` | Compiling source on Pulley costs seconds per import; sources would push the crate past the crates.io 10 MiB cap. No FFI, TLS, sockets or TTY exist | `stdlib_is_bytecode_only` (cpython_integration_tests) | ## Text Processing diff --git a/knowledge/operations/release-process.md b/knowledge/operations/release-process.md index 247055857..5b3c8739b 100644 --- a/knowledge/operations/release-process.md +++ b/knowledge/operations/release-process.md @@ -82,7 +82,7 @@ silently failed. `cargo publish` packaging step, missing files, version drift): - `cargo publish --dry-run -p bashkit-cpython-wasm` must succeed and the `.crate` must stay under crates.io's 10 MiB limit (it embeds the CPython - snapshot; ~9.5 MiB as of 0.18). `bashkit` depends on the same version + snapshot; ~9.8 MiB after the requests/httpx modules, 236 KB under the cap). `bashkit` depends on the same version of it, so `publish.yml` publishes it first and the `bashkit` dry-run only resolves once that version is live. - `cargo publish --dry-run -p bashkit` must succeed. Package diff --git a/knowledge/runtimes/cpython-wasm.md b/knowledge/runtimes/cpython-wasm.md index 63024d84e..8d7403bb0 100644 --- a/knowledge/runtimes/cpython-wasm.md +++ b/knowledge/runtimes/cpython-wasm.md @@ -151,9 +151,39 @@ process bridge, quoting on every request). - The guest is linked with the `bashkit` imports; `wasmtime wizer` and the build smoke test run with `-W unknown-imports-trap=y` since init never calls them. -- Next: `requests`/`urllib3` and `httpx2` ship in a separate packages crate - behind the opt-in `cpython-http` feature (`import httpx` aliases - `httpx2`). + +### `requests` and `httpx` (bashkit's own modules) + +`requests`, `httpx` and `httpx2` in the guest are bashkit's own compact +implementations of the common API (`guest/requests/`, `guest/httpx.py`, +`guest/httpx2.py`), sharing `guest/_bashkit_webcore.py`, and call +`_bashkit.http` directly. Upstream packages were rejected: vendored +requests + urllib3 + idna + charset_normalizer imported 155 modules and +cost ~3.1 s per call on Pulley (urllib3 1.3 s, mostly module-level regex +compiles and class creation; http.client alone ~0.47 s), and since each +call starts from the snapshot that cost repeats every call. Preloading the +upstream stack instead would have grown the snapshot by far more and kept +the dependency on stdlib http.client/email. + +- Only snapshot-preloaded stdlib is imported (json, urllib.parse, base64, + zlib, `http` for status phrases), and the modules are preloaded + themselves: `import requests, httpx` adds nothing to a call (criterion + `python_import/cpython/requests_httpx` ~= `python_call/cpython/print`). +- Always available with `cpython` (no extra feature); without a network + allowlist every request fails like `curl`. +- Redirects are followed in the guest, so each hop is a new host-checked + request; `Authorization` is dropped when a redirect changes host. Cookies + from responses are kept per host. Both are convenience, not the + boundary: the host allowlist and credential injection are. +- The host does not decompress (TM-NET-013); gzip/deflate bodies are + decoded in the guest within its memory limit. +- Scope: verbs, sessions/clients, params/data/json/files/headers/cookies/ + basic auth/timeouts/redirects, response helpers, upstream exception + hierarchies, httpx `AsyncClient` (sequential) and `MockTransport`. + Out of scope: retries, proxies, client certs, HTTP/2, digest auth, + streaming uploads, OPTIONS (not in the host's method set). Versions + report `2.32.0+bashkit` / `0.28.1+bashkit`. +- `httpx2` (pydantic/httpx2) is the `httpx` module under a second name. ### WASI host decisions @@ -217,12 +247,15 @@ never use it). asyncio's self-pipe is disabled | Integration (CLI, stdio, env, VFS, isolation, bash interop) | `tests/integration/cpython_integration_tests.rs` | | Capability (~85 language/stdlib programs) | `tests/integration/cpython_capability_tests.rs`; the same table is diffed against the host `python3` when present | | Security (threats below, limits, crash containment, proptest fuzz) | `tests/integration/cpython_security_tests.rs` | -| Fuzz | `crates/bashkit/fuzz/fuzz_targets/cpython_fuzz.rs` (`--features cpython`, nightly job) | +| HTTP bridge, requests, httpx | `tests/integration/cpython_http_tests.rs` (fake transport: allowlist, redirects, cookies/auth scoping, caps, timeouts) | +| requests/httpx API parity | `guest/tests/differential.py`: the same scenarios against upstream `requests`/`httpx` (patched transport, shared fake server), outputs must match | +| Fuzz | `crates/bashkit/fuzz/fuzz_targets/cpython_fuzz.rs`, `cpython_http_fuzz.rs` (`--features cpython`, nightly job) | | Bench / load | `benches/python.rs`, `examples/python_startup.rs`, `examples/cpython_load.rs`, `just bench-python` | -CI: the Test job runs `cargo test -p bashkit --features cpython --lib --test -integration -- cpython`; the examples job runs `cpython_scripts`; fuzz-check -builds `cpython_fuzz`. +CI: the Test job runs `cargo test -p bashkit --features cpython,http_client +--lib --test integration -- cpython` and the requests/httpx differential +(pinned upstream versions in a venv); the examples job runs +`cpython_scripts`; fuzz-check builds `cpython_fuzz` and `cpython_http_fuzz`. ## Rebuilding the guest