diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dae707938..1a4c99135 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -306,11 +306,11 @@ jobs: # the runner's python3), security and load tests. Feature-gated, so the # main test step above does not build it into bashkit. - name: Run CPython tests - run: cargo test -p bashkit --features cpython --lib --test integration -- cpython + run: cargo test -p bashkit --features cpython,http_client --lib --test integration -- cpython # Same suites on the opt-in native-code build (`cpython-native`). - name: Run CPython tests (native code) - run: cargo test -p bashkit --features cpython-native --lib --test integration -- cpython + run: cargo test -p bashkit --features cpython-native,http_client --lib --test integration -- cpython - name: Run realfs tests run: cargo test --features realfs -p bashkit --test realfs_tests -p bashkit-cli @@ -568,6 +568,7 @@ jobs: run: | cargo +nightly fuzz build cargo +nightly fuzz build --features cpython cpython_fuzz + cargo +nightly fuzz build --features cpython cpython_http_fuzz # Gate job for branch protection — name must stay "Check" check: diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index 3e475093b..21bb2baf9 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -26,7 +26,7 @@ jobs: strategy: fail-fast: false matrix: - target: [parser_fuzz, lexer_fuzz, analyze_fuzz, arithmetic_fuzz, glob_fuzz, cpython_fuzz] + target: [parser_fuzz, lexer_fuzz, analyze_fuzz, arithmetic_fuzz, glob_fuzz, cpython_fuzz, cpython_http_fuzz] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -61,7 +61,7 @@ jobs: exit 1 fi features=() - if [ "${{ matrix.target }}" = cpython_fuzz ]; then features=(--features cpython); fi + case "${{ matrix.target }}" in cpython_*) features=(--features cpython) ;; esac cargo +nightly fuzz run "${features[@]}" ${{ matrix.target }} -- \ -max_total_time="$DURATION_INPUT" \ -print_final_stats=1 diff --git a/crates/bashkit-cpython-wasm/artifacts/MANIFEST b/crates/bashkit-cpython-wasm/artifacts/MANIFEST index 41c3a5fec..8978e92fd 100644 --- a/crates/bashkit-cpython-wasm/artifacts/MANIFEST +++ b/crates/bashkit-cpython-wasm/artifacts/MANIFEST @@ -3,5 +3,5 @@ wasi_sdk=24 wasmtime_wizer=49.0.1 zlib=1.3.1 sqlite=sqlite-amalgamation-3500400 -6f4c6d7b93992426af97884dfb603c2ef9ddda9ad20e51e2b79e84c18eb1e140 python.wasm.gz -672dd03d9e31bee0d92c14f10f772b97c5daedbc34b9f14d5ebc21b086b77daf python314.zip +8cbd501c5b6e0f73d986be9b5bf48f4f0f28d40ef83774e666ffbb854758b0a8 python.wasm.gz +abf5df19ec01bb180672e94b816711fc03dfd92e109b94fc10e5399523bb464c python314.zip diff --git a/crates/bashkit-cpython-wasm/artifacts/python.wasm.gz b/crates/bashkit-cpython-wasm/artifacts/python.wasm.gz index 759146588..74258d477 100644 Binary files a/crates/bashkit-cpython-wasm/artifacts/python.wasm.gz and b/crates/bashkit-cpython-wasm/artifacts/python.wasm.gz differ diff --git a/crates/bashkit-cpython-wasm/artifacts/python314.zip b/crates/bashkit-cpython-wasm/artifacts/python314.zip index cdb3084a0..eed7c046a 100644 Binary files a/crates/bashkit-cpython-wasm/artifacts/python314.zip and b/crates/bashkit-cpython-wasm/artifacts/python314.zip differ diff --git a/crates/bashkit-cpython-wasm/guest/_bashkit_http.py b/crates/bashkit-cpython-wasm/guest/_bashkit_http.py new file mode 100644 index 000000000..2057d3955 --- /dev/null +++ b/crates/bashkit-cpython-wasm/guest/_bashkit_http.py @@ -0,0 +1,215 @@ +"""HTTP for the bashkit CPython guest: stdlib adapters over `_bashkit.http`. + +`_bashkit.http` hands one request to the host, which runs it through +bashkit's egress pipeline (allowlist, SSRF checks, credential injection, +signing, the embedder's transport, size caps). The guest has no sockets and +no `ssl`; TLS is the host's. + +Decisions: +- http.client keeps its own request and response code; only the socket is + replaced. A connection gets a bridge socket that collects the bytes + http.client writes, sends the request through the host once it is + complete, and serves the reply as an HTTP/1.1 byte stream that the real + `HTTPResponse` parses. Status line, headers, chunked uploads, keep-alive + and HEAD behave as in CPython, and host errors surface from `request()` + so urllib wraps them in `URLError` like a failed connect. +- `HTTPSConnection` is defined although `ssl` is missing (context arguments + are accepted and ignored), so urllib.request registers its https handler. +- Proxies and CONNECT tunnels are not used: the bridge sends to the tunnel + target directly, and egress policy is the embedder's. +- Applied by a footer appended to `http/client.py` at build time, so nothing + runs for scripts that never import http.client. +""" + +import io +import sys + +import _bashkit + +_DEFAULT_PORTS = {"http": 80, "https": 443} + + +class _NoTLSContext: + """Stand-in for `ssl.SSLContext`: the host verifies TLS itself.""" + + check_hostname = True + verify_mode = 2 # ssl.CERT_REQUIRED + post_handshake_auth = None + + def __getattr__(self, name): + # load_verify_locations, set_alpn_protocols, load_cert_chain, ... + return lambda *args, **kwargs: None + + +def _parse_request(data): + """Split one complete request off `data`. + + Returns (method, target, headers, body, rest) or None if incomplete. + """ + end = data.find(b"\r\n\r\n") + if end < 0: + return None + head = bytes(data[:end]).decode("latin-1").split("\r\n") + method, target, _ = head[0].split(" ", 2) + headers = [] + length = 0 + chunked = False + for line in head[1:]: + name, _, value = line.partition(":") + value = value.strip() + lower = name.strip().lower() + if lower == "content-length": + length = int(value) + elif lower == "transfer-encoding" and "chunked" in value.lower(): + chunked = True + headers.append((name.strip(), value)) + pos = end + 4 + if not chunked: + if len(data) - pos < length: + return None + return method, target, headers, bytes(data[pos : pos + length]), pos + length + body = bytearray() + while True: + eol = data.find(b"\r\n", pos) + if eol < 0: + return None + size = int(bytes(data[pos:eol]).split(b";", 1)[0], 16) + pos = eol + 2 + if size == 0: + # Skip trailers up to the blank line. + while True: + eol = data.find(b"\r\n", pos) + if eol < 0: + return None + blank = eol == pos + pos = eol + 2 + if blank: + return method, target, headers, bytes(body), pos + if len(data) - pos < size + 2: + return None + body += data[pos : pos + size] + pos += size + 2 + + +def _status_line(status): + from http import HTTPStatus + + try: + reason = HTTPStatus(status).phrase + except ValueError: + reason = "" + return f"HTTP/1.1 {status} {reason}\r\n" + + +class _BridgeSocket: + """What http.client sees as its socket.""" + + def __init__(self, scheme, host, port, timeout): + if ":" in host and not host.startswith("["): + host = f"[{host}]" + if port is None or port == _DEFAULT_PORTS[scheme]: + self._base = f"{scheme}://{host}" + else: + self._base = f"{scheme}://{host}:{port}" + self._timeout = timeout if isinstance(timeout, (int, float)) else None + self._out = bytearray() + self._replies = [] + + def sendall(self, data): + self._out += data + while True: + parsed = _parse_request(self._out) + if parsed is None: + return + method, target, headers, body, used = parsed + del self._out[:used] + self._replies.append(self._exchange(method, target, headers, body)) + + def send(self, data): + self.sendall(data) + return len(data) + + def _exchange(self, method, target, headers, body): + if target.startswith(("http://", "https://")): + url = target # absolute form (proxy-style request) + else: + url = self._base + target + headers = [ + (k, v) for k, v in headers if k.lower() not in ("host", "transfer-encoding") + ] + status, reply_headers, reply_body = _bashkit.http( + method, url, headers, body, self._timeout + ) + lines = [_status_line(status)] + for name, value in reply_headers: + lower = name.lower() + if lower == "transfer-encoding": + continue + if lower == "content-length" and method != "HEAD": + continue + lines.append(f"{name}: {value}\r\n") + if method != "HEAD": + lines.append(f"Content-Length: {len(reply_body)}\r\n") + lines.append("\r\n") + return "".join(lines).encode("utf-8") + reply_body + + def makefile(self, mode="r", *args, **kwargs): + if "w" in mode: + raise OSError("bridge socket is read-only") + if not self._replies: + raise ConnectionError("no HTTP request was sent") + return io.BytesIO(self._replies.pop(0)) + + def settimeout(self, timeout): + self._timeout = timeout + + def setsockopt(self, *args): + pass + + def close(self): + self._out.clear() + self._replies.clear() + + +def patch_http_client(ns): + """Route `http.client` connections through the host. `ns` is the + module's globals (called from the footer of http/client.py).""" + base = ns["HTTPConnection"] + + def connect(self): + if self._tunnel_host: + host, port = self._tunnel_host, self._tunnel_port + else: + host, port = self.host, self.port + sys.audit("http.client.connect", self, host, port) + self.sock = _BridgeSocket(self._bashkit_scheme, host, port, self.timeout) + + base.connect = connect + base._bashkit_scheme = "http" + + if "HTTPSConnection" not in ns: + socket = ns["socket"] + + class HTTPSConnection(base): + "HTTPS connection; TLS is terminated by the bashkit host." + + default_port = ns["HTTPS_PORT"] + _bashkit_scheme = "https" + + def __init__( + self, + host, + port=None, + *, + timeout=socket._GLOBAL_DEFAULT_TIMEOUT, + source_address=None, + context=None, + blocksize=8192, + ): + super().__init__(host, port, timeout, source_address, blocksize=blocksize) + self._context = context + + HTTPSConnection.__module__ = "http.client" + ns["HTTPSConnection"] = HTTPSConnection + ns["__all__"].append("HTTPSConnection") + ns["_create_https_context"] = lambda http_version: _NoTLSContext() diff --git a/crates/bashkit-cpython-wasm/guest/bashkit_main.c b/crates/bashkit-cpython-wasm/guest/bashkit_main.c index f58e4a7d6..4daf0cd07 100644 --- a/crates/bashkit-cpython-wasm/guest/bashkit_main.c +++ b/crates/bashkit-cpython-wasm/guest/bashkit_main.c @@ -178,11 +178,234 @@ static PyObject *bk_immortalize(PyObject *self, PyObject *objects) { Py_RETURN_NONE; } +// --- HTTP bridge --------------------------------------------------------- +// The host import runs the request through bashkit's HttpClient (allowlist, +// SSRF checks, credential injection, signing, transport, size caps). The +// guest never opens a socket and never sees TLS. Records are u32 +// little-endian length-prefixed fields; see `cpython/http.rs` on the host. + +__attribute__((import_module("bashkit"), import_name("http_request"))) +int32_t bk_host_http_request(const uint8_t *req, int32_t len, int64_t timeout_ms, + uint32_t *out_len); +__attribute__((import_module("bashkit"), import_name("http_take"))) +int32_t bk_host_http_take(uint8_t *buf, int32_t len); + +enum { BK_HTTP_OK = 0, BK_HTTP_NETWORK = 1, BK_HTTP_TIMEOUT = 2, BK_HTTP_INVALID = 3 }; + +typedef struct { + uint8_t *data; + size_t len, cap; +} bk_buf; + +static int bk_buf_put(bk_buf *b, const void *src, size_t n) { + if (b->len + n > b->cap) { + size_t cap = b->cap ? b->cap * 2 : 256; + while (cap < b->len + n) { + cap *= 2; + } + uint8_t *data = realloc(b->data, cap); + if (data == NULL) { + return -1; + } + b->data = data; + b->cap = cap; + } + memcpy(b->data + b->len, src, n); + b->len += n; + return 0; +} + +static int bk_buf_u32(bk_buf *b, uint32_t v) { + uint8_t le[4] = {v & 0xff, (v >> 8) & 0xff, (v >> 16) & 0xff, (v >> 24) & 0xff}; + return bk_buf_put(b, le, 4); +} + +static int bk_buf_field(bk_buf *b, const char *src, size_t n) { + if (n > UINT32_MAX) { + return -1; + } + return (bk_buf_u32(b, (uint32_t)n) < 0 || bk_buf_put(b, src, n) < 0) ? -1 : 0; +} + +static int bk_buf_str(bk_buf *b, PyObject *s) { + Py_ssize_t n; + const char *utf8 = PyUnicode_AsUTF8AndSize(s, &n); + if (utf8 == NULL) { + return -1; + } + if (bk_buf_field(b, utf8, (size_t)n) < 0) { + PyErr_NoMemory(); + return -1; + } + return 0; +} + +typedef struct { + const uint8_t *p, *end; +} bk_cursor; + +static int bk_take_u32(bk_cursor *c, uint32_t *v) { + if (c->end - c->p < 4) { + return -1; + } + *v = (uint32_t)c->p[0] | ((uint32_t)c->p[1] << 8) | ((uint32_t)c->p[2] << 16) | + ((uint32_t)c->p[3] << 24); + c->p += 4; + return 0; +} + +static int bk_take_field(bk_cursor *c, const uint8_t **ptr, uint32_t *n) { + if (bk_take_u32(c, n) < 0 || (size_t)(c->end - c->p) < *n) { + return -1; + } + *ptr = c->p; + c->p += *n; + return 0; +} + +// Decode `status, headers, body` into (int, list[tuple[str, str]], bytes). +static PyObject *bk_decode_response(const uint8_t *data, size_t len) { + bk_cursor c = {data, data + len}; + uint32_t status, count; + if (bk_take_u32(&c, &status) < 0 || bk_take_u32(&c, &count) < 0) { + goto malformed; + } + PyObject *headers = PyList_New(0); + if (headers == NULL) { + return NULL; + } + for (uint32_t i = 0; i < count; i++) { + const uint8_t *k, *v; + uint32_t kn, vn; + if (bk_take_field(&c, &k, &kn) < 0 || bk_take_field(&c, &v, &vn) < 0) { + Py_DECREF(headers); + goto malformed; + } + PyObject *pair = Py_BuildValue("(s#s#)", (const char *)k, (Py_ssize_t)kn, + (const char *)v, (Py_ssize_t)vn); + if (pair == NULL || PyList_Append(headers, pair) < 0) { + Py_XDECREF(pair); + Py_DECREF(headers); + return NULL; + } + Py_DECREF(pair); + } + const uint8_t *body; + uint32_t body_len; + if (bk_take_field(&c, &body, &body_len) < 0) { + Py_DECREF(headers); + goto malformed; + } + return Py_BuildValue("(INy#)", (unsigned int)status, headers, (const char *)body, + (Py_ssize_t)body_len); +malformed: + PyErr_SetString(PyExc_OSError, "malformed HTTP response from host"); + return NULL; +} + +// _bashkit.http(method, url, headers, body, timeout) -> (status, headers, body) +// headers: iterable of (name, value) str pairs; body: bytes-like or None; +// timeout: seconds (float) or None. Raises ConnectionError, TimeoutError or +// ValueError (request rejected by the host). +static PyObject *bk_http(PyObject *self, PyObject *args) { + (void)self; + PyObject *method, *url, *headers, *body_obj, *timeout_obj; + if (!PyArg_ParseTuple(args, "UUOOO:http", &method, &url, &headers, &body_obj, + &timeout_obj)) { + return NULL; + } + int64_t timeout_ms = -1; + if (timeout_obj != Py_None) { + double t = PyFloat_AsDouble(timeout_obj); + if (t == -1.0 && PyErr_Occurred()) { + return NULL; + } + timeout_ms = t <= 0 ? 0 : (t > 9.0e15 ? INT64_MAX : (int64_t)(t * 1000.0)); + } + Py_buffer body = {0}; + if (body_obj != Py_None && PyObject_GetBuffer(body_obj, &body, PyBUF_SIMPLE) < 0) { + return NULL; + } + PyObject *result = NULL; + PyObject *seq = NULL; + uint8_t *resp = NULL; + bk_buf req = {0}; + if (bk_buf_str(&req, method) < 0 || bk_buf_str(&req, url) < 0) { + goto done; + } + seq = PySequence_Fast(headers, "headers must be a sequence of (name, value) pairs"); + if (seq == NULL) { + goto done; + } + Py_ssize_t n = PySequence_Fast_GET_SIZE(seq); + if (n > UINT32_MAX || bk_buf_u32(&req, (uint32_t)n) < 0) { + PyErr_NoMemory(); + goto done; + } + for (Py_ssize_t i = 0; i < n; i++) { + PyObject *pair = PySequence_Fast_GET_ITEM(seq, i); + if (!PyTuple_Check(pair) || PyTuple_GET_SIZE(pair) != 2 || + !PyUnicode_Check(PyTuple_GET_ITEM(pair, 0)) || + !PyUnicode_Check(PyTuple_GET_ITEM(pair, 1))) { + PyErr_SetString(PyExc_TypeError, "headers must be (str, str) pairs"); + goto done; + } + if (bk_buf_str(&req, PyTuple_GET_ITEM(pair, 0)) < 0 || + bk_buf_str(&req, PyTuple_GET_ITEM(pair, 1)) < 0) { + goto done; + } + } + if (bk_buf_field(&req, body.buf ? body.buf : "", (size_t)body.len) < 0 || + req.len > INT32_MAX) { + PyErr_SetString(PyExc_ValueError, "request too large"); + goto done; + } + uint32_t resp_len = 0; + int32_t code; + code = bk_host_http_request(req.data, (int32_t)req.len, timeout_ms, &resp_len); + if (code > BK_HTTP_INVALID) { + PyErr_SetString(PyExc_OSError, "HTTP bridge fault"); + goto done; + } + resp = malloc(resp_len ? resp_len : 1); + if (resp == NULL) { + // Drop the pending reply so the next request starts clean. + bk_host_http_take(NULL, -1); + PyErr_NoMemory(); + goto done; + } + if (bk_host_http_take(resp, (int32_t)resp_len) != BK_HTTP_OK) { + PyErr_SetString(PyExc_OSError, "HTTP bridge fault"); + goto done; + } + if (code == BK_HTTP_OK) { + result = bk_decode_response(resp, resp_len); + } else { + PyObject *kind = code == BK_HTTP_TIMEOUT ? PyExc_TimeoutError + : code == BK_HTTP_INVALID ? PyExc_ValueError + : PyExc_ConnectionError; + PyObject *msg = PyUnicode_DecodeUTF8((const char *)resp, resp_len, "replace"); + if (msg != NULL) { + PyErr_SetObject(kind, msg); + Py_DECREF(msg); + } + } +done: + free(resp); + free(req.data); + Py_XDECREF(seq); + if (body.obj != NULL) { + PyBuffer_Release(&body); + } + return result; +} + static PyMethodDef bk_methods[] = { {"argv", bk_argv, METH_NOARGS, "WASI argv for this call."}, {"environ", bk_environ, METH_NOARGS, "WASI environment for this call."}, {"load_environ", bk_load_environ, METH_NOARGS, "Install this call's environment."}, {"immortalize", bk_immortalize, METH_O, "Make snapshot objects immortal."}, + {"http", bk_http, METH_VARARGS, "Send one HTTP request through the host."}, {NULL, NULL, 0, NULL}, }; diff --git a/crates/bashkit-cpython-wasm/guest/build.sh b/crates/bashkit-cpython-wasm/guest/build.sh index 9da72d783..2e1acb94d 100755 --- a/crates/bashkit-cpython-wasm/guest/build.sh +++ b/crates/bashkit-cpython-wasm/guest/build.sh @@ -122,7 +122,7 @@ rm -rf "$ROOT" "$WORK/stdlib" mkdir -p "$ROOT/usr/local/lib" "$WORK/stdlib" cp -r "$SRC/Lib/." "$WORK/stdlib/" cp "$HOST_BUILD"/build/lib.wasi-wasm32-3.14/_sysconfigdata_*.py "$WORK/stdlib/" -cp "$HERE/_bashkit_boot.py" "$WORK/stdlib/" +cp "$HERE/_bashkit_boot.py" "$HERE/_bashkit_http.py" "$WORK/stdlib/" ( cd "$WORK/stdlib" # Not usable or not useful in a sandboxed, single-threaded, headless guest. @@ -132,6 +132,9 @@ cp "$HERE/_bashkit_boot.py" "$WORK/stdlib/" rm -rf ctypes ssl.py ftplib.py imaplib.py poplib.py smtplib.py socketserver.py \ http/server.py wsgiref xmlrpc webbrowser.py _pyrepl pdb.py bdb.py pydoc.py \ _aix_support.py _android_support.py _ios_support.py _osx_support.py + # HTTP goes through the host's egress pipeline, not sockets + # (_bashkit_http.py); http.client patches itself when first imported. + printf '\n# bashkit: connections go through the host (see _bashkit_http).\nimport _bashkit_http\n_bashkit_http.patch_http_client(globals())\ndel _bashkit_http\n' >>http/client.py # Pure-Python twins of C modules the guest always has. rm -f _pydecimal.py _pyio.py _pydatetime.py find . -name __pycache__ -prune -exec rm -rf {} + @@ -166,11 +169,12 @@ PY ) # --- snapshot ---------------------------------------------------------------- -"$WASMTIME" wizer -S cli --dir "$ROOT::/" -o "$WORK/bashkit_wizer.wasm" "$WORK/bashkit_raw.wasm" +# The `bashkit` host imports (HTTP) are never called during init. +"$WASMTIME" wizer -S cli -W unknown-imports-trap=y --dir "$ROOT::/" -o "$WORK/bashkit_wizer.wasm" "$WORK/bashkit_raw.wasm" "$SDK/bin/llvm-strip" -o "$WORK/python.wasm" "$WORK/bashkit_wizer.wasm" # Smoke test the snapshot with the reference runtime before committing it. -out="$("$WASMTIME" run -S cli --dir "$ROOT::/" --env PWD=/ --invoke bashkit_run \ +out="$("$WASMTIME" run -S cli -W unknown-imports-trap=y --dir "$ROOT::/" --env PWD=/ --invoke bashkit_run \ "$WORK/python.wasm" -c 'import json, re, csv, sqlite3, zlib; print(json.dumps({"ok": 1}))' 2>/dev/null)" case "$out" in *'{"ok": 1}'*) ;; diff --git a/crates/bashkit/docs/cpython.md b/crates/bashkit/docs/cpython.md index 5d591332f..29ef37cc3 100644 --- a/crates/bashkit/docs/cpython.md +++ b/crates/bashkit/docs/cpython.md @@ -63,6 +63,7 @@ return a value. Scripts written by people and agents expect a Python | Start per call | ~15 µs | ~4.4 ms (first call in a process ~19 ms) | | CPU-bound speed | Native | ~4-30x slower than Monty (interpreted wasm); ~4x slower with `cpython-native` | | Host callbacks | Yes (external functions) | Not yet | +| HTTP | No | `urllib.request`, `http.client` through Bashkit's egress allowlist | Pick CPython when scripts need real Python behavior; pick Monty when you need host function callbacks or microsecond start-up for tiny snippets. @@ -112,12 +113,50 @@ let bash = Bash::builder() - **Concurrency**: up to 512 CPython calls run at once per process; further calls wait for a free slot within their own deadline. +## HTTP + +With the `http_client` feature and a network allowlist +(`BashBuilder::network`), `urllib.request` +and `http.client` work for `http://` and `https://` URLs. Each request is +handed to the host and goes through the same pipeline as `curl`: allowlist, +private-IP (SSRF) checks, credential injection, request signing, your +`HttpTransport` and the response size cap. TLS runs +on the host; the guest has no sockets. + +```rust,ignore +let mut bash = Bash::builder() + .cpython() + .network(NetworkAllowlist::new().allow("https://api.example.com")) + .build(); +bash.exec(r#"python3 -c ' +import json, urllib.request +with urllib.request.urlopen("https://api.example.com/v1/items") as r: + print(json.load(r)) +'"#).await?; +``` + +- A denied URL raises `URLError` (or `ConnectionError` from `http.client`) + with the same "access denied" text `curl` prints; a timeout raises + `TimeoutError`. Without a network allowlist every request fails with + "network access not configured". +- Redirects are followed by Python, so every hop is checked again. +- Methods: GET, POST, PUT, DELETE, HEAD, PATCH. Headers the host owns + (`Host`, `Content-Length`, `Transfer-Encoding`, `Connection`, proxy + headers) are set by the host; values with CR/LF are rejected. +- Responses are buffered (up to the client's `max_response_bytes`), so + streaming past that cap fails. `ssl` contexts, `verify=False`, client + certificates and proxy settings are ignored: the host verifies TLS. +- `CPythonLimits::max_http_requests` caps requests per call (default + 100); a request's timeout never outlasts the call's deadline. + +`requests` and `httpx` are not bundled yet. + ## Limitations - **No subprocesses**: `subprocess`, `os.system`, `os.fork`, `os.popen` raise `OSError`/`AttributeError`. Python cannot call back into the shell yet. -- **No network**: `socket`, `urllib.request`, `http.client` cannot connect. - Use the shell's `curl`/`http` builtins. +- **HTTP only, through Bashkit's egress**: see [HTTP](#http). Raw + `socket` connections, `ssl` and non-HTTP protocols are unavailable. - **No threads**: `threading.Thread.start()` raises `RuntimeError`; `multiprocessing` and `concurrent.futures.ProcessPoolExecutor` are absent. `asyncio` works. @@ -129,9 +168,9 @@ let bash = Bash::builder() there is no REPL, and `pdb` and `pydoc` (`help()`) are not shipped. - **Stdlib is bytecode only**: tracebacks through stdlib code show no source line, and `inspect.getsource()` fails on stdlib objects. Your own code - keeps full tracebacks. Network clients and servers (`smtplib`, `ftplib`, - `http.server`, `xmlrpc`, ...) are not shipped since the guest has no - sockets. + keeps full tracebacks. Non-HTTP network clients and servers (`smtplib`, + `ftplib`, `http.server`, `xmlrpc`, ...) are not shipped since the guest has + no sockets. - **Symlinks are not followed**, like everywhere in the Bashkit VFS. - **`errno` numbers are WASI's** (`ENOENT` is 44, not 2). Exception types (`FileNotFoundError`, ...) and messages are correct; code comparing diff --git a/crates/bashkit/fuzz/Cargo.toml b/crates/bashkit/fuzz/Cargo.toml index c683ad0db..32e54d5db 100644 --- a/crates/bashkit/fuzz/Cargo.toml +++ b/crates/bashkit/fuzz/Cargo.toml @@ -164,3 +164,11 @@ required-features = ["cpython"] test = false doc = false bench = false + +[[bin]] +name = "cpython_http_fuzz" +path = "fuzz_targets/cpython_http_fuzz.rs" +required-features = ["cpython"] +test = false +doc = false +bench = false diff --git a/crates/bashkit/fuzz/fuzz_targets/cpython_http_fuzz.rs b/crates/bashkit/fuzz/fuzz_targets/cpython_http_fuzz.rs new file mode 100644 index 000000000..20c02184b --- /dev/null +++ b/crates/bashkit/fuzz/fuzz_targets/cpython_http_fuzz.rs @@ -0,0 +1,18 @@ +//! Fuzz target for the CPython HTTP bridge request decoder +//! +//! Feeds arbitrary bytes as the request record the guest hands to the +//! `bashkit.http_request` host import, to find: +//! - Host panics on malformed or truncated records +//! - Requests the host would dispatch with CR/LF/NUL in header values, +//! invalid header names, host-owned headers (`Host`, framing), unsupported +//! methods or oversized fields (TM-PY-CPY-003) +//! +//! Run with: cargo +nightly fuzz run cpython_http_fuzz --features cpython -- -max_total_time=300 + +#![no_main] + +use libfuzzer_sys::fuzz_target; + +fuzz_target!(|data: &[u8]| { + bashkit::testing::check_cpython_http_request(data); +}); diff --git a/crates/bashkit/src/builtins/cpython/http.rs b/crates/bashkit/src/builtins/cpython/http.rs new file mode 100644 index 000000000..d007529c3 --- /dev/null +++ b/crates/bashkit/src/builtins/cpython/http.rs @@ -0,0 +1,467 @@ +//! `bashkit.http_request` host import: HTTP from the CPython guest through +//! bashkit's egress pipeline. +//! +//! Decisions (see `knowledge/runtimes/cpython-wasm.md`, TM-PY-CPY-003): +//! - Requests leave the guest at the HTTP level, never as sockets. TLS runs on +//! the host, and every request goes through [`HttpClient`]: allowlist, SSRF +//! precheck, `before_http` hooks (credential injection), signing, the +//! embedder's transport, the response cap and `after_http` hooks. Python +//! traffic is indistinguishable from `curl` at the egress boundary. +//! - No network configured (or no `http_client` feature) means every request +//! fails with "network access not configured". +//! - The request is a length-prefixed binary record the guest's C module +//! builds; the host validates it before dispatch (method, URL and header +//! size, header name/value syntax) and drops headers the host owns (`Host`, +//! framing, hop-by-hop), so a guest cannot smuggle a second request or +//! aim at a virtual host the allowlist never checked. +//! - Redirects are not followed here: each hop is a new guest request and +//! is re-checked, like `curl`. +//! - The response comes back in two steps (`http_request` reports its size, +//! `http_take` copies it), so the guest allocates exactly once and the +//! copy counts against its own memory limit. +//! - A per-call request cap bounds request floods. + +#[cfg(feature = "http_client")] +use std::sync::Arc; +use std::time::Duration; + +use wasmtime::{Caller, Linker}; + +use super::wasi::{GuestState, read_bytes, write_bytes, write_u32}; + +const MODULE: &str = "bashkit"; + +/// `http_request` / `http_take` status codes (mirrored in `bashkit_main.c`). +const OK: i32 = 0; +const NETWORK_ERROR: i32 = 1; +#[cfg_attr(not(feature = "http_client"), allow(dead_code))] +const TIMED_OUT: i32 = 2; +const INVALID_REQUEST: i32 = 3; +const FAULT: i32 = 4; + +const MAX_URL_LEN: usize = 8192; +const MAX_HEADERS: usize = 128; +const MAX_HEADER_BYTES: usize = 64 * 1024; +/// Largest request body the guest may send (same order as the default +/// response cap). +const MAX_REQUEST_BODY: usize = 16 * 1024 * 1024; +/// Longest error text handed back to the guest (TM-INF-022). +const MAX_ERROR_LEN: usize = 512; + +/// Headers the host (or reqwest) sets itself; guest values are dropped. +const HOST_OWNED_HEADERS: &[&str] = &[ + "host", + "content-length", + "transfer-encoding", + "connection", + "keep-alive", + "upgrade", + "te", + "trailer", + "proxy-connection", + "proxy-authorization", +]; + +/// Per-call HTTP state of one guest instance. +pub(crate) struct HttpState { + #[cfg(feature = "http_client")] + client: Option>, + requests: usize, + max_requests: usize, + /// Encoded response (or error text) waiting for `http_take`. + pending: Option>, +} + +impl HttpState { + pub(crate) fn new( + #[cfg(feature = "http_client")] client: Option>, + max_requests: usize, + ) -> Self { + Self { + #[cfg(feature = "http_client")] + client, + requests: 0, + max_requests, + pending: None, + } + } +} + +/// A validated guest request. +#[derive(Debug, PartialEq)] +pub(crate) struct Request { + pub(crate) method: String, + pub(crate) url: String, + pub(crate) headers: Vec<(String, String)>, + pub(crate) body: Vec, +} + +struct Reader<'a> { + buf: &'a [u8], +} + +impl<'a> Reader<'a> { + fn bytes(&mut self, max: usize, what: &str) -> Result<&'a [u8], String> { + let len = self.u32()? as usize; + if len > max { + return Err(format!("{what} too long")); + } + if self.buf.len() < len { + return Err("truncated request".to_string()); + } + let (head, rest) = self.buf.split_at(len); + self.buf = rest; + Ok(head) + } + + fn str(&mut self, max: usize, what: &str) -> Result<&'a str, String> { + std::str::from_utf8(self.bytes(max, what)?).map_err(|_| format!("{what} is not UTF-8")) + } + + fn u32(&mut self) -> Result { + let Some((head, rest)) = self.buf.split_first_chunk::<4>() else { + return Err("truncated request".to_string()); + }; + self.buf = rest; + Ok(u32::from_le_bytes(*head)) + } +} + +fn is_token(s: &str) -> bool { + !s.is_empty() + && s.bytes() + .all(|b| b.is_ascii_alphanumeric() || b"!#$%&'*+-.^_`|~".contains(&b)) +} + +/// Decode and validate `method, url, headers, body` (u32 little-endian +/// lengths and counts). +pub(crate) fn decode_request(buf: &[u8]) -> Result { + let mut r = Reader { buf }; + let method = r.str(16, "method")?.to_ascii_uppercase(); + if !matches!( + method.as_str(), + "GET" | "POST" | "PUT" | "DELETE" | "HEAD" | "PATCH" + ) { + return Err(format!("unsupported HTTP method: {method}")); + } + let url = r.str(MAX_URL_LEN, "URL")?.to_string(); + if url.bytes().any(|b| b.is_ascii_control() || b == b' ') { + return Err("URL contains control characters or spaces".to_string()); + } + let count = r.u32()? as usize; + if count > MAX_HEADERS { + return Err("too many headers".to_string()); + } + let mut headers = Vec::with_capacity(count); + let mut header_bytes = 0usize; + for _ in 0..count { + let name = r.str(MAX_HEADER_BYTES, "header name")?; + let value = r.str(MAX_HEADER_BYTES, "header value")?; + header_bytes += name.len() + value.len(); + if header_bytes > MAX_HEADER_BYTES { + return Err("headers too large".to_string()); + } + if !is_token(name) { + return Err("invalid header name".to_string()); + } + if value.bytes().any(|b| b == b'\r' || b == b'\n' || b == 0) { + return Err("invalid header value".to_string()); + } + let lower = name.to_ascii_lowercase(); + if HOST_OWNED_HEADERS.contains(&lower.as_str()) { + continue; + } + headers.push((name.to_string(), value.trim().to_string())); + } + let body = r.bytes(MAX_REQUEST_BODY, "request body")?.to_vec(); + if !r.buf.is_empty() { + return Err("trailing bytes in request".to_string()); + } + Ok(Request { + method, + url, + headers, + body, + }) +} + +/// Decode `buf` and assert what the host guarantees about anything it +/// dispatches (fuzzing entry point, TM-PY-CPY-003). +pub(crate) fn check_request_invariants(buf: &[u8]) { + let Ok(req) = decode_request(buf) else { + return; + }; + assert!(matches!( + req.method.as_str(), + "GET" | "POST" | "PUT" | "DELETE" | "HEAD" | "PATCH" + )); + assert!(req.url.len() <= MAX_URL_LEN); + assert!(!req.url.bytes().any(|b| b.is_ascii_control() || b == b' ')); + assert!(req.headers.len() <= MAX_HEADERS); + for (name, value) in &req.headers { + assert!(is_token(name), "header name {name:?}"); // debug-ok: fuzz assertion + assert!(!value.bytes().any(|b| b == b'\r' || b == b'\n' || b == 0)); + assert!(!HOST_OWNED_HEADERS.contains(&name.to_ascii_lowercase().as_str())); + } + assert!(req.body.len() <= MAX_REQUEST_BODY); +} + +/// Encode `status, headers, body` for the guest. +#[cfg_attr(not(feature = "http_client"), allow(dead_code))] +pub(crate) fn encode_response(status: u16, headers: &[(String, String)], body: &[u8]) -> Vec { + let size = 12 + + headers + .iter() + .map(|(k, v)| 8 + k.len() + v.len()) + .sum::() + + body.len(); + let mut out = Vec::with_capacity(size); + out.extend_from_slice(&u32::from(status).to_le_bytes()); + out.extend_from_slice(&(headers.len() as u32).to_le_bytes()); + for (k, v) in headers { + out.extend_from_slice(&(k.len() as u32).to_le_bytes()); + out.extend_from_slice(k.as_bytes()); + out.extend_from_slice(&(v.len() as u32).to_le_bytes()); + out.extend_from_slice(v.as_bytes()); + } + out.extend_from_slice(&(body.len() as u32).to_le_bytes()); + out.extend_from_slice(body); + out +} + +fn error_text(msg: &str) -> Vec { + let mut msg = msg.to_string(); + msg.truncate(msg.floor_char_boundary(MAX_ERROR_LEN)); + msg.into_bytes() +} + +/// Run one request; `Err((code, message))` on failure. +async fn send( + state: &HttpState, + request: Request, + timeout: Duration, +) -> Result, (i32, String)> { + #[cfg(feature = "http_client")] + if let Some(client) = &state.client { + use crate::network::Method; + let method = match request.method.as_str() { + "GET" => Method::Get, + "POST" => Method::Post, + "PUT" => Method::Put, + "DELETE" => Method::Delete, + "HEAD" => Method::Head, + _ => Method::Patch, + }; + let body = (!request.body.is_empty() + || matches!(method, Method::Post | Method::Put | Method::Patch)) + .then_some(request.body.as_slice()); + let secs = timeout.as_secs_f64().ceil() as u64; + let fut = client.request_with_timeouts( + method, + &request.url, + body, + &request.headers, + Some(secs), + None, + ); + return match tokio::time::timeout(timeout, fut).await { + Err(_) => Err((TIMED_OUT, "timed out".to_string())), + Ok(Ok(resp)) => Ok(encode_response(resp.status, &resp.headers, &resp.body)), + Ok(Err(crate::Error::Network(msg))) if msg == "operation timed out" => { + Err((TIMED_OUT, "timed out".to_string())) + } + Ok(Err(crate::Error::Network(msg))) => Err((NETWORK_ERROR, msg)), + Ok(Err(e)) => Err((NETWORK_ERROR, e.to_string())), + }; + } + let _ = (state, request, timeout); + Err((NETWORK_ERROR, "network access not configured".to_string())) +} + +async fn http_request( + c: &mut Caller<'_, GuestState>, + ptr: i32, + len: i32, + timeout_ms: i64, + out_len: i32, +) -> i32 { + let raw = match read_bytes(c, ptr, len) { + Ok(raw) => raw, + Err(_) => return FAULT, + }; + let (code, payload) = match decode_request(&raw) { + Err(msg) => (INVALID_REQUEST, error_text(&msg)), + Ok(request) => { + let state = c.data_mut(); + state.http.pending = None; + if state.http.requests >= state.http.max_requests { + ( + NETWORK_ERROR, + error_text(&format!( + "too many HTTP requests (limit {} per call)", + state.http.max_requests + )), + ) + } else { + state.http.requests += 1; + // The call deadline always wins over the request's own timeout. + let remaining = state.remaining(); + let timeout = if timeout_ms < 0 { + remaining + } else { + remaining.min(Duration::from_millis(timeout_ms as u64)) + }; + match send(&state.http, request, timeout).await { + Ok(body) => (OK, body), + Err((code, msg)) => (code, error_text(&msg)), + } + } + } + }; + let Ok(size) = u32::try_from(payload.len()) else { + return FAULT; + }; + if write_u32(c, out_len, size).is_err() { + return FAULT; + } + c.data_mut().http.pending = Some(payload); + code +} + +pub(crate) fn add_to_linker(linker: &mut Linker) -> wasmtime::Result<()> { + linker.func_wrap_async( + MODULE, + "http_request", + |mut c: Caller<'_, GuestState>, (ptr, len, timeout_ms, out_len): (i32, i32, i64, i32)| { + Box::new(async move { http_request(&mut c, ptr, len, timeout_ms, out_len).await }) + }, + )?; + linker.func_wrap( + MODULE, + "http_take", + |mut c: Caller<'_, GuestState>, buf: i32, len: i32| -> i32 { + let Some(pending) = c.data_mut().http.pending.take() else { + return FAULT; + }; + if pending.len() != len as u32 as usize || write_bytes(&mut c, buf, &pending).is_err() { + return FAULT; + } + OK + }, + )?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn encode(method: &str, url: &str, headers: &[(&str, &str)], body: &[u8]) -> Vec { + let mut out = Vec::new(); + let mut put = |b: &[u8]| { + out.extend_from_slice(&(b.len() as u32).to_le_bytes()); + out.extend_from_slice(b); + }; + put(method.as_bytes()); + put(url.as_bytes()); + let mut rest = Vec::new(); + rest.extend_from_slice(&(headers.len() as u32).to_le_bytes()); + for (k, v) in headers { + rest.extend_from_slice(&(k.len() as u32).to_le_bytes()); + rest.extend_from_slice(k.as_bytes()); + rest.extend_from_slice(&(v.len() as u32).to_le_bytes()); + rest.extend_from_slice(v.as_bytes()); + } + rest.extend_from_slice(&(body.len() as u32).to_le_bytes()); + rest.extend_from_slice(body); + out.extend_from_slice(&rest); + out + } + + #[test] + fn decodes_a_request() { + let raw = encode( + "post", + "https://example.com/x", + &[("Accept", " text/plain "), ("X-A", "1")], + b"hi", + ); + let req = decode_request(&raw).unwrap(); + assert_eq!(req.method, "POST"); + assert_eq!(req.url, "https://example.com/x"); + assert_eq!( + req.headers, + vec![ + ("Accept".to_string(), "text/plain".to_string()), + ("X-A".to_string(), "1".to_string()) + ] + ); + assert_eq!(req.body, b"hi"); + } + + #[test] + fn drops_host_owned_headers() { + let raw = encode( + "GET", + "http://a.test/", + &[ + ("Host", "internal.test"), + ("Content-Length", "0"), + ("transfer-encoding", "chunked"), + ("Connection", "keep-alive"), + ("Proxy-Authorization", "x"), + ("X-Keep", "y"), + ], + b"", + ); + let req = decode_request(&raw).unwrap(); + assert_eq!(req.headers, vec![("X-Keep".to_string(), "y".to_string())]); + } + + #[test] + fn rejects_smuggling_and_bad_input() { + let cases: Vec> = vec![ + encode("GET", "http://a.test/", &[("X", "a\r\nHost: b")], b""), + encode("GET", "http://a.test/", &[("X Y", "a")], b""), + encode("GET", "http://a.test/", &[("", "a")], b""), + encode("GET", "http://a.test/ HTTP/1.1\r\n", &[], b""), + encode("CONNECT", "http://a.test/", &[], b""), + encode("GET\r\n", "http://a.test/", &[], b""), + encode("GET", &"x".repeat(MAX_URL_LEN + 1), &[], b""), + vec![1, 2, 3], + Vec::new(), + ]; + for raw in cases { + assert!(decode_request(&raw).is_err(), "{raw:?}"); // debug-ok: test assert + } + let mut trailing = encode("GET", "http://a.test/", &[], b""); + trailing.push(0); + assert!(decode_request(&trailing).is_err()); + let many: Vec<(String, String)> = (0..=MAX_HEADERS) + .map(|i| (format!("X-{i}"), "v".to_string())) + .collect(); + let refs: Vec<(&str, &str)> = many.iter().map(|(k, v)| (k.as_str(), v.as_str())).collect(); + assert!(decode_request(&encode("GET", "http://a.test/", &refs, b"")).is_err()); + } + + #[test] + fn truncated_lengths_never_panic() { + let raw = encode("GET", "http://a.test/", &[("A", "b")], b"body"); + for cut in 0..=raw.len() { + check_request_invariants(&raw[..cut]); + } + // A length prefix far past the buffer. + let mut raw = Vec::new(); + raw.extend_from_slice(&u32::MAX.to_le_bytes()); + assert!(decode_request(&raw).is_err()); + } + + #[test] + fn encodes_a_response() { + let out = encode_response(201, &[("a".to_string(), "b".to_string())], b"xyz"); + assert_eq!(&out[..4], &201u32.to_le_bytes()); + assert_eq!(&out[4..8], &1u32.to_le_bytes()); + assert_eq!(out.len(), 4 + 4 + (4 + 1 + 4 + 1) + 4 + 3); + assert!(out.ends_with(b"xyz")); + } +} diff --git a/crates/bashkit/src/builtins/cpython/mod.rs b/crates/bashkit/src/builtins/cpython/mod.rs index 01a789a6f..0555fa138 100644 --- a/crates/bashkit/src/builtins/cpython/mod.rs +++ b/crates/bashkit/src/builtins/cpython/mod.rs @@ -4,7 +4,8 @@ //! Decisions (see `knowledge/runtimes/cpython-wasm.md`): //! - Isolation comes from the wasm boundary, not from CPython. The guest can //! only reach what [`wasi`] hands it: the VFS, captured stdio, a host RNG and -//! clocks. No host filesystem, network, process or thread API exists. +//! clocks, plus HTTP through bashkit's `HttpClient` ([`http`]). No host +//! filesystem, socket, process or thread API exists. //! - No runtime opt-in env var (unlike Monty's `BASHKIT_ALLOW_INPROCESS_PYTHON`): //! the guest never runs native code in the host, so calling //! `BashBuilder::cpython()` is the opt-in. @@ -24,8 +25,11 @@ //! - Memory is capped by a store limiter; `memory.grow` past the cap fails and //! surfaces as `MemoryError` inside Python, not as a host error. +mod http; mod wasi; +pub(crate) use http::check_request_invariants as check_http_request_invariants; + use std::future::Future; use std::sync::OnceLock; use std::task::Poll; @@ -48,6 +52,8 @@ const DEFAULT_MAX_MEMORY: usize = 256 * 1024 * 1024; const DEFAULT_MAX_RECURSION: usize = 1000; /// Default cap on captured stdout + stderr. const DEFAULT_MAX_OUTPUT: usize = 16 * 1024 * 1024; +/// Default cap on HTTP requests per call. +const DEFAULT_MAX_HTTP_REQUESTS: usize = 100; /// Guest instructions between cooperative yields (~1 ms on Pulley). const YIELD_INTERVAL_FUEL: u64 = 100_000; /// Request-budget work units charged per call before the guest runs. @@ -72,6 +78,8 @@ pub struct CPythonLimits { /// Maximum captured stdout + stderr bytes; output past this is dropped /// and the result is marked truncated. pub max_output: usize, + /// Maximum HTTP requests one call may make (each redirect hop counts). + pub max_http_requests: usize, } impl Default for CPythonLimits { @@ -83,6 +91,7 @@ impl Default for CPythonLimits { max_call_depth: DEFAULT_MAX_RECURSION, }, max_output: DEFAULT_MAX_OUTPUT, + max_http_requests: DEFAULT_MAX_HTTP_REQUESTS, } } } @@ -116,6 +125,13 @@ impl CPythonLimits { self.max_output = bytes; self } + + /// Set the maximum number of HTTP requests per call. + #[must_use] + pub fn max_http_requests(mut self, n: usize) -> Self { + self.max_http_requests = n; + self + } } /// Concurrent guest instances per process (pooled allocator slots). Calls @@ -186,6 +202,7 @@ fn link(engine: Engine, slots: Option) -> wasmtime::Resu let module = bashkit_cpython_wasm::load_module(&engine)?; let mut linker = Linker::new(&engine); wasi::add_to_linker(&mut linker)?; + http::add_to_linker(&mut linker)?; let pre = linker.instantiate_pre(&module)?; Ok(Runtime { engine, pre, slots }) } @@ -228,7 +245,9 @@ impl Builtin for CPython { "python/python3: CPython 3.14 in a WebAssembly sandbox. Full pure-Python \ stdlib plus json, re, csv, sqlite3, zlib, hashlib, decimal, datetime. \ open()/pathlib/os work on the virtual filesystem. -c, -m, script files and \ - stdin work. No network, no subprocess, no threads, no pip or third-party packages.", + stdin work. HTTP only via urllib.request/http.client and only to hosts the \ + network allowlist permits; no raw sockets, no subprocess, no threads, no pip \ + or third-party packages.", ) } @@ -301,6 +320,11 @@ impl Builtin for CPython { stdlib_path: bashkit_cpython_wasm::STDLIB_ZIP_PATH, stdlib: bashkit_cpython_wasm::STDLIB_ZIP, deadline, + http: http::HttpState::new( + #[cfg(feature = "http_client")] + ctx.http_client.cloned(), + self.limits.max_http_requests, + ), }, ); diff --git a/crates/bashkit/src/builtins/cpython/wasi.rs b/crates/bashkit/src/builtins/cpython/wasi.rs index d357cafb9..43e6df45d 100644 --- a/crates/bashkit/src/builtins/cpython/wasi.rs +++ b/crates/bashkit/src/builtins/cpython/wasi.rs @@ -153,6 +153,7 @@ pub(crate) struct GuestConfig { pub(crate) stdlib_path: &'static str, pub(crate) stdlib: &'static [u8], pub(crate) deadline: crate::time_compat::Instant, + pub(crate) http: super::http::HttpState, } /// Everything one `python3` call's wasm instance can reach. @@ -176,6 +177,8 @@ pub(crate) struct GuestState { max_file_size: usize, /// Errno of the first failed write-back at exit, if any. pub(crate) flush_error: Option, + /// HTTP bridge state (`bashkit.http_request`). + pub(crate) http: super::http::HttpState, } /// Exit requested through `proc_exit`. @@ -237,9 +240,16 @@ impl GuestState { buffer_cap: config.max_memory, max_file_size, flush_error: None, + http: config.http, } } + /// Time left before the call deadline. + pub(crate) fn remaining(&self) -> Duration { + self.deadline + .saturating_duration_since(crate::time_compat::Instant::now()) + } + fn is_stdlib(&self, path: &Path) -> bool { path == self.stdlib_path } @@ -423,14 +433,22 @@ fn range(ptr: i32, len: u32, mem_len: usize) -> Result, E Ok(start..end) } -fn read_bytes(caller: &mut Caller<'_, GuestState>, ptr: i32, len: i32) -> Result, Errno> { +pub(super) fn read_bytes( + caller: &mut Caller<'_, GuestState>, + ptr: i32, + len: i32, +) -> Result, Errno> { let mem = memory(caller)?; let data = mem.data(&*caller); let r = range(ptr, len as u32, data.len())?; Ok(data[r].to_vec()) } -fn write_bytes(caller: &mut Caller<'_, GuestState>, ptr: i32, bytes: &[u8]) -> Result<(), Errno> { +pub(super) fn write_bytes( + caller: &mut Caller<'_, GuestState>, + ptr: i32, + bytes: &[u8], +) -> Result<(), Errno> { let mem = memory(caller)?; let data = mem.data_mut(&mut *caller); let r = range( @@ -442,7 +460,11 @@ fn write_bytes(caller: &mut Caller<'_, GuestState>, ptr: i32, bytes: &[u8]) -> R Ok(()) } -fn write_u32(caller: &mut Caller<'_, GuestState>, ptr: i32, v: u32) -> Result<(), Errno> { +pub(super) fn write_u32( + caller: &mut Caller<'_, GuestState>, + ptr: i32, + v: u32, +) -> Result<(), Errno> { write_bytes(caller, ptr, &v.to_le_bytes()) } diff --git a/crates/bashkit/src/builtins/mod.rs b/crates/bashkit/src/builtins/mod.rs index 91e878d36..7e6ada38c 100644 --- a/crates/bashkit/src/builtins/mod.rs +++ b/crates/bashkit/src/builtins/mod.rs @@ -281,6 +281,8 @@ pub use ssh::{Scp, Sftp, Ssh}; #[cfg(any(feature = "python", feature = "typescript", feature = "cpython"))] pub use runtime_limits::RuntimeLimits; +#[cfg(feature = "cpython")] +pub(crate) use cpython::check_http_request_invariants; #[cfg(feature = "cpython")] pub use cpython::{CPython, CPythonLimits}; #[cfg(feature = "python")] @@ -913,7 +915,7 @@ pub struct Context<'a> { /// a [`NetworkAllowlist`](crate::NetworkAllowlist) is configured via /// [`BashBuilder::network`](crate::BashBuilder::network). #[cfg(feature = "http_client")] - pub http_client: Option<&'a crate::network::HttpClient>, + pub http_client: Option<&'a std::sync::Arc>, /// Git client for git operations. /// diff --git a/crates/bashkit/src/interpreter/mod.rs b/crates/bashkit/src/interpreter/mod.rs index 5a1c50abe..8e0d08711 100644 --- a/crates/bashkit/src/interpreter/mod.rs +++ b/crates/bashkit/src/interpreter/mod.rs @@ -7339,7 +7339,7 @@ impl Interpreter { fs, stdin, #[cfg(feature = "http_client")] - http_client: self.http_client.as_deref(), + http_client: self.http_client.as_ref(), #[cfg(feature = "git")] git_client: self.git_client.as_ref(), #[cfg(feature = "ssh")] @@ -7404,7 +7404,7 @@ impl Interpreter { fs, stdin, #[cfg(feature = "http_client")] - http_client: self.http_client.as_deref(), + http_client: self.http_client.as_ref(), #[cfg(feature = "git")] git_client: self.git_client.as_ref(), #[cfg(feature = "ssh")] diff --git a/crates/bashkit/src/testing.rs b/crates/bashkit/src/testing.rs index 1aadcd953..d19167c3e 100644 --- a/crates/bashkit/src/testing.rs +++ b/crates/bashkit/src/testing.rs @@ -353,6 +353,16 @@ fn truncate(s: &str, max: usize) -> String { } } +/// Decode one CPython HTTP bridge request record (the bytes the guest hands +/// to `bashkit.http_request`) and assert what the host guarantees before +/// dispatch: allowed method, bounded URL and headers, no CR/LF/NUL in +/// values, no host-owned headers (`Host`, framing, hop-by-hop). Never +/// panics on malformed input. Used by the `cpython_http_fuzz` target. +#[cfg(feature = "cpython")] +pub fn check_cpython_http_request(data: &[u8]) { + crate::builtins::check_http_request_invariants(data); +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/bashkit/tests/integration/cpython_http_tests.rs b/crates/bashkit/tests/integration/cpython_http_tests.rs new file mode 100644 index 000000000..1e0e0eaa1 --- /dev/null +++ b/crates/bashkit/tests/integration/cpython_http_tests.rs @@ -0,0 +1,464 @@ +// HTTP from the CPython (WASI) python3 builtin through bashkit's egress +// pipeline (`cpython` + `http_client` features). +// +// The guest has no sockets: `http.client` / `urllib.request` hand each +// request to the host (`bashkit.http_request`), which runs it through +// `HttpClient` like `curl`. A recording transport stands in for the network. +// Threat IDs refer to knowledge/security/threat-model.md (TM-PY-CPY-003). + +#![cfg(all(feature = "cpython", feature = "http_client"))] + +use async_trait::async_trait; +use bashkit::testing::assert_no_leak; +use bashkit::{ + Bash, CPythonLimits, Credential, HttpResponse, HttpTransport, HttpTransportError, + HttpTransportRequest, NetworkAllowlist, +}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +const HOST: &str = "93.184.216.34"; + +type Log = Arc>>; + +/// Answers by path; records every request it receives. +#[derive(Clone)] +struct FakeServer { + log: Log, +} + +#[async_trait] +impl HttpTransport for FakeServer { + async fn execute( + &self, + request: HttpTransportRequest, + ) -> Result { + self.log.lock().unwrap().push(request.clone()); + let path = url::Url::parse(&request.url).unwrap().path().to_string(); + let reply = |status: u16, headers: &[(&str, &str)], body: &[u8]| HttpResponse { + status, + headers: headers + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(), + body: body.to_vec(), + }; + Ok(match path.as_str() { + "/json" => reply( + 200, + &[ + ("Content-Type", "application/json"), + ("X-Multi", "a"), + ("X-Multi", "b"), + ], + br#"{"ok": true}"#, + ), + "/echo" => { + let mut body = format!("{} ", request.method.as_str()).into_bytes(); + body.extend_from_slice(request.body.as_deref().unwrap_or_default()); + reply(200, &[("Transfer-Encoding", "chunked")], &body) + } + "/missing" => reply(404, &[("Content-Type", "text/plain")], b"nope"), + "/redirect" => reply(302, &[("Location", "/json")], b""), + "/redirect-out" => reply(302, &[("Location", "http://203.0.113.9/x")], b""), + "/slow" => { + tokio::time::sleep(Duration::from_secs(10)).await; + reply(200, &[], b"late") + } + _ => reply(200, &[("Content-Length", "5")], b"hello"), + }) + } +} + +fn server(allowlist: NetworkAllowlist) -> (bashkit::BashBuilder, Log) { + let log = Log::default(); + let builder = Bash::builder() + .cpython() + .network(allowlist) + .http_transport(Arc::new(FakeServer { log: log.clone() })); + (builder, log) +} + +fn allow_host() -> NetworkAllowlist { + NetworkAllowlist::new() + .allow(format!("http://{HOST}")) + .allow(format!("https://{HOST}")) +} + +async fn run(bash: &mut Bash, script: &str) -> bashkit::ExecResult { + let r = bash.exec(script).await.unwrap(); + // Error text from the bridge reaches scripts (and often stdout): neither + // stream may carry internal shapes or the injected secret. + assert_no_leak(&r, "cpython-http stderr", &["real-secret"]); + let mut out = r.clone(); + out.stderr = r.stdout.clone(); + assert_no_leak(&out, "cpython-http stdout", &["real-secret"]); + r +} + +fn header<'a>(req: &'a HttpTransportRequest, name: &str) -> Option<&'a str> { + req.headers + .iter() + .find(|(k, _)| k.eq_ignore_ascii_case(name)) + .map(|(_, v)| v.as_str()) +} + +#[tokio::test] +async fn urllib_https_get() { + let (builder, log) = server(allow_host()); + let mut bash = builder.build(); + let r = run( + &mut bash, + &format!( + "python3 -c ' +import json, urllib.request +with urllib.request.urlopen(\"https://{HOST}/json\") as resp: + print(resp.status, resp.headers[\"Content-Type\"], resp.headers.get_all(\"X-Multi\")) + print(json.load(resp)) +'" + ), + ) + .await; + assert_eq!(r.exit_code, 0, "{}", r.stderr); + assert_eq!(r.stdout, "200 application/json ['a', 'b']\n{'ok': True}\n"); + let log = log.lock().unwrap(); + assert_eq!(log.len(), 1); + assert_eq!(log[0].method.as_str(), "GET"); + assert_eq!(log[0].url, format!("https://{HOST}/json")); + assert!( + header(&log[0], "User-Agent") + .unwrap() + .starts_with("Python-urllib/3.14") + ); + // Host-owned headers never come from the guest. + assert!(header(&log[0], "Host").is_none()); + assert!(header(&log[0], "Connection").is_none()); +} + +#[tokio::test] +async fn urllib_post_body_and_http_error() { + let (builder, log) = server(allow_host()); + let mut bash = builder.build(); + let r = run( + &mut bash, + &format!( + "python3 -c ' +import urllib.request, urllib.error +req = urllib.request.Request(\"http://{HOST}/echo\", data=b\"a=1&b=2\", + headers={{\"Content-Type\": \"application/x-www-form-urlencoded\"}}) +print(urllib.request.urlopen(req).read().decode()) +try: + urllib.request.urlopen(\"http://{HOST}/missing\") +except urllib.error.HTTPError as e: + print(e.code, e.read().decode()) +'" + ), + ) + .await; + assert_eq!(r.exit_code, 0, "{}", r.stderr); + assert_eq!(r.stdout, "POST a=1&b=2\n404 nope\n"); + let log = log.lock().unwrap(); + assert_eq!(log[0].body.as_deref(), Some(b"a=1&b=2".as_slice())); + assert_eq!( + header(&log[0], "Content-Type"), + Some("application/x-www-form-urlencoded") + ); +} + +#[tokio::test] +async fn http_client_keep_alive_head_and_chunked_upload() { + let (builder, log) = server(allow_host()); + let mut bash = builder.build(); + let r = run( + &mut bash, + &format!( + "python3 -c ' +import http.client +c = http.client.HTTPSConnection(\"{HOST}\", 443, timeout=5) +c.request(\"GET\", \"/plain?q=1\") +r = c.getresponse(); print(r.status, r.reason, r.read()) +c.request(\"HEAD\", \"/plain\") +r = c.getresponse(); print(r.status, r.getheader(\"Content-Length\"), r.read()) +c.request(\"PUT\", \"/echo\", body=iter([b\"ab\", b\"cd\"]), encode_chunked=True) +r = c.getresponse(); print(r.status, r.read()) +c.close() +'" + ), + ) + .await; + assert_eq!(r.exit_code, 0, "{}", r.stderr); + assert_eq!(r.stdout, "200 OK b'hello'\n200 5 b''\n200 b'PUT abcd'\n"); + let log = log.lock().unwrap(); + assert_eq!(log.len(), 3); + assert_eq!(log[0].url, format!("https://{HOST}/plain?q=1")); + assert_eq!(log[1].method.as_str(), "HEAD"); + assert_eq!(log[2].body.as_deref(), Some(b"abcd".as_slice())); + assert!(header(&log[2], "Transfer-Encoding").is_none()); +} + +#[tokio::test] +async fn redirects_are_rechecked_per_hop() { + let (builder, log) = server(allow_host()); + let mut bash = builder.build(); + let r = run( + &mut bash, + &format!( + "python3 -c ' +import urllib.request, urllib.error +print(urllib.request.urlopen(\"http://{HOST}/redirect\").read()) +try: + urllib.request.urlopen(\"http://{HOST}/redirect-out\") +except urllib.error.URLError as e: + print(\"denied:\", e.reason) +'" + ), + ) + .await; + assert_eq!(r.exit_code, 0, "{}", r.stderr); + assert!( + r.stdout.starts_with("b'{\"ok\": true}'\ndenied: "), + "{}", + r.stdout + ); + assert!(r.stdout.contains("access denied"), "{}", r.stdout); + let urls: Vec = log.lock().unwrap().iter().map(|r| r.url.clone()).collect(); + // The off-allowlist hop never reached the transport. + assert_eq!( + urls, + vec![ + format!("http://{HOST}/redirect"), + format!("http://{HOST}/json"), + format!("http://{HOST}/redirect-out"), + ] + ); +} + +#[tokio::test] +async fn denied_and_private_hosts_never_reach_transport() { + let (builder, log) = server(NetworkAllowlist::allow_all()); + let mut bash = builder.build(); + let r = run( + &mut bash, + "python3 -c ' +import urllib.request, urllib.error +for url in (\"http://127.0.0.1/\", \"http://10.0.0.1/\", \"http://[::1]:8080/\", \"http://169.254.169.254/latest\"): + try: + urllib.request.urlopen(url) + print(\"ALLOWED\", url) + except urllib.error.URLError as e: + print(\"denied\", \"access denied\" in str(e.reason)) +'", + ) + .await; + assert_eq!(r.exit_code, 0, "{}", r.stderr); + assert_eq!(r.stdout, "denied True\n".repeat(4)); + assert!(log.lock().unwrap().is_empty()); + + let (builder, log) = server(allow_host()); + let mut bash = builder.build(); + let r = run( + &mut bash, + "python3 -c ' +import http.client +c = http.client.HTTPConnection(\"203.0.113.9\") +try: + c.request(\"GET\", \"/\") +except ConnectionError as e: + print(type(e).__name__, e) +'", + ) + .await; + assert!( + r.stdout.starts_with("ConnectionError access denied"), + "{}", + r.stdout + ); + assert!(log.lock().unwrap().is_empty()); +} + +#[tokio::test] +async fn host_header_cannot_retarget_request() { + // THREAT[TM-PY-CPY-003]: the allowlist checks the URL; a guest Host + // header must not steer the request to another virtual host. + let (builder, log) = server(allow_host()); + let mut bash = builder.build(); + let r = run( + &mut bash, + &format!( + "python3 -c ' +import http.client +c = http.client.HTTPConnection(\"{HOST}\") +c.putrequest(\"GET\", \"/plain\", skip_host=True) +c.putheader(\"Host\", \"internal.corp\") +c.putheader(\"X-Ok\", \"1\") +c.endheaders() +print(c.getresponse().status) +'" + ), + ) + .await; + assert_eq!(r.stdout, "200\n", "{}", r.stderr); + let log = log.lock().unwrap(); + assert_eq!(log[0].url, format!("http://{HOST}/plain")); + assert!(header(&log[0], "Host").is_none()); + assert_eq!(header(&log[0], "X-Ok"), Some("1")); +} + +#[tokio::test] +async fn header_injection_is_rejected() { + let (builder, log) = server(allow_host()); + let mut bash = builder.build(); + let r = run( + &mut bash, + &format!( + "python3 -c ' +import _bashkit +for headers in ([(\"X\", \"a\\r\\nEvil: 1\")], [(\"Bad Name\", \"v\")]): + try: + _bashkit.http(\"GET\", \"http://{HOST}/\", headers, None, None) + print(\"SENT\") + except ValueError as e: + print(\"rejected:\", e) +for method in (\"CONNECT\", \"TRACE\"): + try: + _bashkit.http(method, \"http://{HOST}/\", [], None, None) + print(\"SENT\") + except ValueError as e: + print(\"rejected:\", e) +'" + ), + ) + .await; + assert_eq!( + r.stdout, + "rejected: invalid header value\nrejected: invalid header name\n\ + rejected: unsupported HTTP method: CONNECT\nrejected: unsupported HTTP method: TRACE\n", + "{}", + r.stderr + ); + assert!(log.lock().unwrap().is_empty()); +} + +#[tokio::test] +async fn credentials_injected_on_host_only() { + let (builder, log) = server(allow_host()); + let mut bash = builder + .credential_placeholder( + "API_TOKEN", + &format!("https://{HOST}"), + Credential::bearer("real-secret"), + ) + .build(); + let r = run( + &mut bash, + &format!( + "python3 -c ' +import os, urllib.request +tok = os.environ[\"API_TOKEN\"] +print(tok.startswith(\"bk_placeholder_\"), \"real-secret\" in tok) +req = urllib.request.Request(\"https://{HOST}/plain\", headers={{\"Authorization\": \"Bearer \" + tok}}) +print(urllib.request.urlopen(req).read()) +'" + ), + ) + .await; + assert_eq!(r.stdout, "True False\nb'hello'\n", "{}", r.stderr); + let log = log.lock().unwrap(); + assert_eq!(header(&log[0], "Authorization"), Some("Bearer real-secret")); +} + +#[tokio::test] +async fn request_cap_per_call() { + let log = Log::default(); + let mut bash = Bash::builder() + .cpython_with_limits(CPythonLimits::default().max_http_requests(2)) + .network(allow_host()) + .http_transport(Arc::new(FakeServer { log: log.clone() })) + .build(); + let r = run( + &mut bash, + &format!( + "python3 -c ' +import urllib.request, urllib.error +for i in range(3): + try: + urllib.request.urlopen(\"http://{HOST}/plain\").read(); print(\"ok\") + except urllib.error.URLError as e: + print(e.reason) +'" + ), + ) + .await; + assert_eq!( + r.stdout, "ok\nok\ntoo many HTTP requests (limit 2 per call)\n", + "{}", + r.stderr + ); + assert_eq!(log.lock().unwrap().len(), 2); + // The cap is per call, not per session. + let r = run( + &mut bash, + &format!("python3 -c 'import urllib.request; print(urllib.request.urlopen(\"http://{HOST}/plain\").status)'"), + ) + .await; + assert_eq!(r.stdout, "200\n", "{}", r.stderr); +} + +#[tokio::test] +async fn request_timeout_raises() { + let (builder, _log) = server(allow_host()); + let mut bash = builder.build(); + let start = std::time::Instant::now(); + let r = run( + &mut bash, + &format!( + "python3 -c ' +import urllib.request, urllib.error +try: + urllib.request.urlopen(\"http://{HOST}/slow\", timeout=0.3) +except urllib.error.URLError as e: + print(type(e.reason).__name__, e.reason) +'" + ), + ) + .await; + assert_eq!(r.stdout, "TimeoutError timed out\n", "{}", r.stderr); + assert!(start.elapsed() < Duration::from_secs(5)); +} + +#[tokio::test] +async fn call_deadline_bounds_requests() { + let log = Log::default(); + let mut bash = Bash::builder() + .cpython_with_limits(CPythonLimits::default().max_duration(Duration::from_millis(800))) + .network(allow_host()) + .http_transport(Arc::new(FakeServer { log: log.clone() })) + .build(); + let start = std::time::Instant::now(); + let r = run( + &mut bash, + &format!( + "python3 -c 'import urllib.request; urllib.request.urlopen(\"http://{HOST}/slow\")'" + ), + ) + .await; + assert_ne!(r.exit_code, 0); + assert!(start.elapsed() < Duration::from_secs(5)); +} + +#[tokio::test] +async fn no_network_configured() { + let mut bash = Bash::builder().cpython().build(); + let r = run( + &mut bash, + "python3 -c ' +import urllib.request, urllib.error +try: + urllib.request.urlopen(\"https://example.com/\") +except urllib.error.URLError as e: + print(e.reason) +'", + ) + .await; + assert_eq!(r.stdout, "network access not configured\n", "{}", r.stderr); +} diff --git a/crates/bashkit/tests/integration/main.rs b/crates/bashkit/tests/integration/main.rs index 24dd28af9..422410653 100644 --- a/crates/bashkit/tests/integration/main.rs +++ b/crates/bashkit/tests/integration/main.rs @@ -49,6 +49,7 @@ pub mod memory_growth_security_tests; // Both assert named-IANA-zone behavior, which only exists with `tzdata`. // `date_timezone_no_tzdata_tests` covers the closed-to-UTC side. pub mod cpython_capability_tests; +pub mod cpython_http_tests; pub mod cpython_integration_tests; pub mod cpython_security_tests; #[cfg(feature = "tzdata")] diff --git a/knowledge/log.md b/knowledge/log.md index fe2655019..4fd44cb4a 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -5,6 +5,7 @@ * **Fix**: A backslash outside quotes keeps `$` and `` ` `` literal. `echo \$HOME` printed the home directory; it now prints `$HOME`, as do `a\$x`, `\${x}` and `x=\$y` (the lexer emits the same NUL literal marker as quoted `"\$"`). Found while working gap telemetry. * **Fix**: `$'\x1e'` and `$'\x1f'` bytes (record/unit separators) are kept as data. They collided with the lexer's quote-boundary markers and were silently dropped, which broke the common `SEP=$'\x1f'` composite-key idiom for associative arrays (`v["$section"$'\x1f'"$key"]`). Decoded ANSI-C content now NUL-escapes those bytes like `$`. Found by gap telemetry (`config_ini_merge` replay printed nothing). * **Fix**: A usage error in a bundled builtin no longer aborts the script. `grep -E '('`, `grep -P` with a bad pattern, `rg` with a bad regex, an awk syntax error, `grep` with no pattern and `grep -f missing` now fail that one command with exit 2 and the message on stderr, and the script continues (bash semantics; `set -e` still stops). `Err` from custom builtins (`BashBuilder::builtin`, registry, resolver) keeps aborting, per the public contract. See foundations/builtins "Errors: Failed Command vs Aborted Execution". +* **Feature**: CPython `urllib.request`/`http.client` reach the network through bashkit's egress pipeline. New host import `bashkit.http_request` runs each request through `HttpClient` like `curl` (allowlist, SSRF, credential injection, signing, transport, response cap); TLS on the host, no sockets in the guest. The stdlib adapter swaps only http.client's socket for a bridge socket, so the real request/response code runs. Host validates the request record and drops host-owned headers (TM-PY-CPY-012, `cpython_http_fuzz`); `CPythonLimits::max_http_requests` caps requests per call. L-CPY-002 narrowed. Next: `requests` and `httpx2` (pydantic/httpx2, with `import httpx` aliased) in a separate packages crate behind opt-in `cpython-http`. See [CPython WebAssembly Runtime](runtimes/cpython-wasm.md#http-bridge). * **Feature**: `gawk`, `mawk` and `nawk` run the awk builtin (same engine, including gawk extensions it already had such as `gensub`, `toupper`). Gap telemetry showed agents probing `gawk` by name. `--version` still identifies as bashkit awk. * **Fix**: `tar -P`/`--absolute-names` keeps the leading `/` (no notice), `-T FILE`/`--files-from` reads member names (blank lines skipped, `-` is stdin), and `--` ends options. `ls` implements the quoting styles: `-b`/`--escape`, `-N`, `-Q`, `--quoting-style=literal|escape|c|shell|shell-always|shell-escape|shell-escape-always` (GNU rules, including `"it's"` and headers left unescaped under `-b`); `locale`/`clocale` still report not implemented. Found by gap telemetry. * **Fix**: Arithmetic assigns to array elements. `((a[i]++))`, `((c[$k]+=3))`, `((a[2]=5))`, `((--a[0]))`, `let a[1]++` and `$((c[x]--))` read and write the element (indexed subscripts are arithmetic and accept negative indexes; associative keys are literal, quotes dropped as in `((c["x y"]++))`). Before, the update was lost or written to a scalar. Writes go through the budgeted `${a[i]:=v}` writer, so array memory limits still apply. Found by gap telemetry. diff --git a/knowledge/operations/limitations.md b/knowledge/operations/limitations.md index 58d621966..9d57724ac 100644 --- a/knowledge/operations/limitations.md +++ b/knowledge/operations/limitations.md @@ -126,7 +126,7 @@ Boundaries of the WebAssembly CPython guest; see | ID | Limitation | Why | Evidence | |----|------------|-----|----------| | L-CPY-001 | No subprocesses from Python (`subprocess`, `os.system`, `os.fork`, `os.popen` fail); Python cannot call back into the shell | The guest has no process API; a shell bridge is a deliberate follow-up, not an accident | `processes_unavailable` (cpython_security_tests) | -| L-CPY-002 | No network from Python (`socket`, `urllib.request` cannot connect) | No socket imports in the WASI host; egress stays with the allowlisted shell builtins | TM-PY-CPY-003, `network_unavailable` | +| L-CPY-002 | No raw network from Python: `socket` cannot connect and `ssl` is not built. HTTP works only through `urllib.request`/`http.client` via the host's egress pipeline (needs `http_client` + an allowlist); responses are buffered up to the response cap; proxy, `ssl` context and client-certificate settings are ignored | No socket imports in the WASI host; HTTP goes through `HttpClient` like `curl`, TLS on the host | TM-PY-CPY-003, `network_unavailable`, `cpython_http_tests` | | L-CPY-003 | No threads, `multiprocessing`, `ctypes`, native extensions or third-party packages | Single-threaded wasm guest with only the bundled pure-Python stdlib | `threads_and_native_code_unavailable` | | L-CPY-004 | `errno` values are WASI's (`ENOENT` is 44); exception types and the `errno` module agree | wasi-libc numbering; rewriting it would desync the guest's `errno` module | `cpython_integration_tests::missing_script_exits_2`, stance | | L-CPY-005 | `hash()` of `str`/`bytes` uses one seed baked into the snapshot | The seed is chosen during snapshot initialization; per-call re-seeding would require re-hashing every interned object | stance | diff --git a/knowledge/runtimes/cpython-wasm.md b/knowledge/runtimes/cpython-wasm.md index 673fa6221..63024d84e 100644 --- a/knowledge/runtimes/cpython-wasm.md +++ b/knowledge/runtimes/cpython-wasm.md @@ -113,11 +113,55 @@ attack surface (TM-PY-CPY-011). from one `build.sh` run. - One instance serves one call; state is never reused. +### HTTP bridge + +Python HTTP leaves the guest at the HTTP level, never as sockets: +`_bashkit.http(method, url, headers, body, timeout)` (C, `bashkit_main.c`) +calls the host import `bashkit.http_request`, which decodes the request and +runs it through `HttpClient::request_with_timeouts`, the same pipeline as +`curl` (allowlist, SSRF precheck, `before_http` hooks and credential +injection, signing, the embedder's `HttpTransport`, response cap, +`after_http`). Rejected: socket emulation (needs OpenSSL and TLS in the guest, +and policy could only see host:port) and shelling out to `curl` (needs a +process bridge, quoting on every request). + +- Wire format: u32 little-endian length-prefixed fields. The response comes + back in two steps (`http_request` reports the size, `http_take` copies), + so the guest allocates once and the copy counts against its memory limit. +- Host validation before dispatch: methods GET/POST/PUT/DELETE/HEAD/PATCH, + URL <= 8 KiB with no controls or spaces, <= 128 headers and 64 KiB, + token header names, no CR/LF/NUL in values, body <= 16 MiB. Host-owned + headers (`Host`, framing, hop-by-hop, proxy) are dropped, so a guest + cannot retarget the request past the allowlist or smuggle a second one. + `check_request_invariants` is the fuzz entry (`cpython_http_fuzz`). +- Timeout = min(request timeout, call deadline); `CPythonLimits::max_http_requests` + (default 100) caps requests per call. Redirects are followed by Python, + so every hop is a new, re-checked request. +- No network configured, or no `http_client` feature: every request fails + with "network access not configured". Errors reach Python as + `ConnectionError`, `TimeoutError` or `ValueError` (invalid request), text + capped at 512 bytes. +- Stdlib adapter (`_bashkit_http.py`): http.client keeps its own request + and response code; only the socket is swapped for a bridge socket that + parses the bytes http.client writes (incl. chunked uploads), sends them + once complete, and serves the reply as HTTP/1.1 bytes the real + `HTTPResponse` parses. `HTTPSConnection` is defined although `ssl` is not + built, so urllib.request registers https. Applied by a footer `build.sh` + appends to `http/client.py`, so scripts that never import it pay nothing. +- The guest is linked with the `bashkit` imports; `wasmtime wizer` and the + build smoke test run with `-W unknown-imports-trap=y` since init never + calls them. +- Next: `requests`/`urllib3` and `httpx2` ship in a separate packages crate + behind the opt-in `cpython-http` feature (`import httpx` aliases + `httpx2`). + ### WASI host decisions - Only `wasi_snapshot_preview1`, only against bashkit state: VFS, captured stdin bytes, in-memory stdout/stderr, host RNG and clocks. Socket calls return `ENOTSUP`; there is no process API. +- One more import module, `bashkit` (`builtins/cpython/http.rs`), carries + HTTP. See [HTTP bridge](#http-bridge). - Files are whole in-memory buffers written back on close/sync/exit. Buffers are bounded by `max_file_size` and, in total, by the call's memory budget. - Paths normalize lexically and clamp at `/`. Symlinks resolve inside the diff --git a/knowledge/security/threat-model.md b/knowledge/security/threat-model.md index a20e0479e..e322139a6 100644 --- a/knowledge/security/threat-model.md +++ b/knowledge/security/threat-model.md @@ -1814,7 +1814,8 @@ python code -> CPython (wasm guest) -> wasi_snapshot_preview1 imports -> WASI ho |----|--------|----------|------------|------| | TM-PY-CPY-001 | Host filesystem access (absolute paths, `..`, `/proc`, symlinks) | Critical | Only the VFS is reachable; paths clamp at `/`; symlinks resolve inside the VFS only, `lstat` reports the link | `host_files_are_not_reachable`, `parent_traversal_is_clamped_at_vfs_root`, `host_system_paths_absent`, `rootfs_files_come_from_the_vfs_not_the_host`, `symlinks_resolve_inside_the_vfs_only` | | TM-PY-CPY-002 | Tampering with or shadowing the stdlib | High | Stdlib zip is a read-only host overlay; tenant files next to it are not on `sys.path`; preloaded modules come from the snapshot | `stdlib_zip_is_read_only`, `vfs_files_cannot_shadow_stdlib` | -| TM-PY-CPY-003 | Network, process, thread or native-code escape | Critical | No socket or process imports; sockets return `ENOTSUP`; no threads, `ctypes` or dynamic loading in the guest | `network_unavailable`, `urllib_cannot_fetch`, `processes_unavailable`, `threads_and_native_code_unavailable` | +| TM-PY-CPY-003 | Network, process, thread or native-code escape | Critical | No socket or process imports; sockets return `ENOTSUP`; no threads, `ctypes` or dynamic loading in the guest. HTTP only through the `bashkit.http_request` import, which runs `HttpClient` (allowlist, SSRF precheck, hooks, signing, transport, response cap); no network configured means every request is denied | `network_unavailable`, `urllib_cannot_fetch`, `processes_unavailable`, `threads_and_native_code_unavailable`, `denied_and_private_hosts_never_reach_transport`, `redirects_are_rechecked_per_hop`, `no_network_configured` | +| TM-PY-CPY-012 | HTTP bridge abuse: request smuggling or retargeting (CRLF, `Host`, framing headers, odd methods), reading injected credentials, request floods, hanging calls | High | Host validates the request record before dispatch (method set, URL and header caps, token names, no CR/LF/NUL) and drops host-owned headers; credentials are injected on the host after the guest hands over the request; per-call request cap (`max_http_requests`); timeout capped by the call deadline; errors Display-only, <= 512 bytes | `host_header_cannot_retarget_request`, `header_injection_is_rejected`, `credentials_injected_on_host_only`, `request_cap_per_call`, `call_deadline_bounds_requests`, http.rs unit tests, `cpython_http_fuzz` | | TM-PY-CPY-004 | CPU exhaustion (busy loops, swallowed exceptions, sleeps) | High | Fuel-driven async yields; every poll checks the call deadline and the request `ExecutionBudget`; sleeps never pass the deadline | `infinite_loop_times_out`, `sleep_is_bounded_by_deadline`, `swallowing_exceptions_cannot_escape_timeout`, `shell_timeout_tighter_than_python_limit_wins`, `cancellation_stops_busy_guest` | | TM-PY-CPY-005 | Memory exhaustion (guest heap, host-side file buffers) | High | Store limiter caps linear memory (MemoryError in Python); open-file buffers share the same budget; VFS limits apply | `huge_allocation_raises_memory_error`, `incremental_growth_raises_memory_error`, `file_buffers_share_the_memory_budget`, `vfs_file_size_limit_applies` | | TM-PY-CPY-006 | Output and descriptor floods | Medium | `max_output` cap with truncation note; 1024 fds per call | `output_is_capped_and_marked_truncated`, `output_flood_loop_is_bounded`, `descriptor_table_is_bounded` |