feat(drivers): host.http_request with a per-driver cookie jar

.changeset/lua-http-request-cookie-jar.md

@@ -0,0 +1,5 @@
+---
+"ftw": patch
+---
+
+Lua drivers can now sign in through a web login. The new `host.http_request` returns the status, headers and redirect target. The host keeps the session cookies for the driver's allowed hosts, in memory only. A read-only driver may declare several sign-in paths with `auth_post_paths`. This lets the VW Group driver renew its portal session itself, instead of the owner pasting a new cookie every hour.

docs/writing-a-driver.md

@@ -41,7 +41,7 @@ current host API and the source of truth for it. Today it registers:
 | Decoding | `decode_string`, `decode_i16`, `decode_i32_be`, `decode_i32_le`, `decode_u32_be`, `decode_u32_le` |
 | Modbus | `modbus_read`, `write`, `write_registers` (canonical); `modbus_write`, `modbus_write_multi` (legacy aliases) |
 | MQTT | `mqtt_pub`, `mqtt_sub`, `mqtt_publish`, `mqtt_subscribe`, `mqtt_messages` |
-| HTTP | `http_get`, `http_post`, `http_patch` |
+| HTTP | `http_get`, `http_post`, `http_patch`, `http_request`, `http_cookies_clear` |
 | WebSocket | `ws_open`, `ws_send`, `ws_messages`, `ws_is_open`, `ws_close` |
 | Raw TCP | `tcp_open`, `tcp_recv`, `tcp_close`, `tcp_is_open` |
 | Serial | `serial_read` |
@@ -60,6 +60,22 @@ not enough, it also needs `capabilities.http.allow_write`. It refuses to follow
 redirects, because Go re-issues a redirected `PATCH` as a body-less GET and a
 device write that never landed would otherwise report success.
 
+`http_request{method, url, headers, body}` is for a driver that has to sign in
+through a web login before it can read. It returns `{status, headers,
+location, body}` for every status instead of turning 4xx into an error, and it
+never follows a redirect: the driver reads `location` and makes the next call
+itself, so the host checks every hop against `allowed_hosts`. It accepts GET
+and POST over https only, and only with a non-empty `allowed_hosts`. POST has
+the same gate as `http_post`. The host keeps the driver's session cookies in
+an in-memory jar that stores and sends them only for allowed hosts. Lua never
+sees them: `headers` leaves out `Set-Cookie`. `http_cookies_clear()` empties
+the jar before a fresh sign-in. Header names are lowercase, so a driver
+without a wall clock can read the server time from `headers.date`.
+
+A read-only driver may POST only to the sign-in paths its `DRIVER` block
+declares: `auth_post_path` for one path, or `auth_post_paths` when the login
+posts more than one form. The host matches each exactly.
+
 A driver with an opt-in write path names it in its `DRIVER` block —
 `write_capabilities = { "solar_pv" }` for a driver that feeds a heat pump's
 own solar-surplus input. The Settings UI offers a switch for a path it
@@ -122,7 +138,7 @@ may omit the default hook because Core cannot dispatch commands to them.
 
 `driver_fingerprint(target)` is an optional passive setup probe. It must never
 reconfigure the device. The host denies mutating verbs (`modbus_write`,
-`mqtt_pub`, `http_post`, `http_patch`) for that VM, including bundled drivers
+`mqtt_pub`, `http_post`, `http_patch`, and POST through `http_request`) for that VM, including bundled drivers
 that may otherwise write.
 
 Call `host.set_make` and `host.set_sn` as soon as stable identity is known.

go/internal/driverrepo/runtime_policy.go

@@ -269,6 +269,7 @@ func (m *Manager) directManifestRuntimePolicy(
 		// manifest names. An unsigned or absent value leaves it empty, which
 		// is the same as having no exemption at all.
 		AuthPostPath:  matched.Metadata.AuthPostPath,
+		AuthPostPaths: append([]string(nil), matched.Metadata.AuthPostPaths...),
 		ConfigSecrets: append([]string(nil), matched.Metadata.ConfigSecrets...),
 	}, nil
 }

go/internal/drivers/catalog.go

@@ -48,6 +48,8 @@ type CatalogEntry struct {
 	// that reads a vendor cloud has to POST for a token before it can read,
 	// and that POST is not actuation. Only meaningful with ReadOnly.
 	AuthPostPath string `json:"auth_post_path,omitempty"`
+	// AuthPostPaths lists further sign-in paths for a multi-step login.
+	AuthPostPaths []string `json:"auth_post_paths,omitempty"`
 	// ReadOnly means the driver never accepts dispatch commands. The catalog
 	// UI uses it to avoid presenting battery capacity as a control opt-in.
 	ReadOnly bool `json:"read_only,omitempty"`
@@ -227,6 +229,7 @@ func parseCatalogEntry(path string) (CatalogEntry, error) {
 	e.WriteCapabilities = pickList(block, "write_capabilities")
 	e.Replaces = pickList(block, "replaces")
 	e.AuthPostPath = pickString(block, "auth_post_path")
+	e.AuthPostPaths = pickList(block, "auth_post_paths")
 	e.Controls = pickControls(block)
 	return e, nil
 }

go/internal/drivers/catalog_test.go

@@ -163,6 +163,23 @@ func TestLoadCatalogReadsAuthPostPath(t *testing.T) {
 	}
 }
 
+func TestLoadCatalogReadsAuthPostPaths(t *testing.T) {
+	dir := t.TempDir()
+	src := "DRIVER = {\n  id = \"vag\",\n  name = \"VAG\",\n  read_only = true,\n" +
+		"  auth_post_paths = { \"/a/login/identifier\", \"/a/login/authenticate\" },\n}\n"
+	if err := os.WriteFile(filepath.Join(dir, "vag.lua"), []byte(src), 0644); err != nil {
+		t.Fatal(err)
+	}
+	entries, err := LoadCatalog(dir)
+	if err != nil || len(entries) != 1 {
+		t.Fatalf("LoadCatalog: %v %v", entries, err)
+	}
+	got := entries[0].AuthPostPaths
+	if len(got) != 2 || got[0] != "/a/login/identifier" || got[1] != "/a/login/authenticate" {
+		t.Fatalf("AuthPostPaths = %q", got)
+	}
+}
+
 func TestCatalogMyUplinkDeclaresAuthPostPath(t *testing.T) {
 	entries, err := LoadCatalog("../../../drivers")
 	if err != nil {

go/internal/drivers/host.go

@@ -240,15 +240,20 @@ func (h *HostEnv) allowAuthPost(rawURL string) bool {
 	if h.RuntimePolicy == nil || !h.RuntimePolicy.ReadOnly {
 		return false
 	}
-	declared := h.RuntimePolicy.AuthPostPath
-	if declared == "" || !h.RuntimePolicy.allows("http.post") {
+	declared := h.RuntimePolicy.authPaths()
+	if len(declared) == 0 || !h.RuntimePolicy.allows("http.post") {
 		return false
 	}
 	parsed, err := net_url.Parse(rawURL)
 	if err != nil {
 		return false
 	}
-	return parsed.Path == declared
+	for _, path := range declared {
+		if parsed.Path == path {
+			return true
+		}
+	}
+	return false
 }
 
 func (h *HostEnv) allowWrite(permission string) error {

go/internal/drivers/lua.go

@@ -1073,6 +1073,8 @@ func registerHost(L *lua.LState, env *HostEnv) {
 	// host.http_post(url, body, headers?) → (body, nil) or (nil, error_string)
 	// host.http_patch(url, body, headers?) → (body, nil) or (nil, error_string);
 	//   the mutating verb, gated by capabilities.http.allow_write (default off)
+	// host.http_request{method, url, headers?, body?} → response table; see
+	//   lua_http_request.go
 	// headers is an optional Lua table {["Content-Type"]="application/json", ...}
 	rawTLSPin := strings.TrimSpace(env.HTTPTLSPinSHA256)
 	tlsPin := normalizeHexFingerprint(rawTLSPin)
@@ -1403,6 +1405,8 @@ func registerHost(L *lua.LState, env *HostEnv) {
 		return 1
 	}))
 
+	registerHTTPRequest(L, host, env, httpClient, hostAllowed)
+
 	// ---- WebSocket capability ----
 	// host.ws_open(url, headers?)      → (true, nil) or (nil, error_string)
 	// host.ws_send(text)               → (true, nil) or (nil, error_string)

go/internal/drivers/lua_http_request.go

@@ -0,0 +1,179 @@
+package drivers
+
+import (
+	"fmt"
+	"io"
+	net_http "net/http"
+	"net/http/cookiejar"
+	net_url "net/url"
+	"sort"
+	"strings"
+
+	lua "github.com/yuin/gopher-lua"
+	"golang.org/x/net/publicsuffix"
+)
+
+// host.http_request{method, url, headers?, body?} returns
+// {status, headers, location, body} or (nil, error_string).
+//
+// It is for a driver that has to sign in through a web login before it can
+// read: the status and Location come back to Lua instead of being followed or
+// turned into an error, and session cookies live in a jar the host keeps for
+// this driver. The jar is in memory only, so a driver restart starts a clean
+// session. Lua never sees the cookies: Set-Cookie is left out of headers.
+//
+// Rules on top of the http_get/http_post ones:
+//   - allowed_hosts must be non-empty, and the URL must be https;
+//   - only GET and POST, with POST under the same write or sign-in gate as
+//     http_post;
+//   - redirects are never followed, so every hop is a separate call the
+//     host checks against allowed_hosts;
+//   - the jar stores and sends cookies only for allowed hosts.
+//
+// host.http_cookies_clear() empties the jar before a fresh sign-in.
+func registerHTTPRequest(L *lua.LState, host *lua.LTable, env *HostEnv, base *net_http.Client, hostAllowed func(string) (bool, string)) {
+	newJar := func() *allowedHostJar {
+		inner, _ := cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List})
+		return &allowedHostJar{inner: inner, allowed: hostAllowed}
+	}
+	jar := newJar()
+	client := &net_http.Client{
+		Timeout:   base.Timeout,
+		Transport: base.Transport,
+		Jar:       jar,
+		CheckRedirect: func(*net_http.Request, []*net_http.Request) error {
+			return net_http.ErrUseLastResponse
+		},
+	}
+
+	host.RawSetString("http_cookies_clear", L.NewFunction(func(L *lua.LState) int {
+		jar = newJar()
+		client.Jar = jar
+		return 0
+	}))
+
+	host.RawSetString("http_request", L.NewFunction(func(L *lua.LState) int {
+		fail := func(msg string) int {
+			L.Push(lua.LNil)
+			L.Push(lua.LString(msg))
+			return 2
+		}
+		opts := L.CheckTable(1)
+		method := strings.ToUpper(lua.LVAsString(opts.RawGetString("method")))
+		if method == "" {
+			method = "GET"
+		}
+		rawURL := lua.LVAsString(opts.RawGetString("url"))
+		if !env.HTTP {
+			return fail("http: capability not granted")
+		}
+		if len(env.HTTPAllowedHosts) == 0 {
+			return fail("http_request: requires a non-empty allowed_hosts")
+		}
+		switch method {
+		case "GET":
+			if !env.permissionAllowed("http.get") {
+				return fail("http.get: permission not granted by signed package")
+			}
+		case "POST":
+			if !env.allowAuthPost(rawURL) {
+				if err := env.allowWrite("http.post"); err != nil {
+					return fail(err.Error())
+				}
+			}
+		default:
+			return fail(fmt.Sprintf("http_request: method %q not supported (GET or POST)", method))
+		}
+		u, err := net_url.Parse(rawURL)
+		if err != nil || !strings.EqualFold(u.Scheme, "https") {
+			return fail("http_request: requires an https URL")
+		}
+		if ok, reason := hostAllowed(rawURL); !ok {
+			return fail("http: " + reason)
+		}
+
+		var body io.Reader
+		if v := opts.RawGetString("body"); v != lua.LNil {
+			body = strings.NewReader(lua.LVAsString(v))
+		}
+		var req *net_http.Request
+		if method == "GET" {
+			req, err = net_http.NewRequestWithContext(luaCallContext(L), method, rawURL, body)
+		} else {
+			// Same ordering rule as http_post: do not cancel a request the
+			// server may already have acted on.
+			req, err = net_http.NewRequest(method, rawURL, body)
+		}
+		if err != nil {
+			return fail(err.Error())
+		}
+		if headers, ok := opts.RawGetString("headers").(*lua.LTable); ok {
+			headers.ForEach(func(k, v lua.LValue) {
+				if ks, ok := k.(lua.LString); ok {
+					req.Header.Set(string(ks), v.String())
+				}
+			})
+		}
+		resp, err := client.Do(req)
+		if err != nil {
+			return fail(err.Error())
+		}
+		defer resp.Body.Close()
+		data, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
+		if err != nil {
+			return fail(err.Error())
+		}
+
+		out := L.NewTable()
+		out.RawSetString("status", lua.LNumber(resp.StatusCode))
+		hdrs := L.NewTable()
+		names := make([]string, 0, len(resp.Header))
+		for name := range resp.Header {
+			names = append(names, name)
+		}
+		sort.Strings(names)
+		for _, name := range names {
+			lower := strings.ToLower(name)
+			if lower == "set-cookie" {
+				continue
+			}
+			hdrs.RawSetString(lower, lua.LString(strings.Join(resp.Header[name], ", ")))
+		}
+		out.RawSetString("headers", hdrs)
+		if loc, err := resp.Location(); err == nil {
+			out.RawSetString("location", lua.LString(loc.String()))
+		}
+		out.RawSetString("body", lua.LString(string(data)))
+		L.Push(out)
+		return 1
+	}))
+}
+
+// allowedHostJar keeps a driver's session cookies to the hosts it may reach
+// over https. A cookie a server sets for any other host is dropped, and none
+// is ever sent elsewhere.
+type allowedHostJar struct {
+	inner   *cookiejar.Jar
+	allowed func(string) (bool, string)
+}
+
+func (j *allowedHostJar) ok(u *net_url.URL) bool {
+	if u == nil || !strings.EqualFold(u.Scheme, "https") {
+		return false
+	}
+	ok, _ := j.allowed(u.String())
+	return ok
+}
+
+func (j *allowedHostJar) SetCookies(u *net_url.URL, cookies []*net_http.Cookie) {
+	if j.ok(u) {
+		j.inner.SetCookies(u, cookies)
+	}
+}
+
+func (j *allowedHostJar) Cookies(u *net_url.URL) []*net_http.Cookie {
+	if !j.ok(u) {
+		return nil
+	}
+	return j.inner.Cookies(u)
+}