Fix archival protocol compatibility and fallback

.github/workflows/ci.yml

@@ -44,6 +44,9 @@ jobs:
       - name: test
         run: go test -mod=readonly -timeout=5m ./...
 
+      - name: historical method generator tests
+        run: python3 -m unittest discover -s tools -p test_generate_history_methods.py
+
   race:
     name: race detector
     runs-on: ubuntu-latest

.gitignore

@@ -63,3 +63,4 @@ docker-compose.override.yaml
 *.test.bin
 *.tmp
 testdata.local/
+__pycache__/

Dockerfile

@@ -17,6 +17,6 @@ RUN go build -mod=readonly -trimpath \
 FROM gcr.io/distroless/base-debian12:nonroot
 COPY --from=build /out/stitch /usr/local/bin/stitch
 USER nonroot:nonroot
-EXPOSE 5001 5002 5003 5005 5006 5007 5008 9091
+EXPOSE 5001 5002 5003 5004 5005 5006 5007 5008 9091
 ENTRYPOINT ["/usr/local/bin/stitch"]
 CMD ["start", "--config", "/etc/stitch/config.yaml"]

docs/archival-compatibility.md

@@ -0,0 +1,68 @@
+# Archival protocol support
+
+The RPC listener accepts CometBFT `/websocket` connections. Ordinary JSON-RPC
+requests use the same height routing and historical fallback as HTTP, including
+when sent on a socket with an active subscription. `subscribe`, `unsubscribe`
+and `unsubscribe_all` use a selected tip backend and preserve request IDs.
+The upstream WebSocket address is derived from the backend's RPC URL.
+
+On upstream loss or a full message queue, Stitch closes the client with code
+1013. Clients must reconnect and reconcile missed events; CometBFT subscriptions
+do not provide replay. Ping/pong, disconnect cancellation and graceful shutdown
+are supported. Each message is limited to 32 MiB, with queues bounded by both
+message count and bytes. Subscription traffic is not cached.
+
+## Browser and native gRPC on one hostname
+
+The optional `grpc_web` listener shares the native gRPC server and its routing:
+
+```yaml
+listen:
+  grpc: { addr: "0.0.0.0:5002" }
+  grpc_web:
+    addr: "0.0.0.0:5004"
+    allowed_origins: ["https://app.example.com"]
+```
+
+An explicit `"*"` allows any browser origin; an empty list denies cross-origin
+requests. The listener supports binary/text gRPC-Web, response status/trailers,
+unary and server-streaming calls, and ordinary HTTP fallback to Cosmos REST.
+Native `application/grpc` and `application/grpc+proto` requests use HTTP/2 h2c.
+Native gRPC over HTTP/1 is rejected. The separate native port is unchanged.
+The nonstandard gRPC-Web WebSocket transport is not enabled.
+
+The listener requires `listen.grpc`. Configuration changes require a restart.
+Native messages remain capped at 64 MiB; encoded web request bodies are capped
+at 90 MiB. The ingress must allow configured CORS preflights while retaining
+authorization on actual requests. Listener CORS does not replace ingress auth.
+
+## Missing historical state
+
+For an explicitly historical, idempotent request, known Cosmos store-retention
+errors try another eligible backend within `policies.failover.max_attempts`.
+The requested height, payload and metadata are preserved. Backend coverage is
+not expanded and a historical query never silently becomes a latest query.
+
+REST errors and CometBFT ABCI errors use a bounded structured-error inspection.
+Other responses pass through unchanged. If all candidates lack the requested
+state, the final upstream error is returned. Missing state does not count as a
+shared circuit failure, and failed ABCI responses are not cached.
+
+gRPC retries are limited to schema-verified unary read methods. Unknown methods,
+broadcasts and streaming methods retain transparent forwarding. A response is
+never retried after any message has been delivered. Headers and trailers from
+failed attempts do not leak into a successful response; the final failure's
+metadata is retained when all candidates fail. Transport/deadline failures still
+affect backend health; unrelated application errors are returned unchanged.
+
+The exact method inventory is generated from pinned Injective, Cosmos SDK, IBC
+and CosmWasm schemas. Run `python3 tools/generate-history-methods.py` with an
+authenticated `gh` CLI to regenerate it; source revisions and schema links are
+recorded in `internal/server/cosmos_grpc/history_methods.go`.
+
+## Asia migration
+
+The archival Stitch release handles Cosmos/CometBFT chain traffic. EVM traffic
+uses the existing Asia EVM Gateway, configured from height 127250000, without
+archival IP/key authorization or rate tiers. Exchange endpoints remain on nginx.
+Public routes and DNS are separate rollout steps.

go.mod

@@ -7,7 +7,9 @@ require (
 	github.com/gorilla/websocket v1.5.3
 	github.com/mwitkow/grpc-proxy v0.0.0-20230212185441-f345521cb9c9
 	github.com/prometheus/client_golang v1.20.5
+	github.com/rs/cors v1.11.1
 	github.com/spf13/cobra v1.8.1
+	github.com/traefik/grpc-web v0.16.0
 	golang.org/x/sync v0.10.0
 	google.golang.org/grpc v1.66.0
 	google.golang.org/protobuf v1.36.11
@@ -16,7 +18,9 @@ require (
 
 require (
 	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/klauspost/compress v1.17.9 // indirect
 	github.com/kr/text v0.2.0 // indirect
@@ -30,4 +34,5 @@ require (
 	golang.org/x/sys v0.22.0 // indirect
 	golang.org/x/text v0.16.0 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20260504160031-60b97b32f348 // indirect
+	nhooyr.io/websocket v1.8.17 // indirect
 )

go.sum

@@ -2,6 +2,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM=
+github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
@@ -12,6 +14,8 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f h1:U5y3Y5UE0w7amNe7Z5G/twsBW0KEalRQXZzf8ufSh9I=
+github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f/go.mod h1:xH/i4TFMt8koVQZ6WFms69WAsDWr2XsYL3Hkl7jkoLE=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
@@ -30,6 +34,8 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
 github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM=
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
+github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
@@ -56,6 +62,8 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU=
+github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mwitkow/grpc-proxy v0.0.0-20230212185441-f345521cb9c9 h1:62uLwA3l2JMH84liO4ZhnjTH5PjFyCYxbHLgXPaJMtI=
 github.com/mwitkow/grpc-proxy v0.0.0-20230212185441-f345521cb9c9/go.mod h1:MvMXoufZAtqExNexqi4cjrNYE9MefKddKylxjS+//n0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -71,6 +79,8 @@ github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0leargg
 github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
 github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
 github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
+github.com/rs/cors v1.11.1 h1:eU3gRzXLRK57F5rKMGMZURNdIG4EoAmX8k94r9wXWHA=
+github.com/rs/cors v1.11.1/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM=
 github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y=
@@ -81,6 +91,8 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/traefik/grpc-web v0.16.0 h1:eeUWZaFg6ZU0I9dWOYE2D5qkNzRBmXzzuRlxdltascY=
+github.com/traefik/grpc-web v0.16.0/go.mod h1:2ttniSv7pTgBWIU2HZLokxRfFX3SA60c/DTmQQgVml4=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@@ -180,3 +192,5 @@ gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las=
+nhooyr.io/websocket v1.8.17 h1:KEVeLJkUywCKVsnLIDlD/5gtayKp8VoCkksHCGGfT9Y=
+nhooyr.io/websocket v1.8.17/go.mod h1:rN9OFWIUwuxg4fR5tELlYC04bXYowCP9GX47ivo2l+c=

internal/cmd/start.go

@@ -76,6 +76,7 @@ func startCmd() *cobra.Command {
 				HedgeAfter:        cfg.Policies.Hedging.HedgeAfter,
 			})
 			grpcDirector := cosmos_grpc.NewDirector(selCore, cmgr, grpcPool)
+			grpcDirector.SetFailoverPolicy(cfg.Policies.Failover.MaxAttempts, cfg.Policies.Failover.PerAttemptTimeout)
 
 			log.L().Info("stitch starting",
 				"version", version,
@@ -110,6 +111,7 @@ func startCmd() *cobra.Command {
 
 			if cfg.Listen.RPC.Enabled() {
 				cmtSrv := cmt_rpc.New(cfg.Listen.RPC.Addr, fwd)
+				cmtSrv.SetWebSocketSelector(selCore)
 				cmtSrv.SetHashCache(hashIdx)
 				if cfg.Policies.Cache.Enabled {
 					cmtSrv.SetResponseCache(respCache, headFn, cfg.Policies.Cache.ConfirmationDepth, cfg.Policies.Cache.TTL)
@@ -125,6 +127,10 @@ func startCmd() *cobra.Command {
 					return fmt.Errorf("cosmos_grpc: %w", err)
 				}
 				mgr.Add(gs)
+				if cfg.Listen.GRPCWeb.Enabled() {
+					web := gs.WebHandler(cosmos_grpc.WebOptions{AllowedOrigins: cfg.Listen.GRPCWeb.AllowedOrigins}, cosmos_rest.New("", fwd).Handler())
+					mgr.Add(server.NewHTTP("cosmos_grpc_web", cfg.Listen.GRPCWeb.Addr, web))
+				}
 			}
 			if cfg.Listen.EthRPC.Enabled() {
 				ethSrv := eth_rpc.New(cfg.Listen.EthRPC.Addr, fwd)

internal/config/grpc_web_test.go

@@ -0,0 +1,22 @@
+package config
+
+import "testing"
+
+func TestGRPCWebRequiresNativeListener(t *testing.T) {
+	c := diffFixture()
+	applyDefaults(c)
+	c.Listen.GRPCWeb = GRPCWebConfig{Addr: ":5004", AllowedOrigins: []string{"https://app.example.com"}}
+	if Validate(c) == nil {
+		t.Fatal("web listener without native server accepted")
+	}
+	c.Listen.GRPC = AddrConfig{Addr: ":5002"}
+	if err := Validate(c); err != nil {
+		t.Fatal(err)
+	}
+	next := *c
+	next.Listen.GRPCWeb.AllowedOrigins = []string{"https://other.example.com"}
+	diff := DiffNonReloadable(c, &next)
+	if len(diff) != 1 || diff[0] != "listen" {
+		t.Fatalf("origin changes must require restart: %v", diff)
+	}
+}

internal/config/types.go

@@ -15,16 +15,26 @@ type Config struct {
 // ListenConfig groups the addresses for every protocol listener.
 // An empty Addr means the listener is disabled.
 type ListenConfig struct {
-	RPC         AddrConfig `yaml:"rpc"`
-	GRPC        AddrConfig `yaml:"grpc"`
-	API         AddrConfig `yaml:"api"`
-	EthRPC      AddrConfig `yaml:"eth_rpc"`
-	EthWS       AddrConfig `yaml:"eth_ws"`
-	ChainStream AddrConfig `yaml:"chainstream"`
-	InjWS       AddrConfig `yaml:"inj_ws"`
-	Admin       AddrConfig `yaml:"admin"`
+	RPC         AddrConfig    `yaml:"rpc"`
+	GRPC        AddrConfig    `yaml:"grpc"`
+	GRPCWeb     GRPCWebConfig `yaml:"grpc_web"`
+	API         AddrConfig    `yaml:"api"`
+	EthRPC      AddrConfig    `yaml:"eth_rpc"`
+	EthWS       AddrConfig    `yaml:"eth_ws"`
+	ChainStream AddrConfig    `yaml:"chainstream"`
+	InjWS       AddrConfig    `yaml:"inj_ws"`
+	Admin       AddrConfig    `yaml:"admin"`
 }
 
+// GRPCWebConfig enables an optional browser-compatible listener with REST fallback.
+// Origin access is explicit; an empty list rejects cross-origin browser calls.
+type GRPCWebConfig struct {
+	Addr           string   `yaml:"addr"`
+	AllowedOrigins []string `yaml:"allowed_origins,omitempty"`
+}
+
+func (g GRPCWebConfig) Enabled() bool { return g.Addr != "" }
+
 // AddrConfig is a listen address with optional TLS. An empty Addr disables
 // the listener it belongs to.
 type AddrConfig struct {

internal/config/validate.go

@@ -10,6 +10,9 @@ func Validate(c *Config) error {
 	if c == nil {
 		return errors.New("nil config")
 	}
+	if c.Listen.GRPCWeb.Enabled() && !c.Listen.GRPC.Enabled() {
+		return errors.New("listen.grpc_web requires listen.grpc")
+	}
 	if err := validateLog(c.Log); err != nil {
 		return err
 	}

internal/forwarder/history.go

@@ -0,0 +1,84 @@
+package forwarder
+
+import (
+	"bytes"
+	"encoding/json"
+	"io"
+	"net/http"
+
+	"github.com/InjectiveLabs/stitch/internal/history"
+	"github.com/InjectiveLabs/stitch/internal/types"
+)
+
+const maxHistoricalErrorBytes = 64 * 1024
+
+type retainedResponse struct {
+	status int
+	header http.Header
+	body   []byte
+}
+
+type replayBody struct {
+	io.Reader
+	io.Closer
+}
+
+type failedRead struct{ err error }
+
+func (r failedRead) Read([]byte) (int, error) { return 0, r.err }
+
+// historicalError peeks only at bounded, structured Cosmos/Comet errors. It
+// restores every byte for the normal relay, including over-limit responses.
+func historicalError(resp *http.Response, key types.RouteKey) *retainedResponse {
+	if !key.Idempotent || key.Class != types.ClassByHeight || key.HeightOrZero() <= 0 {
+		return nil
+	}
+	if key.Protocol != types.ProtoAPI && key.Protocol != types.ProtoRPC {
+		return nil
+	}
+	if key.Protocol == types.ProtoAPI && resp.StatusCode < 400 {
+		return nil
+	}
+	original := resp.Body
+	body, err := io.ReadAll(io.LimitReader(original, maxHistoricalErrorBytes+1))
+	resp.Body = &replayBody{Reader: io.MultiReader(bytes.NewReader(body), original), Closer: original}
+	if err != nil {
+		resp.Body = &replayBody{Reader: io.MultiReader(bytes.NewReader(body), failedRead{err}), Closer: original}
+	}
+	if err != nil || len(body) > maxHistoricalErrorBytes {
+		return nil
+	}
+	var envelope struct {
+		Code    json.RawMessage `json:"code"`
+		Message string          `json:"message"`
+		Error   *struct {
+			Message string `json:"message"`
+			Data    string `json:"data"`
+		} `json:"error"`
+		Result struct {
+			Response struct {
+				Code json.RawMessage `json:"code"`
+				Log  string          `json:"log"`
+			} `json:"response"`
+		} `json:"result"`
+	}
+	if json.Unmarshal(body, &envelope) != nil {
+		return nil
+	}
+	missing := false
+	if key.Protocol == types.ProtoAPI {
+		missing = nonzeroCode(envelope.Code) && history.Unavailable(envelope.Message)
+	} else if envelope.Error != nil {
+		missing = history.Unavailable(envelope.Error.Message + " " + envelope.Error.Data)
+	} else if key.Method == "abci_query" {
+		missing = nonzeroCode(envelope.Result.Response.Code) && history.Unavailable(envelope.Result.Response.Log)
+	}
+	if !missing {
+		return nil
+	}
+	return &retainedResponse{status: resp.StatusCode, header: resp.Header.Clone(), body: body}
+}
+
+func nonzeroCode(raw json.RawMessage) bool {
+	return len(raw) > 0 && string(raw) != "0" && string(raw) != `"0"` && string(raw) != "null"
+}