From a8ea993857ab9639c8e6793ab434689080ad1d19 Mon Sep 17 00:00:00 2001 From: Ri-go Date: Fri, 18 Sep 2026 19:40:28 +0800 Subject: [PATCH] Fix archival protocol compatibility and historical state fallback --- .github/workflows/ci.yml | 3 + .gitignore | 1 + Dockerfile | 2 +- docs/archival-compatibility.md | 68 ++ go.mod | 5 + go.sum | 14 + internal/cmd/start.go | 6 + internal/config/grpc_web_test.go | 22 + internal/config/types.go | 26 +- internal/config/validate.go | 3 + internal/forwarder/history.go | 84 +++ internal/forwarder/history_test.go | 119 ++++ internal/forwarder/http.go | 19 + internal/history/errors.go | 14 + internal/history/errors_test.go | 20 + internal/server/cmt_rpc/cache_test.go | 86 +++ internal/server/cmt_rpc/decode.go | 2 +- internal/server/cmt_rpc/decode_test.go | 11 + internal/server/cmt_rpc/manifest.go | 2 + internal/server/cmt_rpc/server.go | 77 ++- internal/server/cmt_rpc/websocket.go | 354 ++++++++++ internal/server/cmt_rpc/websocket_test.go | 431 ++++++++++++ .../server/cosmos_grpc/compat_review_test.go | 151 +++++ internal/server/cosmos_grpc/director.go | 21 +- internal/server/cosmos_grpc/handler.go | 7 + internal/server/cosmos_grpc/history.go | 174 +++++ .../server/cosmos_grpc/history_methods.go | 442 ++++++++++++ .../history_methods_review_test.go | 57 ++ internal/server/cosmos_grpc/history_test.go | 222 ++++++ internal/server/cosmos_grpc/server.go | 3 +- internal/server/cosmos_grpc/web.go | 134 ++++ internal/server/cosmos_grpc/web_test.go | 635 ++++++++++++++++++ internal/server/http.go | 31 + tools/generate-history-methods.py | 109 +++ tools/test_generate_history_methods.py | 51 ++ 35 files changed, 3367 insertions(+), 39 deletions(-) create mode 100644 docs/archival-compatibility.md create mode 100644 internal/config/grpc_web_test.go create mode 100644 internal/forwarder/history.go create mode 100644 internal/forwarder/history_test.go create mode 100644 internal/history/errors.go create mode 100644 internal/history/errors_test.go create mode 100644 internal/server/cmt_rpc/cache_test.go create mode 100644 internal/server/cmt_rpc/websocket.go create mode 100644 internal/server/cmt_rpc/websocket_test.go create mode 100644 internal/server/cosmos_grpc/compat_review_test.go create mode 100644 internal/server/cosmos_grpc/history.go create mode 100644 internal/server/cosmos_grpc/history_methods.go create mode 100644 internal/server/cosmos_grpc/history_methods_review_test.go create mode 100644 internal/server/cosmos_grpc/history_test.go create mode 100644 internal/server/cosmos_grpc/web.go create mode 100644 internal/server/cosmos_grpc/web_test.go create mode 100644 internal/server/http.go create mode 100644 tools/generate-history-methods.py create mode 100644 tools/test_generate_history_methods.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 08f9d25..f61c324 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,6 +44,9 @@ jobs: - name: test run: go test -mod=readonly -timeout=5m ./... + - name: historical method generator tests + run: python3 -m unittest discover -s tools -p test_generate_history_methods.py + race: name: race detector runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index e0307b7..6f99f43 100644 --- a/.gitignore +++ b/.gitignore @@ -63,3 +63,4 @@ docker-compose.override.yaml *.test.bin *.tmp testdata.local/ +__pycache__/ diff --git a/Dockerfile b/Dockerfile index aa9de75..c98dff8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,6 +17,6 @@ RUN go build -mod=readonly -trimpath \ FROM gcr.io/distroless/base-debian12:nonroot COPY --from=build /out/stitch /usr/local/bin/stitch USER nonroot:nonroot -EXPOSE 5001 5002 5003 5005 5006 5007 5008 9091 +EXPOSE 5001 5002 5003 5004 5005 5006 5007 5008 9091 ENTRYPOINT ["/usr/local/bin/stitch"] CMD ["start", "--config", "/etc/stitch/config.yaml"] diff --git a/docs/archival-compatibility.md b/docs/archival-compatibility.md new file mode 100644 index 0000000..f769bf9 --- /dev/null +++ b/docs/archival-compatibility.md @@ -0,0 +1,68 @@ +# Archival protocol support + +The RPC listener accepts CometBFT `/websocket` connections. Ordinary JSON-RPC +requests use the same height routing and historical fallback as HTTP, including +when sent on a socket with an active subscription. `subscribe`, `unsubscribe` +and `unsubscribe_all` use a selected tip backend and preserve request IDs. +The upstream WebSocket address is derived from the backend's RPC URL. + +On upstream loss or a full message queue, Stitch closes the client with code +1013. Clients must reconnect and reconcile missed events; CometBFT subscriptions +do not provide replay. Ping/pong, disconnect cancellation and graceful shutdown +are supported. Each message is limited to 32 MiB, with queues bounded by both +message count and bytes. Subscription traffic is not cached. + +## Browser and native gRPC on one hostname + +The optional `grpc_web` listener shares the native gRPC server and its routing: + +```yaml +listen: + grpc: { addr: "0.0.0.0:5002" } + grpc_web: + addr: "0.0.0.0:5004" + allowed_origins: ["https://app.example.com"] +``` + +An explicit `"*"` allows any browser origin; an empty list denies cross-origin +requests. The listener supports binary/text gRPC-Web, response status/trailers, +unary and server-streaming calls, and ordinary HTTP fallback to Cosmos REST. +Native `application/grpc` and `application/grpc+proto` requests use HTTP/2 h2c. +Native gRPC over HTTP/1 is rejected. The separate native port is unchanged. +The nonstandard gRPC-Web WebSocket transport is not enabled. + +The listener requires `listen.grpc`. Configuration changes require a restart. +Native messages remain capped at 64 MiB; encoded web request bodies are capped +at 90 MiB. The ingress must allow configured CORS preflights while retaining +authorization on actual requests. Listener CORS does not replace ingress auth. + +## Missing historical state + +For an explicitly historical, idempotent request, known Cosmos store-retention +errors try another eligible backend within `policies.failover.max_attempts`. +The requested height, payload and metadata are preserved. Backend coverage is +not expanded and a historical query never silently becomes a latest query. + +REST errors and CometBFT ABCI errors use a bounded structured-error inspection. +Other responses pass through unchanged. If all candidates lack the requested +state, the final upstream error is returned. Missing state does not count as a +shared circuit failure, and failed ABCI responses are not cached. + +gRPC retries are limited to schema-verified unary read methods. Unknown methods, +broadcasts and streaming methods retain transparent forwarding. A response is +never retried after any message has been delivered. Headers and trailers from +failed attempts do not leak into a successful response; the final failure's +metadata is retained when all candidates fail. Transport/deadline failures still +affect backend health; unrelated application errors are returned unchanged. + +The exact method inventory is generated from pinned Injective, Cosmos SDK, IBC +and CosmWasm schemas. Run `python3 tools/generate-history-methods.py` with an +authenticated `gh` CLI to regenerate it; source revisions and schema links are +recorded in `internal/server/cosmos_grpc/history_methods.go`. + +## Asia migration + +The archival Stitch release handles Cosmos/CometBFT chain traffic. EVM traffic +uses the existing Asia EVM Gateway, configured from height 127250000, without +archival IP/key authorization or rate tiers. Exchange endpoints remain on nginx. +Public routes and DNS are separate rollout steps. diff --git a/go.mod b/go.mod index 96091f1..76f0179 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,9 @@ require ( github.com/gorilla/websocket v1.5.3 github.com/mwitkow/grpc-proxy v0.0.0-20230212185441-f345521cb9c9 github.com/prometheus/client_golang v1.20.5 + github.com/rs/cors v1.11.1 github.com/spf13/cobra v1.8.1 + github.com/traefik/grpc-web v0.16.0 golang.org/x/sync v0.10.0 google.golang.org/grpc v1.66.0 google.golang.org/protobuf v1.36.11 @@ -16,7 +18,9 @@ require ( require ( github.com/beorn7/perks v1.0.1 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/klauspost/compress v1.17.9 // indirect github.com/kr/text v0.2.0 // indirect @@ -30,4 +34,5 @@ require ( golang.org/x/sys v0.22.0 // indirect golang.org/x/text v0.16.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260504160031-60b97b32f348 // indirect + nhooyr.io/websocket v1.8.17 // indirect ) diff --git a/go.sum b/go.sum index 5dc101b..fe758f7 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -12,6 +14,8 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f h1:U5y3Y5UE0w7amNe7Z5G/twsBW0KEalRQXZzf8ufSh9I= +github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f/go.mod h1:xH/i4TFMt8koVQZ6WFms69WAsDWr2XsYL3Hkl7jkoLE= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= @@ -30,6 +34,8 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -56,6 +62,8 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/grpc-proxy v0.0.0-20230212185441-f345521cb9c9 h1:62uLwA3l2JMH84liO4ZhnjTH5PjFyCYxbHLgXPaJMtI= github.com/mwitkow/grpc-proxy v0.0.0-20230212185441-f345521cb9c9/go.mod h1:MvMXoufZAtqExNexqi4cjrNYE9MefKddKylxjS+//n0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -71,6 +79,8 @@ github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0leargg github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/rs/cors v1.11.1 h1:eU3gRzXLRK57F5rKMGMZURNdIG4EoAmX8k94r9wXWHA= +github.com/rs/cors v1.11.1/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= @@ -81,6 +91,8 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/traefik/grpc-web v0.16.0 h1:eeUWZaFg6ZU0I9dWOYE2D5qkNzRBmXzzuRlxdltascY= +github.com/traefik/grpc-web v0.16.0/go.mod h1:2ttniSv7pTgBWIU2HZLokxRfFX3SA60c/DTmQQgVml4= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -180,3 +192,5 @@ gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= +nhooyr.io/websocket v1.8.17 h1:KEVeLJkUywCKVsnLIDlD/5gtayKp8VoCkksHCGGfT9Y= +nhooyr.io/websocket v1.8.17/go.mod h1:rN9OFWIUwuxg4fR5tELlYC04bXYowCP9GX47ivo2l+c= diff --git a/internal/cmd/start.go b/internal/cmd/start.go index 0fcdea9..ffc200c 100644 --- a/internal/cmd/start.go +++ b/internal/cmd/start.go @@ -76,6 +76,7 @@ func startCmd() *cobra.Command { HedgeAfter: cfg.Policies.Hedging.HedgeAfter, }) grpcDirector := cosmos_grpc.NewDirector(selCore, cmgr, grpcPool) + grpcDirector.SetFailoverPolicy(cfg.Policies.Failover.MaxAttempts, cfg.Policies.Failover.PerAttemptTimeout) log.L().Info("stitch starting", "version", version, @@ -110,6 +111,7 @@ func startCmd() *cobra.Command { if cfg.Listen.RPC.Enabled() { cmtSrv := cmt_rpc.New(cfg.Listen.RPC.Addr, fwd) + cmtSrv.SetWebSocketSelector(selCore) cmtSrv.SetHashCache(hashIdx) if cfg.Policies.Cache.Enabled { cmtSrv.SetResponseCache(respCache, headFn, cfg.Policies.Cache.ConfirmationDepth, cfg.Policies.Cache.TTL) @@ -125,6 +127,10 @@ func startCmd() *cobra.Command { return fmt.Errorf("cosmos_grpc: %w", err) } mgr.Add(gs) + if cfg.Listen.GRPCWeb.Enabled() { + web := gs.WebHandler(cosmos_grpc.WebOptions{AllowedOrigins: cfg.Listen.GRPCWeb.AllowedOrigins}, cosmos_rest.New("", fwd).Handler()) + mgr.Add(server.NewHTTP("cosmos_grpc_web", cfg.Listen.GRPCWeb.Addr, web)) + } } if cfg.Listen.EthRPC.Enabled() { ethSrv := eth_rpc.New(cfg.Listen.EthRPC.Addr, fwd) diff --git a/internal/config/grpc_web_test.go b/internal/config/grpc_web_test.go new file mode 100644 index 0000000..8319c83 --- /dev/null +++ b/internal/config/grpc_web_test.go @@ -0,0 +1,22 @@ +package config + +import "testing" + +func TestGRPCWebRequiresNativeListener(t *testing.T) { + c := diffFixture() + applyDefaults(c) + c.Listen.GRPCWeb = GRPCWebConfig{Addr: ":5004", AllowedOrigins: []string{"https://app.example.com"}} + if Validate(c) == nil { + t.Fatal("web listener without native server accepted") + } + c.Listen.GRPC = AddrConfig{Addr: ":5002"} + if err := Validate(c); err != nil { + t.Fatal(err) + } + next := *c + next.Listen.GRPCWeb.AllowedOrigins = []string{"https://other.example.com"} + diff := DiffNonReloadable(c, &next) + if len(diff) != 1 || diff[0] != "listen" { + t.Fatalf("origin changes must require restart: %v", diff) + } +} diff --git a/internal/config/types.go b/internal/config/types.go index 15d4af5..22bbf87 100644 --- a/internal/config/types.go +++ b/internal/config/types.go @@ -15,16 +15,26 @@ type Config struct { // ListenConfig groups the addresses for every protocol listener. // An empty Addr means the listener is disabled. type ListenConfig struct { - RPC AddrConfig `yaml:"rpc"` - GRPC AddrConfig `yaml:"grpc"` - API AddrConfig `yaml:"api"` - EthRPC AddrConfig `yaml:"eth_rpc"` - EthWS AddrConfig `yaml:"eth_ws"` - ChainStream AddrConfig `yaml:"chainstream"` - InjWS AddrConfig `yaml:"inj_ws"` - Admin AddrConfig `yaml:"admin"` + RPC AddrConfig `yaml:"rpc"` + GRPC AddrConfig `yaml:"grpc"` + GRPCWeb GRPCWebConfig `yaml:"grpc_web"` + API AddrConfig `yaml:"api"` + EthRPC AddrConfig `yaml:"eth_rpc"` + EthWS AddrConfig `yaml:"eth_ws"` + ChainStream AddrConfig `yaml:"chainstream"` + InjWS AddrConfig `yaml:"inj_ws"` + Admin AddrConfig `yaml:"admin"` } +// GRPCWebConfig enables an optional browser-compatible listener with REST fallback. +// Origin access is explicit; an empty list rejects cross-origin browser calls. +type GRPCWebConfig struct { + Addr string `yaml:"addr"` + AllowedOrigins []string `yaml:"allowed_origins,omitempty"` +} + +func (g GRPCWebConfig) Enabled() bool { return g.Addr != "" } + // AddrConfig is a listen address with optional TLS. An empty Addr disables // the listener it belongs to. type AddrConfig struct { diff --git a/internal/config/validate.go b/internal/config/validate.go index 9caaddf..5507725 100644 --- a/internal/config/validate.go +++ b/internal/config/validate.go @@ -10,6 +10,9 @@ func Validate(c *Config) error { if c == nil { return errors.New("nil config") } + if c.Listen.GRPCWeb.Enabled() && !c.Listen.GRPC.Enabled() { + return errors.New("listen.grpc_web requires listen.grpc") + } if err := validateLog(c.Log); err != nil { return err } diff --git a/internal/forwarder/history.go b/internal/forwarder/history.go new file mode 100644 index 0000000..fd79f99 --- /dev/null +++ b/internal/forwarder/history.go @@ -0,0 +1,84 @@ +package forwarder + +import ( + "bytes" + "encoding/json" + "io" + "net/http" + + "github.com/InjectiveLabs/stitch/internal/history" + "github.com/InjectiveLabs/stitch/internal/types" +) + +const maxHistoricalErrorBytes = 64 * 1024 + +type retainedResponse struct { + status int + header http.Header + body []byte +} + +type replayBody struct { + io.Reader + io.Closer +} + +type failedRead struct{ err error } + +func (r failedRead) Read([]byte) (int, error) { return 0, r.err } + +// historicalError peeks only at bounded, structured Cosmos/Comet errors. It +// restores every byte for the normal relay, including over-limit responses. +func historicalError(resp *http.Response, key types.RouteKey) *retainedResponse { + if !key.Idempotent || key.Class != types.ClassByHeight || key.HeightOrZero() <= 0 { + return nil + } + if key.Protocol != types.ProtoAPI && key.Protocol != types.ProtoRPC { + return nil + } + if key.Protocol == types.ProtoAPI && resp.StatusCode < 400 { + return nil + } + original := resp.Body + body, err := io.ReadAll(io.LimitReader(original, maxHistoricalErrorBytes+1)) + resp.Body = &replayBody{Reader: io.MultiReader(bytes.NewReader(body), original), Closer: original} + if err != nil { + resp.Body = &replayBody{Reader: io.MultiReader(bytes.NewReader(body), failedRead{err}), Closer: original} + } + if err != nil || len(body) > maxHistoricalErrorBytes { + return nil + } + var envelope struct { + Code json.RawMessage `json:"code"` + Message string `json:"message"` + Error *struct { + Message string `json:"message"` + Data string `json:"data"` + } `json:"error"` + Result struct { + Response struct { + Code json.RawMessage `json:"code"` + Log string `json:"log"` + } `json:"response"` + } `json:"result"` + } + if json.Unmarshal(body, &envelope) != nil { + return nil + } + missing := false + if key.Protocol == types.ProtoAPI { + missing = nonzeroCode(envelope.Code) && history.Unavailable(envelope.Message) + } else if envelope.Error != nil { + missing = history.Unavailable(envelope.Error.Message + " " + envelope.Error.Data) + } else if key.Method == "abci_query" { + missing = nonzeroCode(envelope.Result.Response.Code) && history.Unavailable(envelope.Result.Response.Log) + } + if !missing { + return nil + } + return &retainedResponse{status: resp.StatusCode, header: resp.Header.Clone(), body: body} +} + +func nonzeroCode(raw json.RawMessage) bool { + return len(raw) > 0 && string(raw) != "0" && string(raw) != `"0"` && string(raw) != "null" +} diff --git a/internal/forwarder/history_test.go b/internal/forwarder/history_test.go new file mode 100644 index 0000000..6363cb9 --- /dev/null +++ b/internal/forwarder/history_test.go @@ -0,0 +1,119 @@ +package forwarder + +import ( + "bytes" + "errors" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/InjectiveLabs/stitch/internal/backend" + "github.com/InjectiveLabs/stitch/internal/circuit" + "github.com/InjectiveLabs/stitch/internal/types" +) + +const missingState = `{"code":2,"message":"failed to load state at height 105504992; version mismatch on immutable IAVL tree; version does not exist"}` + +func TestHistoricalHTTPFallback(t *testing.T) { + for _, tc := range []struct { + name string + protocol types.Protocol + method string + status int + body string + class types.MethodClass + idempotent bool + wantRetry bool + }{ + {"rest", types.ProtoAPI, "/cosmos/bank/v1beta1/params", 500, missingState, types.ClassByHeight, true, true}, + {"abci", types.ProtoRPC, "abci_query", 200, `{"jsonrpc":"2.0","id":7,"result":{"response":{"code":18,"log":"failed to load state at height 105504992; version does not exist"}}}`, types.ClassByHeight, true, true}, + {"rpc_error", types.ProtoRPC, "abci_query", 200, `{"jsonrpc":"2.0","id":7,"error":{"message":"internal error","data":"failed to load state at height 105504992; version does not exist"}}`, types.ClassByHeight, true, true}, + {"unrelated_error", types.ProtoAPI, "query", 500, `{"code":2,"message":"permission denied"}`, types.ClassByHeight, true, false}, + {"latest", types.ProtoAPI, "query", 500, missingState, types.ClassLatest, true, false}, + {"write", types.ProtoAPI, "write", 500, missingState, types.ClassByHeight, false, false}, + {"evm", types.ProtoEthRPC, "eth_call", 500, missingState, types.ClassByHeight, true, false}, + {"unstructured", types.ProtoAPI, "query", 500, "failed to load state; version does not exist", types.ClassByHeight, true, false}, + {"oversize", types.ProtoAPI, "query", 500, missingState + strings.Repeat(" ", maxHistoricalErrorBytes), types.ClassByHeight, true, false}, + } { + t.Run(tc.name, func(t *testing.T) { + bad := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(tc.status) + _, _ = io.WriteString(w, tc.body) + })) + defer bad.Close() + var hits atomic.Int32 + good := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits.Add(1) + if r.Header.Get("x-cosmos-block-height") != "105504992" { + t.Error("height header lost") + } + body, _ := io.ReadAll(r.Body) + if string(body) != "request payload" { + t.Errorf("request changed: %q", body) + } + w.Header().Set("x-cosmos-block-height", "105504992") + _, _ = io.WriteString(w, `{"ok":true}`) + })) + defer good.Close() + bs := []*backend.Backend{mkBackend("missing", bad.URL), mkBackend("overlap", good.URL)} + for _, b := range bs { + b.Endpoints[tc.protocol] = b.Endpoints[types.ProtoRPC] + } + cm := circuit.NewManager(circuit.Policy{MinRequests: 1, ErrorThreshold: .5, OpenDuration: time.Minute}) + f := newForwarderWithCircuit(stubSelector{bs}, cm, 3) + h := int64(105504992) + req := httptest.NewRequest(http.MethodGet, "/query", strings.NewReader("request payload")) + req.Header.Set("x-cosmos-block-height", "105504992") + rec := httptest.NewRecorder() + f.Forward(rec, req, types.RouteKey{Protocol: tc.protocol, Method: tc.method, Class: tc.class, Height: &h, Idempotent: tc.idempotent}) + if (hits.Load() == 1) != tc.wantRetry { + t.Fatalf("retry hits=%d wantRetry=%v", hits.Load(), tc.wantRetry) + } + if tc.wantRetry { + if rec.Code != 200 || rec.Body.String() != `{"ok":true}` { + t.Fatalf("response %d %s", rec.Code, rec.Body) + } + if cm.State("missing", tc.protocol) != circuit.StateClosed { + t.Error("retention gap tripped circuit") + } + } else if rec.Code != tc.status || rec.Body.String() != tc.body { + t.Fatal("non-retryable response changed") + } + }) + } +} + +func TestHistoricalHTTPExhaustionPreservesError(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("X-Upstream", "retained") + w.WriteHeader(500) + _, _ = io.WriteString(w, missingState) + })) + defer upstream.Close() + b := mkBackend("only", upstream.URL) + b.Endpoints[types.ProtoAPI] = upstream.URL + f := newForwarder(stubSelector{[]*backend.Backend{b}}) + h := int64(105504992) + rec := httptest.NewRecorder() + f.Forward(rec, httptest.NewRequest("GET", "/query", nil), types.RouteKey{Protocol: types.ProtoAPI, Method: "query", Class: types.ClassByHeight, Height: &h, Idempotent: true}) + if rec.Code != 500 || rec.Body.String() != missingState || rec.Header().Get("X-Upstream") != "retained" { + t.Fatalf("changed error: %d %s", rec.Code, rec.Body) + } +} + +func TestHistoricalPeekPreservesReadFailure(t *testing.T) { + want := errors.New("upstream truncated") + resp := &http.Response{StatusCode: 500, Body: io.NopCloser(io.MultiReader(bytes.NewBufferString("partial"), failedRead{want}))} + h := int64(1) + if historicalError(resp, types.RouteKey{Protocol: types.ProtoAPI, Class: types.ClassByHeight, Height: &h, Idempotent: true}) != nil { + t.Fatal("partial body classified") + } + body, err := io.ReadAll(resp.Body) + if string(body) != "partial" || !errors.Is(err, want) { + t.Fatalf("body=%q error=%v", body, err) + } +} diff --git a/internal/forwarder/http.go b/internal/forwarder/http.go index 49a6af4..589a244 100644 --- a/internal/forwarder/http.go +++ b/internal/forwarder/http.go @@ -92,6 +92,7 @@ func (f *HTTP) Forward(w http.ResponseWriter, r *http.Request, key types.RouteKe } var lastErr error + var lastHistorical *retainedResponse attempts := 0 for _, b := range candidates { // Client gone: nothing we write will be read. Stop attempting and @@ -134,6 +135,7 @@ func (f *HTTP) Forward(w http.ResponseWriter, r *http.Request, key types.RouteKe if err != nil { cancel() lastErr = err + lastHistorical = nil f.circuit.Record(b.Name, key.Protocol, false) continue } @@ -169,7 +171,17 @@ func (f *HTTP) Forward(w http.ResponseWriter, r *http.Request, key types.RouteKe continue } + if missing := historicalError(resp, key); missing != nil { + lastHistorical = missing + lastErr = fmt.Errorf("historical state unavailable on %s", b.Name) + _ = resp.Body.Close() + cancel() + f.circuit.Release(b.Name, key.Protocol) + metrics.FailoverAttempts.WithLabelValues(b.Name, "next", "missing_state").Inc() + continue + } if shouldRetryStatus(resp.StatusCode, key) { + lastHistorical = nil lastErr = fmt.Errorf("upstream status %d", resp.StatusCode) _ = resp.Body.Close() cancel() @@ -226,6 +238,13 @@ func (f *HTTP) Forward(w http.ResponseWriter, r *http.Request, key types.RouteKe return } + if lastHistorical != nil { + copyHeaders(w.Header(), lastHistorical.header) + w.WriteHeader(lastHistorical.status) + _, _ = w.Write(lastHistorical.body) + metrics.RequestsTotal.WithLabelValues(string(key.Protocol), key.Class.String(), "-", "history_unavailable").Inc() + return + } if lastErr == nil { lastErr = ErrNoCandidates } diff --git a/internal/history/errors.go b/internal/history/errors.go new file mode 100644 index 0000000..5c05275 --- /dev/null +++ b/internal/history/errors.go @@ -0,0 +1,14 @@ +// Package history identifies backend retention failures, not generic query errors. +package history + +import "strings" + +// Unavailable reports known Cosmos store-retention errors. Callers must also +// require an idempotent, explicitly historical request before trying another +// backend. A missing version says nothing about that backend's overall health. +func Unavailable(message string) bool { + m := strings.ToLower(message) + return (strings.Contains(m, "version does not exist") && + (strings.Contains(m, "iavl") || strings.Contains(m, "failed to load state") || strings.Contains(m, "failed to load version"))) || + (strings.Contains(m, "no commit info found") && strings.Contains(m, "version")) +} diff --git a/internal/history/errors_test.go b/internal/history/errors_test.go new file mode 100644 index 0000000..46723e5 --- /dev/null +++ b/internal/history/errors_test.go @@ -0,0 +1,20 @@ +package history + +import "testing" + +func TestUnavailable(t *testing.T) { + for _, s := range []string{ + "failed to load state at height 105504992; version mismatch on immutable IAVL tree; version does not exist", + "no commit info found for version 112637000", + "failed to load version 20: version does not exist", + } { + if !Unavailable(s) { + t.Errorf("unrecognized retention failure: %s", s) + } + } + for _, s := range []string{"account does not exist", "permission denied", "invalid request", "version does not exist", "iavl query successful"} { + if Unavailable(s) { + t.Errorf("ordinary error classified as retention failure: %s", s) + } + } +} diff --git a/internal/server/cmt_rpc/cache_test.go b/internal/server/cmt_rpc/cache_test.go new file mode 100644 index 0000000..5cfc26f --- /dev/null +++ b/internal/server/cmt_rpc/cache_test.go @@ -0,0 +1,86 @@ +package cmt_rpc + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/InjectiveLabs/stitch/internal/backend" + "github.com/InjectiveLabs/stitch/internal/cache" + "github.com/InjectiveLabs/stitch/internal/circuit" + "github.com/InjectiveLabs/stitch/internal/forwarder" + "github.com/InjectiveLabs/stitch/internal/pool" + "github.com/InjectiveLabs/stitch/internal/types" +) + +type fixedCMTSelector []*backend.Backend + +func (s fixedCMTSelector) Candidates(types.RouteKey) []*backend.Backend { return s } + +func TestABCIErrorsAreNotCached(t *testing.T) { + for _, code := range []string{`18`, `"18"`, `"invalid"`} { + t.Run(code, func(t *testing.T) { + var calls atomic.Int64 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var req jsonRPCRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Error(err) + return + } + applicationCode := code + if calls.Add(1) > 1 { + applicationCode = "0" + } + w.Header().Set("Content-Type", "application/json") + _, _ = fmt.Fprintf(w, `{"jsonrpc":"2.0","id":%s,"result":{"response":{"code":%s,"value":"retained-state"}}}`, req.ID, applicationCode) + })) + defer upstream.Close() + sel := fixedCMTSelector{{Name: "archive", Endpoints: map[types.Protocol]string{types.ProtoRPC: upstream.URL}}} + cm := circuit.NewManager(circuit.Policy{MinRequests: 10}) + fwd := forwarder.NewHTTP(sel, pool.NewHTTPPool(), cm, forwarder.Policy{MaxAttempts: 1}) + srv := New("ignored", fwd) + srv.SetResponseCache(cache.NewResponseCache(cache.ResponseCacheOpts{Capacity: 10}), func() int64 { return 200 }, 0, time.Minute) + for i := 1; i <= 3; i++ { + request := fmt.Sprintf(`{"jsonrpc":"2.0","id":%d,"method":"abci_query","params":{"path":"/store/bank/key","height":"75"}}`, i) + r := httptest.NewRequest(http.MethodPost, "/", bytes.NewBufferString(request)) + w := httptest.NewRecorder() + srv.ServeHTTP(w, r) + wantCache := "miss" + if i == 3 { + wantCache = "hit" + } + if got := w.Header().Get("x-stitch-cache"); got != wantCache { + t.Fatalf("call %d cache = %q, want %q; %s", i, got, wantCache, w.Body.String()) + } + var response map[string]json.RawMessage + if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil { + t.Fatal(err) + } + assertCMTID(t, response, fmt.Sprint(i)) + } + if calls.Load() != 2 { + t.Fatalf("upstream calls = %d, want 2", calls.Load()) + } + }) + } +} + +func TestABCICacheSuccessCodes(t *testing.T) { + for _, fields := range []string{`"code":0`, `"code":"0"`, `"value":"empty-code"`} { + body := []byte(`{"jsonrpc":"2.0","id":1,"result":{"response":{` + fields + `}}}`) + if !cmtCacheableResponse("abci_query", body) { + t.Fatalf("successful response rejected: %s", body) + } + } + for _, result := range []string{`{}`, `{"response":null}`, `{"response":{"code":null}}`, `{"response":{"code":-1}}`} { + body := []byte(`{"jsonrpc":"2.0","id":1,"result":` + result + `}`) + if cmtCacheableResponse("abci_query", body) { + t.Fatalf("invalid ABCI response cached: %s", body) + } + } +} diff --git a/internal/server/cmt_rpc/decode.go b/internal/server/cmt_rpc/decode.go index 83b1352..18d2080 100644 --- a/internal/server/cmt_rpc/decode.go +++ b/internal/server/cmt_rpc/decode.go @@ -134,7 +134,7 @@ func decodeJSONRPC(r *http.Request) (decoded, error) { }, } if spec.HeightParam != "" { - if hs := paramFromJSON(req.Params, spec.HeightParam, 0); hs != "" { + if hs := paramFromJSON(req.Params, spec.HeightParam, spec.HeightIndex); hs != "" { if h, ok := parseHeight(hs); ok { d.key.Height = &h d.key.Class = types.ClassByHeight diff --git a/internal/server/cmt_rpc/decode_test.go b/internal/server/cmt_rpc/decode_test.go index 0f670ad..e859480 100644 --- a/internal/server/cmt_rpc/decode_test.go +++ b/internal/server/cmt_rpc/decode_test.go @@ -259,3 +259,14 @@ func assertRange(t *testing.T, d decoded, lower, upper int64) { t.Fatalf("upper: %v", d.key.Range.Upper) } } + +func TestDecodeJSONRPCABCIHeightArray(t *testing.T) { + r := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(`{"jsonrpc":"2.0","id":1,"method":"abci_query","params":["/store/bank/key","ABCD","75",false]}`)) + d, err := decode(r) + if err != nil { + t.Fatal(err) + } + if d.key.Class != types.ClassByHeight || d.key.HeightOrZero() != 75 { + t.Fatalf("ABCI positional height was not routed historically: %+v", d.key) + } +} diff --git a/internal/server/cmt_rpc/manifest.go b/internal/server/cmt_rpc/manifest.go index 08f9f72..9d8c79a 100644 --- a/internal/server/cmt_rpc/manifest.go +++ b/internal/server/cmt_rpc/manifest.go @@ -8,6 +8,7 @@ type MethodSpec struct { Name string Class types.MethodClass HeightParam string // param name for height (uri+json-rpc) + HeightIndex int // position of height in JSON-RPC array params HashParam string // param name for hash HeightOptional bool // if true, treat absent height as latest Idempotent bool @@ -45,6 +46,7 @@ var Manifest = func() map[string]MethodSpec { add(MethodSpec{ Name: "abci_query", HeightParam: "height", + HeightIndex: 2, // path, data, height, prove HeightOptional: true, Idempotent: true, Cacheable: true, diff --git a/internal/server/cmt_rpc/server.go b/internal/server/cmt_rpc/server.go index fd628ca..7281573 100644 --- a/internal/server/cmt_rpc/server.go +++ b/internal/server/cmt_rpc/server.go @@ -1,5 +1,4 @@ -// Package cmt_rpc is the CometBFT RPC listener (URI + JSON-RPC over HTTP). -// WebSocket support arrives in phase 5 with the subscription hub. +// Package cmt_rpc is the CometBFT RPC listener (HTTP and WebSocket). package cmt_rpc import ( @@ -9,26 +8,30 @@ import ( "errors" "io" "net/http" + "strconv" "time" "github.com/InjectiveLabs/stitch/internal/cache" "github.com/InjectiveLabs/stitch/internal/forwarder" "github.com/InjectiveLabs/stitch/internal/log" "github.com/InjectiveLabs/stitch/internal/runtime" + "github.com/InjectiveLabs/stitch/internal/selector" "github.com/InjectiveLabs/stitch/internal/server" "github.com/InjectiveLabs/stitch/internal/types" ) // Server is the CometBFT RPC listener. type Server struct { - addr string - fwd *forwarder.HTTP - cache *cache.HashIndex - respCache *cache.ResponseCache - head cache.HeadProvider - confDepth int64 - cacheTTL time.Duration - srv *http.Server + addr string + fwd *forwarder.HTTP + cache *cache.HashIndex + respCache *cache.ResponseCache + head cache.HeadProvider + confDepth int64 + cacheTTL time.Duration + srv *http.Server + wsSelector selector.Selector + wsTracker *server.ConnTracker } // SetHashCache attaches a shared hash→height index for memoization on @@ -45,10 +48,10 @@ func (s *Server) SetResponseCache(c *cache.ResponseCache, head cache.HeadProvide } func New(addr string, fwd *forwarder.HTTP) *Server { - s := &Server{addr: addr, fwd: fwd} + s := &Server{addr: addr, fwd: fwd, wsTracker: server.NewConnTracker()} mux := http.NewServeMux() mux.Handle("/", s) - mux.HandleFunc("/websocket", websocketStub) + mux.HandleFunc("/websocket", s.serveWebSocket) s.srv = &http.Server{ Addr: addr, Handler: mux, @@ -67,7 +70,13 @@ func (s *Server) Start(_ context.Context) error { return nil } -func (s *Server) Shutdown(ctx context.Context) error { return s.srv.Shutdown(ctx) } +func (s *Server) Shutdown(ctx context.Context) error { + err := s.srv.Shutdown(ctx) + if wsErr := s.wsTracker.SweepAndWait(ctx); err == nil { + err = wsErr + } + return err +} // Handler returns the http.Handler the listener uses. Exported for tests // (httptest.NewServer needs a handler, not a *http.Server). @@ -131,10 +140,8 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { cacheKey := cache.BuildKey(protocol, d.key.Method, height, cache.HashParams(params)) if hit, ok := s.respCache.Get(cacheKey); ok { response := hit - var valid bool - if d.uri { - valid = cache.IsSuccessfulResponse(hit) - } else { + valid := cmtCacheableResponse(d.key.Method, hit) + if !d.uri && valid { response, valid = cache.ResponseWithID(hit, d.id) } if valid { @@ -149,7 +156,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { cap.Header().Set("x-stitch-cache", "miss") dispatch(cap, r, d.key) cap.FlushTo(w) - if cap.Status() >= 200 && cap.Status() < 300 && cache.IsSuccessfulResponse(cap.BodyBytes()) { + if cap.Status() >= 200 && cap.Status() < 300 && cmtCacheableResponse(d.key.Method, cap.BodyBytes()) { s.respCache.Set(cacheKey, cap.BodyBytes(), s.cacheTTL) } if s.cache != nil && cmtPopulatable(d.key.Method) { @@ -169,6 +176,34 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { dispatch(w, r, d.key) } +// ABCI application errors are nested inside an otherwise successful JSON-RPC +// envelope. In particular, a missing historical version must not be cached +// after all candidate shards fail: retained state can become available later. +func cmtCacheableResponse(method string, body []byte) bool { + if !cache.IsSuccessfulResponse(body) { + return false + } + if method != "abci_query" { + return true + } + var envelope struct { + Result struct { + Response *struct { + Code json.RawMessage `json:"code"` + } `json:"response"` + } `json:"result"` + } + if json.Unmarshal(body, &envelope) != nil || envelope.Result.Response == nil { + return false + } + code := bytes.TrimSpace(envelope.Result.Response.Code) + if len(code) == 0 { + return true // CometBFT may omit the zero code. + } + value, err := strconv.ParseUint(unquoteRaw(code), 10, 32) + return err == nil && value == 0 +} + func (s *Server) applyHashCache(d *decoded) { if s.cache == nil || d.key.Class != types.ClassByHash || len(d.key.Hash) == 0 { return @@ -201,12 +236,6 @@ func cmtPopulatable(method string) bool { return false } -func websocketStub(w http.ResponseWriter, _ *http.Request) { - w.Header().Set("content-type", "application/json") - w.WriteHeader(http.StatusNotImplemented) - _, _ = w.Write([]byte(`{"error":"/websocket arrives in phase 5 (subscription hub)"}`)) -} - func writeJSONRPCError(w http.ResponseWriter, status int, msg string) { type rpcErr struct { Code int `json:"code"` diff --git a/internal/server/cmt_rpc/websocket.go b/internal/server/cmt_rpc/websocket.go new file mode 100644 index 0000000..b077dc4 --- /dev/null +++ b/internal/server/cmt_rpc/websocket.go @@ -0,0 +1,354 @@ +package cmt_rpc + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "net/http" + "net/url" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/gorilla/websocket" + + "github.com/InjectiveLabs/stitch/internal/cache" + "github.com/InjectiveLabs/stitch/internal/selector" + "github.com/InjectiveLabs/stitch/internal/types" + "github.com/InjectiveLabs/stitch/internal/wsurl" +) + +const ( + wsReadLimit = 32 << 20 + wsQueueSize = 16 + wsQueueBytes = 2 * wsReadLimit + wsWriteWait = 10 * time.Second + wsReadWait = 60 * time.Second + wsPingEvery = 20 * time.Second +) + +// SetWebSocketSelector enables subscription routing. Call before Start. +// Ordinary WebSocket calls use the HTTP forwarder, including its height +// selection, retries and cache. Subscription methods share a dedicated tip +// connection per client. CometBFT subscriptions cannot replay missed events, +// so upstream loss closes the client with 1013 instead of hiding gaps during +// a reconnect. Clients must reconnect and reconcile missed events themselves. +func (s *Server) SetWebSocketSelector(sel selector.Selector) { s.wsSelector = sel } + +func (s *Server) serveWebSocket(w http.ResponseWriter, r *http.Request) { + upgrader := websocket.Upgrader{ + ReadBufferSize: 4096, WriteBufferSize: 4096, + // Match CometBFT: authentication and origin policy belong at ingress. + CheckOrigin: func(*http.Request) bool { return true }, + } + client, err := upgrader.Upgrade(w, r, nil) + if err != nil { + return + } + if !s.wsTracker.Track(client) { + _ = client.Close() + return + } + defer s.wsTracker.Untrack(client) + + ctx, cancel := context.WithCancel(r.Context()) + session := &cmtWSSession{ + server: s, client: client, ctx: ctx, cancel: cancel, + incoming: make(chan []byte, wsQueueSize), outgoing: make(chan []byte, wsQueueSize), + } + session.run() +} + +type cmtWSSession struct { + server *Server + client *websocket.Conn + ctx context.Context + cancel context.CancelFunc + incoming chan []byte + outgoing chan []byte + workers sync.WaitGroup + incomingBytes atomic.Int64 + outgoingBytes atomic.Int64 + // upstream is only read or changed by run/handleRequest. Its reader and + // keepalive goroutines receive their own immutable connection pointer. + upstream *websocket.Conn +} + +func (s *cmtWSSession) run() { + defer func() { + s.cancel() + _ = s.client.Close() + if s.upstream != nil { + _ = s.upstream.Close() + } + s.workers.Wait() + }() + s.workers.Add(2) + go s.readClient() + go s.writeClient() + for { + select { + case <-s.ctx.Done(): + return + case message := <-s.incoming: + s.incomingBytes.Add(-int64(len(message))) + s.handleRequest(message) + } + } +} + +func configureWSReader(conn *websocket.Conn) { + conn.SetReadLimit(wsReadLimit) + _ = conn.SetReadDeadline(time.Now().Add(wsReadWait)) + conn.SetPongHandler(func(string) error { + return conn.SetReadDeadline(time.Now().Add(wsReadWait)) + }) + conn.SetPingHandler(func(data string) error { + _ = conn.SetReadDeadline(time.Now().Add(wsReadWait)) + return conn.WriteControl(websocket.PongMessage, []byte(data), time.Now().Add(wsWriteWait)) + }) +} + +func (s *cmtWSSession) readClient() { + defer s.workers.Done() + defer s.cancel() + configureWSReader(s.client) + for { + kind, message, err := s.client.ReadMessage() + if err != nil { + return + } + _ = s.client.SetReadDeadline(time.Now().Add(wsReadWait)) + if kind != websocket.TextMessage { + s.closeClient(websocket.CloseUnsupportedData, "JSON-RPC requires text messages") + return + } + if s.incomingBytes.Add(int64(len(message))) > wsQueueBytes { + s.incomingBytes.Add(-int64(len(message))) + s.closeClient(websocket.CloseTryAgainLater, "request queue full") + return + } + select { + case s.incoming <- message: + case <-s.ctx.Done(): + s.incomingBytes.Add(-int64(len(message))) + return + default: + s.incomingBytes.Add(-int64(len(message))) + s.closeClient(websocket.CloseTryAgainLater, "request queue full") + return + } + } +} + +func (s *cmtWSSession) writeClient() { + defer s.workers.Done() + defer s.cancel() + ticker := time.NewTicker(wsPingEvery) + defer ticker.Stop() + for { + select { + case <-s.ctx.Done(): + return + case message := <-s.outgoing: + s.outgoingBytes.Add(-int64(len(message))) + _ = s.client.SetWriteDeadline(time.Now().Add(wsWriteWait)) + if err := s.client.WriteMessage(websocket.TextMessage, message); err != nil { + return + } + case <-ticker.C: + if err := s.client.WriteControl(websocket.PingMessage, nil, time.Now().Add(wsWriteWait)); err != nil { + return + } + } + } +} + +func (s *cmtWSSession) closeClient(code int, reason string) { + _ = s.client.WriteControl(websocket.CloseMessage, websocket.FormatCloseMessage(code, reason), time.Now().Add(wsWriteWait)) + s.cancel() + _ = s.client.Close() +} + +func (s *cmtWSSession) send(message []byte) { + if s.outgoingBytes.Add(int64(len(message))) > wsQueueBytes { + s.outgoingBytes.Add(-int64(len(message))) + s.closeClient(websocket.CloseTryAgainLater, "response queue full") + return + } + select { + case <-s.ctx.Done(): + s.outgoingBytes.Add(-int64(len(message))) + case s.outgoing <- message: + default: + s.outgoingBytes.Add(-int64(len(message))) + // Do not drop subscription events and pretend the stream is intact. + s.closeClient(websocket.CloseTryAgainLater, "response queue full") + } +} + +func (s *cmtWSSession) sendError(id json.RawMessage, code int, message string) { + if len(id) == 0 { + id = json.RawMessage("null") + } + out, _ := json.Marshal(struct { + JSONRPC string `json:"jsonrpc"` + ID json.RawMessage `json:"id"` + Error struct { + Code int `json:"code"` + Message string `json:"message"` + } `json:"error"` + }{JSONRPC: "2.0", ID: id, Error: struct { + Code int `json:"code"` + Message string `json:"message"` + }{code, message}}) + s.send(out) +} + +func (s *cmtWSSession) handleRequest(message []byte) { + var request jsonRPCRequest + if err := json.Unmarshal(message, &request); err != nil { + // CometBFT's WebSocket transport accepts one request per frame, not + // batches. Reject arrays rather than accidentally routing to latest. + s.sendError(nil, -32700, "parse error: expected a JSON-RPC request object") + return + } + if request.JSONRPC != "2.0" || request.Method == "" || + (len(request.ID) > 0 && !cache.IsJSONRPCID(request.ID)) { + s.sendError(nil, -32600, "invalid request") + return + } + if len(request.ID) == 0 || bytes.Equal(bytes.TrimSpace(request.ID), []byte("null")) { + // Match CometBFT: notifications are ignored, including subscribe. + return + } + + switch request.Method { + case "subscribe", "unsubscribe", "unsubscribe_all": + if s.upstream == nil { + conn, err := s.dialUpstream() + if err != nil { + s.sendError(request.ID, -32000, "no available subscription backend") + return + } + s.upstream = conn + s.workers.Add(2) + go s.readUpstream(conn) + go s.pingUpstream(conn) + } + _ = s.upstream.SetWriteDeadline(time.Now().Add(wsWriteWait)) + if err := s.upstream.WriteMessage(websocket.TextMessage, message); err != nil { + s.closeClient(websocket.CloseTryAgainLater, "subscription backend disconnected; reconnect required") + } + default: + r, err := http.NewRequestWithContext(s.ctx, http.MethodPost, "http://stitch/", bytes.NewReader(message)) + if err != nil { + s.sendError(request.ID, -32603, "internal error") + return + } + r.Header.Set("Content-Type", "application/json") + response := &wsResponse{header: make(http.Header)} + s.server.ServeHTTP(response, r) + if s.ctx.Err() != nil { + return + } + var envelope struct { + JSONRPC string `json:"jsonrpc"` + Result json.RawMessage `json:"result"` + Error json.RawMessage `json:"error"` + } + body := response.body.Bytes() + if response.overflow || json.Unmarshal(body, &envelope) != nil || envelope.JSONRPC != "2.0" || + (len(envelope.Result) == 0 && (len(envelope.Error) == 0 || envelope.Error[0] != '{')) { + s.sendError(request.ID, -32000, "upstream request failed") + return + } + s.send(body) + } +} + +func (s *cmtWSSession) dialUpstream() (*websocket.Conn, error) { + if s.server.wsSelector == nil { + return nil, errors.New("subscription selector not configured") + } + candidates := s.server.wsSelector.Candidates(types.RouteKey{ + Protocol: types.ProtoRPC, Method: "subscribe", Class: types.ClassLatest, + }) + dialer := websocket.Dialer{HandshakeTimeout: 5 * time.Second} + for _, candidate := range candidates { + endpoint, err := url.Parse(wsurl.Normalize(candidate.Endpoint(types.ProtoRPC))) + if err != nil || endpoint.Host == "" || (endpoint.Scheme != "ws" && endpoint.Scheme != "wss") { + continue + } + endpoint.Path = strings.TrimRight(endpoint.Path, "/") + "/websocket" + endpoint.RawPath = "" + conn, response, err := dialer.DialContext(s.ctx, endpoint.String(), nil) + if response != nil && response.Body != nil { + _ = response.Body.Close() + } + if err == nil { + return conn, nil + } + if s.ctx.Err() != nil { + return nil, s.ctx.Err() + } + } + return nil, errors.New("no available subscription backend") +} + +func (s *cmtWSSession) readUpstream(conn *websocket.Conn) { + defer s.workers.Done() + configureWSReader(conn) + for { + kind, message, err := conn.ReadMessage() + if err != nil { + if s.ctx.Err() == nil { + s.closeClient(websocket.CloseTryAgainLater, "subscription backend disconnected; reconnect required") + } + return + } + _ = conn.SetReadDeadline(time.Now().Add(wsReadWait)) + if kind != websocket.TextMessage || !json.Valid(message) { + s.closeClient(websocket.CloseInternalServerErr, "invalid subscription response") + return + } + s.send(message) + } +} + +func (s *cmtWSSession) pingUpstream(conn *websocket.Conn) { + defer s.workers.Done() + ticker := time.NewTicker(wsPingEvery) + defer ticker.Stop() + for { + select { + case <-s.ctx.Done(): + return + case <-ticker.C: + if err := conn.WriteControl(websocket.PingMessage, nil, time.Now().Add(wsWriteWait)); err != nil { + s.closeClient(websocket.CloseTryAgainLater, "subscription backend disconnected; reconnect required") + return + } + } + } +} + +// Bound the final WebSocket response buffer. The HTTP forwarder has its own +// upstream response handling; writes here never retain more than readLimit. +type wsResponse struct { + header http.Header + body bytes.Buffer + overflow bool +} + +func (w *wsResponse) Header() http.Header { return w.header } +func (w *wsResponse) WriteHeader(int) {} +func (w *wsResponse) Write(b []byte) (int, error) { + if len(b) > wsReadLimit-w.body.Len() { + w.overflow = true + return 0, errors.New("WebSocket response too large") + } + return w.body.Write(b) +} diff --git a/internal/server/cmt_rpc/websocket_test.go b/internal/server/cmt_rpc/websocket_test.go new file mode 100644 index 0000000..5ad05f4 --- /dev/null +++ b/internal/server/cmt_rpc/websocket_test.go @@ -0,0 +1,431 @@ +package cmt_rpc + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/gorilla/websocket" + + "github.com/InjectiveLabs/stitch/internal/backend" + "github.com/InjectiveLabs/stitch/internal/circuit" + "github.com/InjectiveLabs/stitch/internal/forwarder" + "github.com/InjectiveLabs/stitch/internal/health" + "github.com/InjectiveLabs/stitch/internal/pool" + "github.com/InjectiveLabs/stitch/internal/selector" + "github.com/InjectiveLabs/stitch/internal/types" +) + +type cmtWSMock struct { + server *httptest.Server + requests chan jsonRPCRequest + connections chan *websocket.Conn + closed chan struct{} + pongs chan string + httpStarted chan struct{} + httpCanceled chan struct{} + httpCalls atomic.Int64 + wsCalls atomic.Int64 + rejectWS bool +} + +func newCMTWSMock(t *testing.T, name string, reject bool) *cmtWSMock { + t.Helper() + m := &cmtWSMock{ + requests: make(chan jsonRPCRequest, 32), connections: make(chan *websocket.Conn, 16), + closed: make(chan struct{}, 16), rejectWS: reject, + pongs: make(chan string, 16), httpStarted: make(chan struct{}, 1), httpCanceled: make(chan struct{}, 1), + } + m.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/websocket" { + m.httpCalls.Add(1) + var req jsonRPCRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Errorf("decode upstream HTTP call: %v", err) + return + } + if req.Method == "slow_request" { + m.httpStarted <- struct{}{} + <-r.Context().Done() + m.httpCanceled <- struct{}{} + return + } + if req.Method == "upstream_http_error" { + http.Error(w, "unavailable", http.StatusServiceUnavailable) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = fmt.Fprintf(w, `{"jsonrpc":"2.0","id":%s,"result":{"backend":%q,"params":%s}}`, req.ID, name, req.Params) + return + } + m.wsCalls.Add(1) + if m.rejectWS { + http.Error(w, "unavailable", http.StatusServiceUnavailable) + return + } + conn, err := (&websocket.Upgrader{}).Upgrade(w, r, nil) + if err != nil { + return + } + defer conn.Close() + defer func() { m.closed <- struct{}{} }() + conn.SetPongHandler(func(payload string) error { m.pongs <- payload; return nil }) + m.connections <- conn + for { + _, msg, err := conn.ReadMessage() + if err != nil { + return + } + var req jsonRPCRequest + if err := json.Unmarshal(msg, &req); err != nil { + t.Errorf("invalid upstream subscription request: %v", err) + return + } + m.requests <- req + response := []byte(fmt.Sprintf(`{"jsonrpc":"2.0","id":%s,"result":{}}`, req.ID)) + if bytes.Contains(req.Params, []byte("reject")) { + response = []byte(fmt.Sprintf(`{"jsonrpc":"2.0","id":%s,"error":{"code":-32603,"message":"bad query"}}`, req.ID)) + } + if err := conn.WriteMessage(websocket.TextMessage, response); err != nil { + return + } + if req.Method == "subscribe" && !bytes.Contains(req.Params, []byte("reject")) { + event := []byte(fmt.Sprintf(`{"jsonrpc":"2.0","id":%s,"result":{"query":"tm.event='NewBlock'","data":{"type":"tendermint/event/NewBlock","value":{"block":{"header":{"height":"201"}}}}}}`, req.ID)) + if err := conn.WriteMessage(websocket.TextMessage, event); err != nil { + return + } + } + } + })) + t.Cleanup(m.server.Close) + return m +} + +type cmtWSRig struct { + server *Server + front *httptest.Server + history *cmtWSMock + tip *cmtWSMock +} + +func newCMTWSRig(t *testing.T, rejectTip bool, extra ...*backend.Backend) *cmtWSRig { + t.Helper() + history := newCMTWSMock(t, "history", true) + tip := newCMTWSMock(t, "tip", rejectTip) + backends := append([]*backend.Backend{ + {Name: "history", Weight: 100, Coverage: backend.Coverage{Kind: backend.CovBounded, Lower: 1, Upper: 100}, Endpoints: map[types.Protocol]string{types.ProtoRPC: history.server.URL}}, + {Name: "tip", Weight: 200, Coverage: backend.Coverage{Kind: backend.CovPruned, Keep: 10}, Endpoints: map[types.Protocol]string{types.ProtoRPC: tip.server.URL}}, + }, extra...) + h := health.NewRegistry() + for _, b := range backends { + h.Update(health.Snapshot{Backend: b.Name, Protocol: types.ProtoRPC, Healthy: true, LatestHeight: 200}) + } + cm := circuit.NewManager(circuit.Policy{ErrorThreshold: 0.5, MinRequests: 10, OpenDuration: time.Second}) + sel := selector.NewRangeSelector(backend.NewRegistry(backends), h, cm, 0) + fwd := forwarder.NewHTTP(sel, pool.NewHTTPPool(), cm, forwarder.Policy{MaxAttempts: 3, PerAttemptTimeout: 10 * time.Second}) + s := New("ignored", fwd) + s.SetWebSocketSelector(sel) + front := httptest.NewServer(s.Handler()) + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + if err := s.Shutdown(ctx); err != nil { + t.Errorf("shutdown: %v", err) + } + front.Close() + }) + return &cmtWSRig{server: s, front: front, history: history, tip: tip} +} + +func dialCMTWS(t *testing.T, base string) *websocket.Conn { + t.Helper() + conn, _, err := websocket.DefaultDialer.Dial("ws"+strings.TrimPrefix(base, "http")+"/websocket", nil) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = conn.Close() }) + return conn +} + +func writeCMTWS(t *testing.T, conn *websocket.Conn, msg string) { + t.Helper() + if err := conn.WriteMessage(websocket.TextMessage, []byte(msg)); err != nil { + t.Fatal(err) + } +} + +func readCMTWS(t *testing.T, conn *websocket.Conn) map[string]json.RawMessage { + t.Helper() + _ = conn.SetReadDeadline(time.Now().Add(2 * time.Second)) + _, msg, err := conn.ReadMessage() + if err != nil { + t.Fatal(err) + } + var out map[string]json.RawMessage + if err := json.Unmarshal(msg, &out); err != nil { + t.Fatalf("invalid JSON response %s: %v", msg, err) + } + return out +} + +func assertCMTID(t *testing.T, msg map[string]json.RawMessage, want string) { + t.Helper() + if string(msg["id"]) != want { + t.Fatalf("id = %s, want %s", msg["id"], want) + } +} + +func waitCMTClosed(t *testing.T, done <-chan struct{}) { + t.Helper() + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("upstream connection did not close") + } +} + +func TestCMTWebSocketSubscriptionsAndHistoricalReads(t *testing.T) { + for _, id := range []string{`"blocks"`, `9007199254740993`} { + t.Run(id, func(t *testing.T) { + rig := newCMTWSRig(t, false) + conn := dialCMTWS(t, rig.front.URL) + writeCMTWS(t, conn, fmt.Sprintf(`{"jsonrpc":"2.0","id":%s,"method":"subscribe","params":{"query":"tm.event='NewBlock'"}}`, id)) + ack := readCMTWS(t, conn) + assertCMTID(t, ack, id) + event := readCMTWS(t, conn) + assertCMTID(t, event, id) + if !bytes.Contains(event["result"], []byte(`"height":"201"`)) { + t.Fatalf("event lost: %s", event["result"]) + } + for _, request := range []string{ + `{"jsonrpc":"2.0","id":"archive","method":"block","params":{"height":"75"}}`, + `{"jsonrpc":"2.0","id":"archive","method":"block","params":["75"]}`, + `{"jsonrpc":"2.0","id":"archive","method":"abci_query","params":["/store/bank/key","ABCD","75",false]}`, + } { + writeCMTWS(t, conn, request) + response := readCMTWS(t, conn) + assertCMTID(t, response, `"archive"`) + if !bytes.Contains(response["result"], []byte(`"backend":"history"`)) { + t.Fatalf("historical read went to tip: %s", response["result"]) + } + } + for _, method := range []string{"unsubscribe", "unsubscribe_all"} { + writeCMTWS(t, conn, fmt.Sprintf(`{"jsonrpc":"2.0","id":2,"method":%q,"params":{"query":"tm.event='NewBlock'"}}`, method)) + assertCMTID(t, readCMTWS(t, conn), "2") + } + for _, method := range []string{"subscribe", "unsubscribe", "unsubscribe_all"} { + select { + case req := <-rig.tip.requests: + if req.Method != method || !bytes.Contains(req.Params, []byte("tm.event='NewBlock'")) { + t.Fatalf("request changed: %+v", req) + } + case <-time.After(time.Second): + t.Fatal("missing subscription request") + } + } + if rig.tip.httpCalls.Load() != 0 || rig.history.httpCalls.Load() != 3 || rig.tip.wsCalls.Load() != 1 || rig.history.wsCalls.Load() != 0 { + t.Fatal("unexpected upstream routing") + } + _ = conn.Close() + waitCMTClosed(t, rig.tip.closed) + }) + } +} + +func TestCMTWebSocketBadRequestsAndNotifications(t *testing.T) { + rig := newCMTWSRig(t, false) + conn := dialCMTWS(t, rig.front.URL) + for _, message := range []string{`[`, `[]`, `null`, `{"jsonrpc":"2.0","method":"block","id":{}}`} { + writeCMTWS(t, conn, message) + response := readCMTWS(t, conn) + assertCMTID(t, response, "null") + if response["error"] == nil { + t.Fatal("invalid request accepted") + } + } + for _, idField := range []string{"", `,"id":null`} { + writeCMTWS(t, conn, `{"jsonrpc":"2.0","method":"subscribe","params":{"query":"ignored"}`+idField+`}`) + } + writeCMTWS(t, conn, `{"jsonrpc":"2.0","id":3,"method":"block","params":{"height":"75"}}`) + assertCMTID(t, readCMTWS(t, conn), "3") + if rig.tip.wsCalls.Load() != 0 || rig.tip.httpCalls.Load() != 0 { + t.Fatal("invalid frames or notifications reached tip") + } +} + +func TestCMTWebSocketUpstreamErrors(t *testing.T) { + t.Run("subscribe reject is returned unchanged", func(t *testing.T) { + rig := newCMTWSRig(t, false) + conn := dialCMTWS(t, rig.front.URL) + writeCMTWS(t, conn, `{"jsonrpc":"2.0","id":"bad-query","method":"subscribe","params":{"query":"reject"}}`) + response := readCMTWS(t, conn) + assertCMTID(t, response, `"bad-query"`) + if !bytes.Contains(response["error"], []byte("bad query")) { + t.Fatal("upstream error lost") + } + }) + t.Run("unavailable subscription still permits historical reads", func(t *testing.T) { + rig := newCMTWSRig(t, true) + conn := dialCMTWS(t, rig.front.URL) + writeCMTWS(t, conn, `{"jsonrpc":"2.0","id":42,"method":"subscribe","params":{"query":"tm.event='NewBlock'"}}`) + response := readCMTWS(t, conn) + assertCMTID(t, response, "42") + if response["error"] == nil { + t.Fatal("missing unavailable error") + } + writeCMTWS(t, conn, `{"jsonrpc":"2.0","id":43,"method":"block","params":{"height":"75"}}`) + assertCMTID(t, readCMTWS(t, conn), "43") + }) + t.Run("HTTP errors retain request ID", func(t *testing.T) { + rig := newCMTWSRig(t, false) + conn := dialCMTWS(t, rig.front.URL) + writeCMTWS(t, conn, `{"jsonrpc":"2.0","id":"failure","method":"upstream_http_error","params":{}}`) + response := readCMTWS(t, conn) + assertCMTID(t, response, `"failure"`) + if response["error"] == nil { + t.Fatal("HTTP error did not become JSON-RPC error") + } + }) +} + +func TestCMTWebSocketInitialDialFallback(t *testing.T) { + fallback := newCMTWSMock(t, "fallback", false) + rig := newCMTWSRig(t, true, &backend.Backend{ + Name: "fallback", Weight: 100, Coverage: backend.Coverage{Kind: backend.CovPruned, Keep: 10}, + Endpoints: map[types.Protocol]string{types.ProtoRPC: fallback.server.URL}, + }) + conn := dialCMTWS(t, rig.front.URL) + writeCMTWS(t, conn, `{"jsonrpc":"2.0","id":1,"method":"subscribe","params":{"query":"tm.event='NewBlock'"}}`) + assertCMTID(t, readCMTWS(t, conn), "1") + assertCMTID(t, readCMTWS(t, conn), "1") + if rig.tip.wsCalls.Load() != 1 || fallback.wsCalls.Load() != 1 { + t.Fatal("did not try the next subscription backend") + } +} + +func TestCMTWebSocketUpstreamLossRequiresReconnect(t *testing.T) { + rig := newCMTWSRig(t, false) + conn := dialCMTWS(t, rig.front.URL) + writeCMTWS(t, conn, `{"jsonrpc":"2.0","id":1,"method":"subscribe","params":{"query":"tm.event='NewBlock'"}}`) + readCMTWS(t, conn) + readCMTWS(t, conn) + upstream := <-rig.tip.connections + _ = upstream.Close() + _ = conn.SetReadDeadline(time.Now().Add(2 * time.Second)) + _, _, err := conn.ReadMessage() + if !websocket.IsCloseError(err, websocket.CloseTryAgainLater) { + t.Fatalf("expected reconnect close, got %v", err) + } +} + +func TestCMTWebSocketShutdownClosesBothEnds(t *testing.T) { + rig := newCMTWSRig(t, false) + conn := dialCMTWS(t, rig.front.URL) + writeCMTWS(t, conn, `{"jsonrpc":"2.0","id":1,"method":"subscribe","params":{"query":"tm.event='NewBlock'"}}`) + readCMTWS(t, conn) + readCMTWS(t, conn) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if err := rig.server.Shutdown(ctx); err != nil { + t.Fatal(err) + } + waitCMTClosed(t, rig.tip.closed) + _ = conn.SetReadDeadline(time.Now().Add(time.Second)) + if _, _, err := conn.ReadMessage(); err == nil { + t.Fatal("client remains open after shutdown") + } +} + +func TestCMTWebSocketSlowConsumerDisconnects(t *testing.T) { + serverConn := make(chan *websocket.Conn, 1) + front := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := (&websocket.Upgrader{}).Upgrade(w, r, nil) + if err == nil { + serverConn <- conn + } + })) + defer front.Close() + client := dialCMTWS(t, front.URL) + conn := <-serverConn + defer conn.Close() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + session := cmtWSSession{ctx: ctx, cancel: cancel, client: conn, outgoing: make(chan []byte, 1)} + session.send([]byte(`{"result":1}`)) + session.send([]byte(`{"result":2}`)) + _ = client.SetReadDeadline(time.Now().Add(time.Second)) + if _, _, err := client.ReadMessage(); !websocket.IsCloseError(err, websocket.CloseTryAgainLater) { + t.Fatalf("expected explicit slow-consumer close, got %v", err) + } + if ctx.Err() == nil { + t.Fatal("slow session was not cancelled") + } +} + +func TestCMTWebSocketControlFramesAndRequestCancellation(t *testing.T) { + rig := newCMTWSRig(t, false) + conn := dialCMTWS(t, rig.front.URL) + writeCMTWS(t, conn, `{"jsonrpc":"2.0","id":1,"method":"subscribe","params":{"query":"tm.event='NewBlock'"}}`) + readCMTWS(t, conn) + readCMTWS(t, conn) + upstream := <-rig.tip.connections + if err := upstream.WriteControl(websocket.PingMessage, []byte("upstream-ping"), time.Now().Add(time.Second)); err != nil { + t.Fatal(err) + } + select { + case payload := <-rig.tip.pongs: + if payload != "upstream-ping" { + t.Fatalf("upstream pong changed: %q", payload) + } + case <-time.After(time.Second): + t.Fatal("upstream ping unanswered") + } + + writeCMTWS(t, conn, `{"jsonrpc":"2.0","id":2,"method":"slow_request","params":{}}`) + select { + case <-rig.tip.httpStarted: + case <-time.After(time.Second): + t.Fatal("ordinary request did not reach HTTP upstream") + } + pongs := make(chan string, 1) + conn.SetPongHandler(func(payload string) error { pongs <- payload; return nil }) + readerDone := make(chan struct{}) + go func() { + defer close(readerDone) + _, _, _ = conn.ReadMessage() + }() + if err := conn.WriteControl(websocket.PingMessage, []byte("client-ping"), time.Now().Add(time.Second)); err != nil { + t.Fatal(err) + } + select { + case payload := <-pongs: + if payload != "client-ping" { + t.Fatalf("client pong changed: %q", payload) + } + case <-time.After(time.Second): + t.Fatal("client ping blocked behind historical request") + } + _ = conn.Close() + waitCMTClosed(t, readerDone) + waitCMTClosed(t, rig.tip.httpCanceled) + waitCMTClosed(t, rig.tip.closed) +} + +func TestCMTWebSocketRejectsBinaryRequests(t *testing.T) { + rig := newCMTWSRig(t, false) + conn := dialCMTWS(t, rig.front.URL) + if err := conn.WriteMessage(websocket.BinaryMessage, []byte(`{"jsonrpc":"2.0","id":1,"method":"block","params":{}}`)); err != nil { + t.Fatal(err) + } + _ = conn.SetReadDeadline(time.Now().Add(time.Second)) + if _, _, err := conn.ReadMessage(); !websocket.IsCloseError(err, websocket.CloseUnsupportedData) { + t.Fatalf("expected unsupported-data close, got %v", err) + } +} diff --git a/internal/server/cosmos_grpc/compat_review_test.go b/internal/server/cosmos_grpc/compat_review_test.go new file mode 100644 index 0000000..f3ae705 --- /dev/null +++ b/internal/server/cosmos_grpc/compat_review_test.go @@ -0,0 +1,151 @@ +package cosmos_grpc + +import ( + "context" + "testing" + "time" + + "github.com/InjectiveLabs/stitch/internal/circuit" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/emptypb" +) + +// A retryable failure still has an ordinary gRPC response contract when no +// alternate can serve it: preserve its response headers as well as trailers. +func TestReviewHistoricalExhaustionPreservesResponseMetadata(t *testing.T) { + r := setupOutcomeRig(t, circuit.Policy{ErrorThreshold: .5, MinRequests: 1, OpenDuration: time.Minute}, + func(_ string, stream grpc.ServerStream) error { + if err := stream.SendHeader(metadata.Pairs("x-upstream-context", "retained", HeightHeader, "145000000")); err != nil { + return err + } + stream.SetTrailer(metadata.Pairs("x-upstream-detail", "last shard")) + return status.Error(codes.Unknown, "failed to load state; version does not exist") + }) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + ctx = metadata.AppendToOutgoingContext(ctx, HeightHeader, "145000000") + var header, trailer metadata.MD + err := r.conn.Invoke(ctx, historicalMethod, &emptypb.Empty{}, &emptypb.Empty{}, grpc.Header(&header), grpc.Trailer(&trailer)) + r.waitFinished(t, historicalMethod) + if status.Code(err) != codes.Unknown { + t.Fatalf("status changed: %v", err) + } + if got := header.Get("x-upstream-context"); len(got) != 1 || got[0] != "retained" { + t.Errorf("initial response metadata lost: %v", header) + } + if got := header.Get(HeightHeader); len(got) != 1 || got[0] != "145000000" { + t.Errorf("response height lost: %v", header) + } + if got := trailer.Get("x-upstream-detail"); len(got) != 1 || got[0] != "last shard" { + t.Errorf("final trailers lost: %v", trailer) + } +} + +func TestReviewHistoricalRetryOnlyExposesFinalMetadata(t *testing.T) { + for _, succeed := range []bool{false, true} { + name := "exhaustion" + if succeed { + name = "success" + } + t.Run(name, func(t *testing.T) { + conn, _, _ := historicalRig(t, + func(stream grpc.ServerStream, _ *emptypb.Empty) error { + if err := stream.SendHeader(metadata.Pairs("x-first-attempt", "private", "x-backend", "first")); err != nil { + return err + } + stream.SetTrailer(metadata.Pairs("x-first-trailer", "private")) + return status.Error(codes.Unknown, historicalMissing) + }, + func(stream grpc.ServerStream, _ *emptypb.Empty) error { + if err := stream.SendHeader(metadata.Pairs("x-backend", "final")); err != nil { + return err + } + stream.SetTrailer(metadata.Pairs("x-final-trailer", "kept")) + if succeed { + return stream.SendMsg(&emptypb.Empty{}) + } + return status.Error(codes.Unknown, historicalMissing) + }) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + ctx = metadata.AppendToOutgoingContext(ctx, HeightHeader, "105504992") + var header, trailer metadata.MD + err := conn.Invoke(ctx, historicalMethod, &emptypb.Empty{}, &emptypb.Empty{}, grpc.Header(&header), grpc.Trailer(&trailer)) + if succeed && err != nil { + t.Fatal(err) + } + if !succeed && status.Code(err) != codes.Unknown { + t.Fatal(err) + } + if got := header.Get("x-backend"); len(got) != 1 || got[0] != "final" { + t.Errorf("final header lost: %v", header) + } + if len(header.Get("x-first-attempt")) != 0 || len(trailer.Get("x-first-trailer")) != 0 { + t.Errorf("failed-attempt metadata leaked: %v %v", header, trailer) + } + if got := trailer.Get("x-final-trailer"); len(got) != 1 || got[0] != "kept" { + t.Errorf("final trailer lost: %v", trailer) + } + }) + } +} + +func TestReviewHistoricalMissingStateKeepsHalfOpenCircuitNeutral(t *testing.T) { + r := setupOutcomeRig(t, circuit.Policy{ErrorThreshold: .5, MinRequests: 1, OpenDuration: time.Millisecond}, + func(_ string, _ grpc.ServerStream) error { + return status.Error(codes.Unknown, "failed to load state; version does not exist") + }) + r.circuit.Record(outcomeBackend, "grpc", false) + time.Sleep(3 * time.Millisecond) + if err := r.call(t, historicalMethod); status.Code(err) != codes.Unknown { + t.Fatal(err) + } + if got := r.circuit.State(outcomeBackend, "grpc"); got != circuit.StateHalfOpen { + t.Fatalf("retention error changed half-open state: %v", got) + } + if !r.circuit.Acquire(outcomeBackend, "grpc") { + t.Fatal("retention error stranded half-open admission") + } + r.circuit.Release(outcomeBackend, "grpc") +} + +// Unknown services under a familiar namespace may be bidirectional. Merely +// containing ".Query/" is not enough to infer a one-request read contract. +func TestReviewUnknownQueryServiceRetainsBidirectionalForwarding(t *testing.T) { + conn, _, _ := historicalRig(t, func(stream grpc.ServerStream, _ *emptypb.Empty) error { + if err := stream.SendMsg(&emptypb.Empty{}); err != nil { + return err + } + var second emptypb.Empty + if err := stream.RecvMsg(&second); err != nil { + return err + } + return stream.SendMsg(&second) + }) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + ctx = metadata.AppendToOutgoingContext(ctx, HeightHeader, "105504992") + stream, err := conn.NewStream(ctx, &grpc.StreamDesc{ClientStreams: true, ServerStreams: true}, "/cosmos.review.v1.Query/Watch") + if err != nil { + t.Fatal(err) + } + if err := stream.SendMsg(&emptypb.Empty{}); err != nil { + t.Fatal(err) + } + // A streaming peer need not half-close before reading its first response. + if err := stream.RecvMsg(&emptypb.Empty{}); err != nil { + t.Fatalf("first response blocked waiting for request EOF: %v", err) + } + if err := stream.SendMsg(&emptypb.Empty{}); err != nil { + t.Fatal(err) + } + if err := stream.CloseSend(); err != nil { + t.Fatal(err) + } + if err := stream.RecvMsg(&emptypb.Empty{}); err != nil { + t.Fatal(err) + } +} diff --git a/internal/server/cosmos_grpc/director.go b/internal/server/cosmos_grpc/director.go index 7d187aa..fd33f4e 100644 --- a/internal/server/cosmos_grpc/director.go +++ b/internal/server/cosmos_grpc/director.go @@ -5,6 +5,7 @@ import ( "strconv" "strings" "sync" + "time" "google.golang.org/grpc" "google.golang.org/grpc/codes" @@ -81,13 +82,25 @@ func (a *atomicString) Get() string { // retried after partial response delivery; failover only applies before // the first message lands. Phase 5's subscription hub adds proper resume. type Director struct { - selector selector.Selector - circuit *circuit.Manager - pool *pool.GRPCPool + selector selector.Selector + circuit *circuit.Manager + pool *pool.GRPCPool + maxAttempts int + perAttemptTimeout time.Duration } func NewDirector(s selector.Selector, c *circuit.Manager, p *pool.GRPCPool) *Director { - return &Director{selector: s, circuit: c, pool: p} + return &Director{selector: s, circuit: c, pool: p, maxAttempts: 3, perAttemptTimeout: 5 * time.Second} +} + +// SetFailoverPolicy configures historical-query attempts before serving starts. +func (d *Director) SetFailoverPolicy(maxAttempts int, timeout time.Duration) { + if maxAttempts > 0 { + d.maxAttempts = maxAttempts + } + if timeout > 0 { + d.perAttemptTimeout = timeout + } } // Direct chooses an upstream for fullMethodName. Returns the modified diff --git a/internal/server/cosmos_grpc/handler.go b/internal/server/cosmos_grpc/handler.go index 7aad33d..29b3b80 100644 --- a/internal/server/cosmos_grpc/handler.go +++ b/internal/server/cosmos_grpc/handler.go @@ -61,6 +61,13 @@ func streamHandler(dir *Director) grpc.StreamHandler { } } } + if hasMethod { + md, _ := metadata.FromIncomingContext(wrapped.ctx) + key := buildRouteKey(method, md, requestHeight(wrapped.ctx)) + if key.Class == types.ClassByHeight && key.Idempotent && historicalReadMethod(method) { + return dir.forwardHistorical(wrapped, method, key) + } + } err := inner(srv, wrapped) if name := slot.Get(); name != "" { outcome := classifyRPCOutcome(ss.Context(), err) diff --git a/internal/server/cosmos_grpc/history.go b/internal/server/cosmos_grpc/history.go new file mode 100644 index 0000000..a1a9965 --- /dev/null +++ b/internal/server/cosmos_grpc/history.go @@ -0,0 +1,174 @@ +package cosmos_grpc + +import ( + "context" + "errors" + "io" + "strconv" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/emptypb" + + "github.com/InjectiveLabs/stitch/internal/history" + "github.com/InjectiveLabs/stitch/internal/log" + "github.com/InjectiveLabs/stitch/internal/metrics" + "github.com/InjectiveLabs/stitch/internal/pool" + "github.com/InjectiveLabs/stitch/internal/types" +) + +func (d *Director) forwardHistorical(ss *slotStream, method string, key types.RouteKey) (result error) { + started := time.Now() + lastBackend := "" + defer func() { + if lastBackend == "" { + return + } + outcome := classifyRPCOutcome(ss.Context(), result) + log.FromCtx(ss.Context()).Debug("grpc: RPC outcome", "backend", lastBackend, + "protocol", string(types.ProtoGRPC), "method", method, "class", key.Class.String(), + "height", key.HeightOrZero(), "grpc_status", status.Code(result).String(), + "outcome", string(outcome), "duration_ms", time.Since(started).Milliseconds()) + }() + var request, extra emptypb.Empty + if err := ss.RecvMsg(&request); err != nil { + return err + } + if err := ss.RecvMsg(&extra); !errors.Is(err, io.EOF) { + if err != nil { + return err + } + return status.Error(codes.InvalidArgument, "historical query requires one request message") + } + md, _ := metadata.FromIncomingContext(ss.Context()) + md = md.Copy() + if _, hasHeight := metadataHeight(md); !hasHeight { + md.Set(HeightHeader, strconv.FormatInt(key.HeightOrZero(), 10)) + } + md.Set("x-stitch-request-id", log.RequestID(ss.Context())) + var lastErr error + var lastTrailer metadata.MD + var lastHeader metadata.MD + attempts := 0 + for _, b := range d.selector.Candidates(key) { + if err := ss.Context().Err(); err != nil { + return status.FromContextError(err).Err() + } + if attempts >= d.maxAttempts { + break + } + ep := b.Endpoint(types.ProtoGRPC) + if ep == "" || !d.circuit.Acquire(b.Name, types.ProtoGRPC) { + continue + } + attempts++ + lastBackend = b.Name + metrics.RequestsTotal.WithLabelValues(string(types.ProtoGRPC), key.Class.String(), b.Name, "directed").Inc() + attemptStarted := time.Now() + ctx, cancel := context.WithTimeout(ss.Context(), d.perAttemptTimeout) + ctx = metadata.NewOutgoingContext(ctx, md) + conn, err := d.pool.Conn(ctx, b.Name, pool.CleanAddr(ep)) + var upstream grpc.ClientStream + if err == nil { + upstream, err = conn.NewStream(ctx, &grpc.StreamDesc{ServerStreams: true}, method) + } + if err == nil { + err = upstream.SendMsg(&request) + } + if err == nil { + err = upstream.CloseSend() + } + var first emptypb.Empty + if err == nil { + err = upstream.RecvMsg(&first) + } + metrics.BackendLatency.WithLabelValues(b.Name, string(types.ProtoGRPC)).Observe(time.Since(attemptStarted).Seconds()) + if err != nil && !errors.Is(err, io.EOF) { + lastErr = err + lastTrailer = nil + lastHeader = nil + if upstream != nil { + lastTrailer = upstream.Trailer() + lastHeader, _ = upstream.Header() + } + missing := missingStateStatus(err) + outcome := classifyRPCOutcome(ss.Context(), err) + if missing || outcome == rpcNeutral { + d.ReleaseOutcome(b.Name) + } else { + d.RecordOutcome(b.Name, false) + } + cancel() + if ss.Context().Err() != nil { + return status.FromContextError(ss.Context().Err()).Err() + } + if missing || status.Code(err) == codes.Unavailable || status.Code(err) == codes.DeadlineExceeded { + reason := "availability" + if missing { + reason = "missing_state" + } + metrics.FailoverAttempts.WithLabelValues(b.Name, "next", reason).Inc() + continue + } + ss.SetTrailer(lastTrailer) + if len(lastHeader) > 0 { + _ = ss.SendHeader(lastHeader) + } + return err + } + // The first message commits this response. Never replay after this + // point, even if an upstream stream later fails with a retention error. + header, headerErr := upstream.Header() + if headerErr == nil { + headerErr = ss.SendHeader(header) + } + if headerErr != nil { + d.ReleaseOutcome(b.Name) + cancel() + return headerErr + } + for err == nil { + if sendErr := ss.SendMsg(&first); sendErr != nil { + d.ReleaseOutcome(b.Name) + cancel() + return sendErr + } + first.Reset() + err = upstream.RecvMsg(&first) + } + ss.SetTrailer(upstream.Trailer()) + if errors.Is(err, io.EOF) { + err = nil + } + switch classifyRPCOutcome(ss.Context(), err) { + case rpcSuccess: + d.RecordOutcome(b.Name, true) + case rpcFailure: + d.RecordOutcome(b.Name, false) + default: + d.ReleaseOutcome(b.Name) + } + cancel() + return err + } + if lastErr == nil { + lastErr = status.Error(codes.Unavailable, "no eligible backend for historical query") + } + ss.SetTrailer(lastTrailer) + if len(lastHeader) > 0 { + _ = ss.SendHeader(lastHeader) + } + return lastErr +} + +func missingStateStatus(err error) bool { + switch status.Code(err) { + case codes.Unknown, codes.Internal, codes.NotFound, codes.FailedPrecondition, codes.OutOfRange: + return history.Unavailable(status.Convert(err).Message()) + default: + return false + } +} diff --git a/internal/server/cosmos_grpc/history_methods.go b/internal/server/cosmos_grpc/history_methods.go new file mode 100644 index 0000000..a7b3ec2 --- /dev/null +++ b/internal/server/cosmos_grpc/history_methods.go @@ -0,0 +1,442 @@ +// Code generated by tools/generate-history-methods.py; DO NOT EDIT. +// Source versions match Injective core 2e3b4f9a9ef3d2c5c0758dd2e3a2b23c35458190. +// Cosmos: v0.50.14-inj.11; IBC: v8.7.0-inj.4; Wasmd: v0.53.3-inj.3. +// Modules are selected from injective-chain/app/app.go; Query RPCs are +// read-only module contracts. Streaming methods, Msg services, unknown +// future methods, and off-chain indexer services are excluded. +package cosmos_grpc + +var historicalUnaryMethods = map[string]struct{}{ + // https://github.com/InjectiveLabs/cosmos-sdk/blob/3361beeef240618d6dd70ca35259880bb73e7cda/proto/cosmos/auth/v1beta1/query.proto + "/cosmos.auth.v1beta1.Query/Account": {}, + "/cosmos.auth.v1beta1.Query/AccountAddressByID": {}, + "/cosmos.auth.v1beta1.Query/AccountInfo": {}, + "/cosmos.auth.v1beta1.Query/Accounts": {}, + "/cosmos.auth.v1beta1.Query/AddressBytesToString": {}, + "/cosmos.auth.v1beta1.Query/AddressStringToBytes": {}, + "/cosmos.auth.v1beta1.Query/Bech32Prefix": {}, + "/cosmos.auth.v1beta1.Query/ModuleAccountByName": {}, + "/cosmos.auth.v1beta1.Query/ModuleAccounts": {}, + "/cosmos.auth.v1beta1.Query/Params": {}, + // https://github.com/InjectiveLabs/cosmos-sdk/blob/3361beeef240618d6dd70ca35259880bb73e7cda/proto/cosmos/authz/v1beta1/query.proto + "/cosmos.authz.v1beta1.Query/GranteeGrants": {}, + "/cosmos.authz.v1beta1.Query/GranterGrants": {}, + "/cosmos.authz.v1beta1.Query/Grants": {}, + // https://github.com/InjectiveLabs/cosmos-sdk/blob/3361beeef240618d6dd70ca35259880bb73e7cda/proto/cosmos/bank/v1beta1/query.proto + "/cosmos.bank.v1beta1.Query/AllBalances": {}, + "/cosmos.bank.v1beta1.Query/Balance": {}, + "/cosmos.bank.v1beta1.Query/DenomMetadata": {}, + "/cosmos.bank.v1beta1.Query/DenomMetadataByQueryString": {}, + "/cosmos.bank.v1beta1.Query/DenomOwners": {}, + "/cosmos.bank.v1beta1.Query/DenomOwnersByQuery": {}, + "/cosmos.bank.v1beta1.Query/DenomsMetadata": {}, + "/cosmos.bank.v1beta1.Query/Params": {}, + "/cosmos.bank.v1beta1.Query/SendEnabled": {}, + "/cosmos.bank.v1beta1.Query/SpendableBalanceByDenom": {}, + "/cosmos.bank.v1beta1.Query/SpendableBalances": {}, + "/cosmos.bank.v1beta1.Query/SupplyOf": {}, + "/cosmos.bank.v1beta1.Query/TotalSupply": {}, + // https://github.com/InjectiveLabs/cosmos-sdk/blob/3361beeef240618d6dd70ca35259880bb73e7cda/proto/cosmos/base/tendermint/v1beta1/query.proto + "/cosmos.base.tendermint.v1beta1.Service/ABCIQuery": {}, + "/cosmos.base.tendermint.v1beta1.Service/GetBlockByHeight": {}, + "/cosmos.base.tendermint.v1beta1.Service/GetValidatorSetByHeight": {}, + // https://github.com/InjectiveLabs/cosmos-sdk/blob/3361beeef240618d6dd70ca35259880bb73e7cda/proto/cosmos/consensus/v1/query.proto + "/cosmos.consensus.v1.Query/Params": {}, + // https://github.com/InjectiveLabs/cosmos-sdk/blob/3361beeef240618d6dd70ca35259880bb73e7cda/proto/cosmos/distribution/v1beta1/query.proto + "/cosmos.distribution.v1beta1.Query/CommunityPool": {}, + "/cosmos.distribution.v1beta1.Query/DelegationRewards": {}, + "/cosmos.distribution.v1beta1.Query/DelegationTotalRewards": {}, + "/cosmos.distribution.v1beta1.Query/DelegatorValidators": {}, + "/cosmos.distribution.v1beta1.Query/DelegatorWithdrawAddress": {}, + "/cosmos.distribution.v1beta1.Query/Params": {}, + "/cosmos.distribution.v1beta1.Query/ValidatorCommission": {}, + "/cosmos.distribution.v1beta1.Query/ValidatorDistributionInfo": {}, + "/cosmos.distribution.v1beta1.Query/ValidatorOutstandingRewards": {}, + "/cosmos.distribution.v1beta1.Query/ValidatorSlashes": {}, + // https://github.com/InjectiveLabs/cosmos-sdk/blob/3361beeef240618d6dd70ca35259880bb73e7cda/proto/cosmos/evidence/v1beta1/query.proto + "/cosmos.evidence.v1beta1.Query/AllEvidence": {}, + "/cosmos.evidence.v1beta1.Query/Evidence": {}, + // https://github.com/InjectiveLabs/cosmos-sdk/blob/3361beeef240618d6dd70ca35259880bb73e7cda/proto/cosmos/feegrant/v1beta1/query.proto + "/cosmos.feegrant.v1beta1.Query/Allowance": {}, + "/cosmos.feegrant.v1beta1.Query/Allowances": {}, + "/cosmos.feegrant.v1beta1.Query/AllowancesByGranter": {}, + // https://github.com/InjectiveLabs/cosmos-sdk/blob/3361beeef240618d6dd70ca35259880bb73e7cda/proto/cosmos/gov/v1/query.proto + "/cosmos.gov.v1.Query/Constitution": {}, + "/cosmos.gov.v1.Query/Deposit": {}, + "/cosmos.gov.v1.Query/Deposits": {}, + "/cosmos.gov.v1.Query/Params": {}, + "/cosmos.gov.v1.Query/Proposal": {}, + "/cosmos.gov.v1.Query/Proposals": {}, + "/cosmos.gov.v1.Query/TallyResult": {}, + "/cosmos.gov.v1.Query/Vote": {}, + "/cosmos.gov.v1.Query/Votes": {}, + // https://github.com/InjectiveLabs/cosmos-sdk/blob/3361beeef240618d6dd70ca35259880bb73e7cda/proto/cosmos/gov/v1beta1/query.proto + "/cosmos.gov.v1beta1.Query/Deposit": {}, + "/cosmos.gov.v1beta1.Query/Deposits": {}, + "/cosmos.gov.v1beta1.Query/Params": {}, + "/cosmos.gov.v1beta1.Query/Proposal": {}, + "/cosmos.gov.v1beta1.Query/Proposals": {}, + "/cosmos.gov.v1beta1.Query/TallyResult": {}, + "/cosmos.gov.v1beta1.Query/Vote": {}, + "/cosmos.gov.v1beta1.Query/Votes": {}, + // https://github.com/InjectiveLabs/cosmos-sdk/blob/3361beeef240618d6dd70ca35259880bb73e7cda/proto/cosmos/mint/v1beta1/query.proto + "/cosmos.mint.v1beta1.Query/AnnualProvisions": {}, + "/cosmos.mint.v1beta1.Query/Inflation": {}, + "/cosmos.mint.v1beta1.Query/Params": {}, + // https://github.com/InjectiveLabs/cosmos-sdk/blob/3361beeef240618d6dd70ca35259880bb73e7cda/proto/cosmos/params/v1beta1/query.proto + "/cosmos.params.v1beta1.Query/Params": {}, + "/cosmos.params.v1beta1.Query/Subspaces": {}, + // https://github.com/InjectiveLabs/cosmos-sdk/blob/3361beeef240618d6dd70ca35259880bb73e7cda/proto/cosmos/slashing/v1beta1/query.proto + "/cosmos.slashing.v1beta1.Query/Params": {}, + "/cosmos.slashing.v1beta1.Query/SigningInfo": {}, + "/cosmos.slashing.v1beta1.Query/SigningInfos": {}, + // https://github.com/InjectiveLabs/cosmos-sdk/blob/3361beeef240618d6dd70ca35259880bb73e7cda/proto/cosmos/staking/v1beta1/query.proto + "/cosmos.staking.v1beta1.Query/AllowedDelegationTransferReceivers": {}, + "/cosmos.staking.v1beta1.Query/Delegation": {}, + "/cosmos.staking.v1beta1.Query/DelegatorDelegations": {}, + "/cosmos.staking.v1beta1.Query/DelegatorUnbondingDelegations": {}, + "/cosmos.staking.v1beta1.Query/DelegatorValidator": {}, + "/cosmos.staking.v1beta1.Query/DelegatorValidators": {}, + "/cosmos.staking.v1beta1.Query/HistoricalInfo": {}, + "/cosmos.staking.v1beta1.Query/Params": {}, + "/cosmos.staking.v1beta1.Query/Pool": {}, + "/cosmos.staking.v1beta1.Query/Redelegations": {}, + "/cosmos.staking.v1beta1.Query/UnbondingDelegation": {}, + "/cosmos.staking.v1beta1.Query/Validator": {}, + "/cosmos.staking.v1beta1.Query/ValidatorDelegations": {}, + "/cosmos.staking.v1beta1.Query/ValidatorUnbondingDelegations": {}, + "/cosmos.staking.v1beta1.Query/Validators": {}, + // https://github.com/InjectiveLabs/cosmos-sdk/blob/3361beeef240618d6dd70ca35259880bb73e7cda/proto/cosmos/tx/v1beta1/service.proto + "/cosmos.tx.v1beta1.Service/GetBlockWithTxs": {}, + // https://github.com/InjectiveLabs/cosmos-sdk/blob/3361beeef240618d6dd70ca35259880bb73e7cda/proto/cosmos/upgrade/v1beta1/query.proto + "/cosmos.upgrade.v1beta1.Query/AppliedPlan": {}, + "/cosmos.upgrade.v1beta1.Query/Authority": {}, + "/cosmos.upgrade.v1beta1.Query/CurrentPlan": {}, + "/cosmos.upgrade.v1beta1.Query/ModuleVersions": {}, + "/cosmos.upgrade.v1beta1.Query/UpgradedConsensusState": {}, + // https://github.com/InjectiveLabs/ibc-go/blob/dbee2d797b8a34179ece0082eebe96bfe023de1e/proto/ibc/applications/fee/v1/query.proto + "/ibc.applications.fee.v1.Query/CounterpartyPayee": {}, + "/ibc.applications.fee.v1.Query/FeeEnabledChannel": {}, + "/ibc.applications.fee.v1.Query/FeeEnabledChannels": {}, + "/ibc.applications.fee.v1.Query/IncentivizedPacket": {}, + "/ibc.applications.fee.v1.Query/IncentivizedPackets": {}, + "/ibc.applications.fee.v1.Query/IncentivizedPacketsForChannel": {}, + "/ibc.applications.fee.v1.Query/Payee": {}, + "/ibc.applications.fee.v1.Query/TotalAckFees": {}, + "/ibc.applications.fee.v1.Query/TotalRecvFees": {}, + "/ibc.applications.fee.v1.Query/TotalTimeoutFees": {}, + // https://github.com/InjectiveLabs/ibc-go/blob/dbee2d797b8a34179ece0082eebe96bfe023de1e/proto/ibc/applications/interchain_accounts/host/v1/query.proto + "/ibc.applications.interchain_accounts.host.v1.Query/Params": {}, + // https://github.com/InjectiveLabs/ibc-go/blob/dbee2d797b8a34179ece0082eebe96bfe023de1e/proto/ibc/applications/transfer/v1/query.proto + "/ibc.applications.transfer.v1.Query/DenomHash": {}, + "/ibc.applications.transfer.v1.Query/DenomTrace": {}, + "/ibc.applications.transfer.v1.Query/DenomTraces": {}, + "/ibc.applications.transfer.v1.Query/EscrowAddress": {}, + "/ibc.applications.transfer.v1.Query/Params": {}, + "/ibc.applications.transfer.v1.Query/TotalEscrowForDenom": {}, + // https://github.com/InjectiveLabs/ibc-go/blob/dbee2d797b8a34179ece0082eebe96bfe023de1e/proto/ibc/core/channel/v1/query.proto + "/ibc.core.channel.v1.Query/Channel": {}, + "/ibc.core.channel.v1.Query/ChannelClientState": {}, + "/ibc.core.channel.v1.Query/ChannelConsensusState": {}, + "/ibc.core.channel.v1.Query/ChannelParams": {}, + "/ibc.core.channel.v1.Query/Channels": {}, + "/ibc.core.channel.v1.Query/ConnectionChannels": {}, + "/ibc.core.channel.v1.Query/NextSequenceReceive": {}, + "/ibc.core.channel.v1.Query/NextSequenceSend": {}, + "/ibc.core.channel.v1.Query/PacketAcknowledgement": {}, + "/ibc.core.channel.v1.Query/PacketAcknowledgements": {}, + "/ibc.core.channel.v1.Query/PacketCommitment": {}, + "/ibc.core.channel.v1.Query/PacketCommitments": {}, + "/ibc.core.channel.v1.Query/PacketReceipt": {}, + "/ibc.core.channel.v1.Query/UnreceivedAcks": {}, + "/ibc.core.channel.v1.Query/UnreceivedPackets": {}, + "/ibc.core.channel.v1.Query/Upgrade": {}, + "/ibc.core.channel.v1.Query/UpgradeError": {}, + // https://github.com/InjectiveLabs/ibc-go/blob/dbee2d797b8a34179ece0082eebe96bfe023de1e/proto/ibc/core/client/v1/query.proto + "/ibc.core.client.v1.Query/ClientParams": {}, + "/ibc.core.client.v1.Query/ClientState": {}, + "/ibc.core.client.v1.Query/ClientStates": {}, + "/ibc.core.client.v1.Query/ClientStatus": {}, + "/ibc.core.client.v1.Query/ConsensusState": {}, + "/ibc.core.client.v1.Query/ConsensusStateHeights": {}, + "/ibc.core.client.v1.Query/ConsensusStates": {}, + "/ibc.core.client.v1.Query/UpgradedClientState": {}, + "/ibc.core.client.v1.Query/UpgradedConsensusState": {}, + "/ibc.core.client.v1.Query/VerifyMembership": {}, + // https://github.com/InjectiveLabs/ibc-go/blob/dbee2d797b8a34179ece0082eebe96bfe023de1e/proto/ibc/core/connection/v1/query.proto + "/ibc.core.connection.v1.Query/ClientConnections": {}, + "/ibc.core.connection.v1.Query/Connection": {}, + "/ibc.core.connection.v1.Query/ConnectionClientState": {}, + "/ibc.core.connection.v1.Query/ConnectionConsensusState": {}, + "/ibc.core.connection.v1.Query/ConnectionParams": {}, + "/ibc.core.connection.v1.Query/Connections": {}, + // https://github.com/InjectiveLabs/injective-core/blob/2e3b4f9a9ef3d2c5c0758dd2e3a2b23c35458190/proto/injective/auction/v1beta1/query.proto + "/injective.auction.v1beta1.Query/AuctionModuleState": {}, + "/injective.auction.v1beta1.Query/AuctionParams": {}, + "/injective.auction.v1beta1.Query/CurrentAuctionBasket": {}, + "/injective.auction.v1beta1.Query/LastAuctionResult": {}, + "/injective.auction.v1beta1.Query/Voucher": {}, + "/injective.auction.v1beta1.Query/Vouchers": {}, + // https://github.com/InjectiveLabs/injective-core/blob/2e3b4f9a9ef3d2c5c0758dd2e3a2b23c35458190/proto/injective/downtimedetector/v1beta1/query.proto + "/injective.downtimedetector.v1beta1.Query/RecoveredSinceDowntimeOfLength": {}, + // https://github.com/InjectiveLabs/injective-core/blob/2e3b4f9a9ef3d2c5c0758dd2e3a2b23c35458190/proto/injective/erc20/v1beta1/query.proto + "/injective.erc20.v1beta1.Query/AllTokenPairs": {}, + "/injective.erc20.v1beta1.Query/Params": {}, + "/injective.erc20.v1beta1.Query/TokenPairByDenom": {}, + "/injective.erc20.v1beta1.Query/TokenPairByERC20Address": {}, + // https://github.com/InjectiveLabs/injective-core/blob/2e3b4f9a9ef3d2c5c0758dd2e3a2b23c35458190/proto/injective/evm/v1/query.proto + "/injective.evm.v1.Query/Account": {}, + "/injective.evm.v1.Query/Balance": {}, + "/injective.evm.v1.Query/BaseFee": {}, + "/injective.evm.v1.Query/Code": {}, + "/injective.evm.v1.Query/CosmosAccount": {}, + "/injective.evm.v1.Query/EstimateGas": {}, + "/injective.evm.v1.Query/EthCall": {}, + "/injective.evm.v1.Query/Params": {}, + "/injective.evm.v1.Query/Storage": {}, + "/injective.evm.v1.Query/TraceBlock": {}, + "/injective.evm.v1.Query/TraceCall": {}, + "/injective.evm.v1.Query/TraceTx": {}, + "/injective.evm.v1.Query/ValidatorAccount": {}, + // https://github.com/InjectiveLabs/injective-core/blob/2e3b4f9a9ef3d2c5c0758dd2e3a2b23c35458190/proto/injective/exchange/v1beta1/query.proto + "/injective.exchange.v1beta1.Query/AccountAddressDerivativeOrders": {}, + "/injective.exchange.v1beta1.Query/AccountAddressSpotOrders": {}, + "/injective.exchange.v1beta1.Query/ActiveStakeGrant": {}, + "/injective.exchange.v1beta1.Query/AggregateMarketVolume": {}, + "/injective.exchange.v1beta1.Query/AggregateMarketVolumes": {}, + "/injective.exchange.v1beta1.Query/AggregateVolume": {}, + "/injective.exchange.v1beta1.Query/AggregateVolumes": {}, + "/injective.exchange.v1beta1.Query/BalanceMismatches": {}, + "/injective.exchange.v1beta1.Query/BalanceWithBalanceHolds": {}, + "/injective.exchange.v1beta1.Query/BinaryOptionsMarkets": {}, + "/injective.exchange.v1beta1.Query/DenomDecimal": {}, + "/injective.exchange.v1beta1.Query/DenomDecimals": {}, + "/injective.exchange.v1beta1.Query/DenomMinNotional": {}, + "/injective.exchange.v1beta1.Query/DenomMinNotionals": {}, + "/injective.exchange.v1beta1.Query/DerivativeMarket": {}, + "/injective.exchange.v1beta1.Query/DerivativeMarketAddress": {}, + "/injective.exchange.v1beta1.Query/DerivativeMarkets": {}, + "/injective.exchange.v1beta1.Query/DerivativeMidPriceAndTOB": {}, + "/injective.exchange.v1beta1.Query/DerivativeOrderbook": {}, + "/injective.exchange.v1beta1.Query/DerivativeOrdersByHashes": {}, + "/injective.exchange.v1beta1.Query/ExchangeBalances": {}, + "/injective.exchange.v1beta1.Query/ExchangeModuleState": {}, + "/injective.exchange.v1beta1.Query/ExpiryFuturesMarketInfo": {}, + "/injective.exchange.v1beta1.Query/FeeDiscountAccountInfo": {}, + "/injective.exchange.v1beta1.Query/FeeDiscountSchedule": {}, + "/injective.exchange.v1beta1.Query/FeeDiscountTierStatistics": {}, + "/injective.exchange.v1beta1.Query/FullSpotMarket": {}, + "/injective.exchange.v1beta1.Query/FullSpotMarkets": {}, + "/injective.exchange.v1beta1.Query/GrantAuthorization": {}, + "/injective.exchange.v1beta1.Query/GrantAuthorizations": {}, + "/injective.exchange.v1beta1.Query/HistoricalTradeRecords": {}, + "/injective.exchange.v1beta1.Query/IsOptedOutOfRewards": {}, + "/injective.exchange.v1beta1.Query/L3DerivativeOrderBook": {}, + "/injective.exchange.v1beta1.Query/L3SpotOrderBook": {}, + "/injective.exchange.v1beta1.Query/MarketAtomicExecutionFeeMultiplier": {}, + "/injective.exchange.v1beta1.Query/MarketBalance": {}, + "/injective.exchange.v1beta1.Query/MarketBalances": {}, + "/injective.exchange.v1beta1.Query/MarketVolatility": {}, + "/injective.exchange.v1beta1.Query/MitoVaultInfos": {}, + "/injective.exchange.v1beta1.Query/OptedOutOfRewardsAccounts": {}, + "/injective.exchange.v1beta1.Query/PendingTradeRewardPoints": {}, + "/injective.exchange.v1beta1.Query/PerpetualMarketFunding": {}, + "/injective.exchange.v1beta1.Query/PerpetualMarketInfo": {}, + "/injective.exchange.v1beta1.Query/Positions": {}, + "/injective.exchange.v1beta1.Query/QueryExchangeParams": {}, + "/injective.exchange.v1beta1.Query/QueryMarketIDFromVault": {}, + "/injective.exchange.v1beta1.Query/SpotMarket": {}, + "/injective.exchange.v1beta1.Query/SpotMarkets": {}, + "/injective.exchange.v1beta1.Query/SpotMidPriceAndTOB": {}, + "/injective.exchange.v1beta1.Query/SpotOrderbook": {}, + "/injective.exchange.v1beta1.Query/SpotOrdersByHashes": {}, + "/injective.exchange.v1beta1.Query/SubaccountDeposit": {}, + "/injective.exchange.v1beta1.Query/SubaccountDeposits": {}, + "/injective.exchange.v1beta1.Query/SubaccountEffectivePositionInMarket": {}, + "/injective.exchange.v1beta1.Query/SubaccountOrderMetadata": {}, + "/injective.exchange.v1beta1.Query/SubaccountOrders": {}, + "/injective.exchange.v1beta1.Query/SubaccountPositionInMarket": {}, + "/injective.exchange.v1beta1.Query/SubaccountPositions": {}, + "/injective.exchange.v1beta1.Query/SubaccountTradeNonce": {}, + "/injective.exchange.v1beta1.Query/TradeRewardCampaign": {}, + "/injective.exchange.v1beta1.Query/TradeRewardPoints": {}, + "/injective.exchange.v1beta1.Query/TraderDerivativeConditionalOrders": {}, + "/injective.exchange.v1beta1.Query/TraderDerivativeOrders": {}, + "/injective.exchange.v1beta1.Query/TraderDerivativeTransientOrders": {}, + "/injective.exchange.v1beta1.Query/TraderSpotOrders": {}, + "/injective.exchange.v1beta1.Query/TraderSpotTransientOrders": {}, + // https://github.com/InjectiveLabs/injective-core/blob/2e3b4f9a9ef3d2c5c0758dd2e3a2b23c35458190/proto/injective/exchange/v2/query.proto + "/injective.exchange.v2.Query/AccountAddressDerivativeOrders": {}, + "/injective.exchange.v2.Query/AccountAddressSpotOrders": {}, + "/injective.exchange.v2.Query/ActiveStakeGrant": {}, + "/injective.exchange.v2.Query/AggregateMarketVolume": {}, + "/injective.exchange.v2.Query/AggregateMarketVolumes": {}, + "/injective.exchange.v2.Query/AggregateVolume": {}, + "/injective.exchange.v2.Query/AggregateVolumes": {}, + "/injective.exchange.v2.Query/AuctionExchangeTransferDenomDecimal": {}, + "/injective.exchange.v2.Query/AuctionExchangeTransferDenomDecimals": {}, + "/injective.exchange.v2.Query/BalanceMismatches": {}, + "/injective.exchange.v2.Query/BalanceWithBalanceHolds": {}, + "/injective.exchange.v2.Query/BinaryOptionsMarkets": {}, + "/injective.exchange.v2.Query/CrossMarginPoolSnapshot": {}, + "/injective.exchange.v2.Query/DenomMinNotional": {}, + "/injective.exchange.v2.Query/DenomMinNotionals": {}, + "/injective.exchange.v2.Query/DerivativeMarket": {}, + "/injective.exchange.v2.Query/DerivativeMarketAddress": {}, + "/injective.exchange.v2.Query/DerivativeMarkets": {}, + "/injective.exchange.v2.Query/DerivativeMidPriceAndTOB": {}, + "/injective.exchange.v2.Query/DerivativeOrderbook": {}, + "/injective.exchange.v2.Query/DerivativeOrdersByHashes": {}, + "/injective.exchange.v2.Query/EffectiveSubaccountMarketRiskMode": {}, + "/injective.exchange.v2.Query/ExchangeBalances": {}, + "/injective.exchange.v2.Query/ExchangeModuleState": {}, + "/injective.exchange.v2.Query/ExpiryFuturesMarketInfo": {}, + "/injective.exchange.v2.Query/FeeDiscountAccountInfo": {}, + "/injective.exchange.v2.Query/FeeDiscountSchedule": {}, + "/injective.exchange.v2.Query/FeeDiscountTierStatistics": {}, + "/injective.exchange.v2.Query/FullSpotMarket": {}, + "/injective.exchange.v2.Query/FullSpotMarkets": {}, + "/injective.exchange.v2.Query/GrantAuthorization": {}, + "/injective.exchange.v2.Query/GrantAuthorizations": {}, + "/injective.exchange.v2.Query/HistoricalTradeRecords": {}, + "/injective.exchange.v2.Query/IsOptedOutOfRewards": {}, + "/injective.exchange.v2.Query/L3DerivativeOrderBook": {}, + "/injective.exchange.v2.Query/L3SpotOrderBook": {}, + "/injective.exchange.v2.Query/MarketAtomicExecutionFeeMultiplier": {}, + "/injective.exchange.v2.Query/MarketBalance": {}, + "/injective.exchange.v2.Query/MarketBalances": {}, + "/injective.exchange.v2.Query/MarketVolatility": {}, + "/injective.exchange.v2.Query/MitoVaultInfos": {}, + "/injective.exchange.v2.Query/OpenInterest": {}, + "/injective.exchange.v2.Query/OptedOutOfRewardsAccounts": {}, + "/injective.exchange.v2.Query/PendingTradeRewardPoints": {}, + "/injective.exchange.v2.Query/PerpetualMarketFunding": {}, + "/injective.exchange.v2.Query/PerpetualMarketInfo": {}, + "/injective.exchange.v2.Query/Positions": {}, + "/injective.exchange.v2.Query/PositionsInMarket": {}, + "/injective.exchange.v2.Query/QueryExchangeParams": {}, + "/injective.exchange.v2.Query/QueryMarketIDFromVault": {}, + "/injective.exchange.v2.Query/SpotMarket": {}, + "/injective.exchange.v2.Query/SpotMarkets": {}, + "/injective.exchange.v2.Query/SpotMidPriceAndTOB": {}, + "/injective.exchange.v2.Query/SpotOrderbook": {}, + "/injective.exchange.v2.Query/SpotOrdersByHashes": {}, + "/injective.exchange.v2.Query/SpotSwapInput": {}, + "/injective.exchange.v2.Query/SpotSwapOutput": {}, + "/injective.exchange.v2.Query/SubaccountDeposit": {}, + "/injective.exchange.v2.Query/SubaccountDeposits": {}, + "/injective.exchange.v2.Query/SubaccountEffectivePositionInMarket": {}, + "/injective.exchange.v2.Query/SubaccountMarketRiskModes": {}, + "/injective.exchange.v2.Query/SubaccountOrderMetadata": {}, + "/injective.exchange.v2.Query/SubaccountOrders": {}, + "/injective.exchange.v2.Query/SubaccountPositionInMarket": {}, + "/injective.exchange.v2.Query/SubaccountPositions": {}, + "/injective.exchange.v2.Query/SubaccountRiskProfile": {}, + "/injective.exchange.v2.Query/SubaccountTradeNonce": {}, + "/injective.exchange.v2.Query/TradeRewardCampaign": {}, + "/injective.exchange.v2.Query/TradeRewardPoints": {}, + "/injective.exchange.v2.Query/TraderDerivativeConditionalOrders": {}, + "/injective.exchange.v2.Query/TraderDerivativeOrders": {}, + "/injective.exchange.v2.Query/TraderDerivativeTransientOrders": {}, + "/injective.exchange.v2.Query/TraderSpotOrders": {}, + "/injective.exchange.v2.Query/TraderSpotTransientOrders": {}, + // https://github.com/InjectiveLabs/injective-core/blob/2e3b4f9a9ef3d2c5c0758dd2e3a2b23c35458190/proto/injective/insurance/v1beta1/query.proto + "/injective.insurance.v1beta1.Query/EstimatedRedemptions": {}, + "/injective.insurance.v1beta1.Query/FailedRedemptions": {}, + "/injective.insurance.v1beta1.Query/InsuranceFund": {}, + "/injective.insurance.v1beta1.Query/InsuranceFunds": {}, + "/injective.insurance.v1beta1.Query/InsuranceModuleState": {}, + "/injective.insurance.v1beta1.Query/InsuranceParams": {}, + "/injective.insurance.v1beta1.Query/PendingRedemptions": {}, + "/injective.insurance.v1beta1.Query/Voucher": {}, + "/injective.insurance.v1beta1.Query/Vouchers": {}, + // https://github.com/InjectiveLabs/injective-core/blob/2e3b4f9a9ef3d2c5c0758dd2e3a2b23c35458190/proto/injective/oracle/v1beta1/query.proto + "/injective.oracle.v1beta1.Query/BandIBCPriceStates": {}, + "/injective.oracle.v1beta1.Query/BandPriceStates": {}, + "/injective.oracle.v1beta1.Query/BandRelayers": {}, + "/injective.oracle.v1beta1.Query/ChainlinkDataStreamsPriceStates": {}, + "/injective.oracle.v1beta1.Query/CoinbasePriceStates": {}, + "/injective.oracle.v1beta1.Query/HistoricalPriceRecords": {}, + "/injective.oracle.v1beta1.Query/OracleModuleState": {}, + "/injective.oracle.v1beta1.Query/OraclePrice": {}, + "/injective.oracle.v1beta1.Query/OracleProviderPrices": {}, + "/injective.oracle.v1beta1.Query/OracleProvidersInfo": {}, + "/injective.oracle.v1beta1.Query/OracleVolatility": {}, + "/injective.oracle.v1beta1.Query/Params": {}, + "/injective.oracle.v1beta1.Query/PriceFeedPriceStates": {}, + "/injective.oracle.v1beta1.Query/ProviderPriceState": {}, + "/injective.oracle.v1beta1.Query/PythPrice": {}, + "/injective.oracle.v1beta1.Query/PythPriceStates": {}, + "/injective.oracle.v1beta1.Query/PythProPriceStates": {}, + "/injective.oracle.v1beta1.Query/SedaFastPriceStates": {}, + "/injective.oracle.v1beta1.Query/StorkPriceStates": {}, + "/injective.oracle.v1beta1.Query/StorkPublishers": {}, + // https://github.com/InjectiveLabs/injective-core/blob/2e3b4f9a9ef3d2c5c0758dd2e3a2b23c35458190/proto/injective/peggy/v1/query.proto + "/injective.peggy.v1.Query/BatchConfirms": {}, + "/injective.peggy.v1.Query/BatchFees": {}, + "/injective.peggy.v1.Query/BatchRequestByNonce": {}, + "/injective.peggy.v1.Query/CurrentValset": {}, + "/injective.peggy.v1.Query/DenomToERC20": {}, + "/injective.peggy.v1.Query/ERC20ToDenom": {}, + "/injective.peggy.v1.Query/GetDelegateKeyByEth": {}, + "/injective.peggy.v1.Query/GetDelegateKeyByOrchestrator": {}, + "/injective.peggy.v1.Query/GetDelegateKeyByValidator": {}, + "/injective.peggy.v1.Query/GetPendingSendToEth": {}, + "/injective.peggy.v1.Query/LastEventByAddr": {}, + "/injective.peggy.v1.Query/LastPendingBatchRequestByAddr": {}, + "/injective.peggy.v1.Query/LastPendingValsetRequestByAddr": {}, + "/injective.peggy.v1.Query/LastValsetRequests": {}, + "/injective.peggy.v1.Query/MissingPeggoNonces": {}, + "/injective.peggy.v1.Query/OutgoingTxBatches": {}, + "/injective.peggy.v1.Query/Params": {}, + "/injective.peggy.v1.Query/PeggyModuleState": {}, + "/injective.peggy.v1.Query/ValsetConfirm": {}, + "/injective.peggy.v1.Query/ValsetConfirmsByNonce": {}, + "/injective.peggy.v1.Query/ValsetRequest": {}, + // https://github.com/InjectiveLabs/injective-core/blob/2e3b4f9a9ef3d2c5c0758dd2e3a2b23c35458190/proto/injective/permissions/v1beta1/query.proto + "/injective.permissions.v1beta1.Query/ActorsByRole": {}, + "/injective.permissions.v1beta1.Query/Namespace": {}, + "/injective.permissions.v1beta1.Query/NamespaceDenoms": {}, + "/injective.permissions.v1beta1.Query/Namespaces": {}, + "/injective.permissions.v1beta1.Query/Params": {}, + "/injective.permissions.v1beta1.Query/PermissionsModuleState": {}, + "/injective.permissions.v1beta1.Query/PolicyManagerCapabilities": {}, + "/injective.permissions.v1beta1.Query/PolicyStatuses": {}, + "/injective.permissions.v1beta1.Query/RoleManager": {}, + "/injective.permissions.v1beta1.Query/RoleManagers": {}, + "/injective.permissions.v1beta1.Query/RolesByActor": {}, + "/injective.permissions.v1beta1.Query/Voucher": {}, + "/injective.permissions.v1beta1.Query/Vouchers": {}, + // https://github.com/InjectiveLabs/injective-core/blob/2e3b4f9a9ef3d2c5c0758dd2e3a2b23c35458190/proto/injective/tokenfactory/v1beta1/query.proto + "/injective.tokenfactory.v1beta1.Query/DenomAuthorityMetadata": {}, + "/injective.tokenfactory.v1beta1.Query/DenomsFromCreator": {}, + "/injective.tokenfactory.v1beta1.Query/Params": {}, + "/injective.tokenfactory.v1beta1.Query/TokenfactoryModuleState": {}, + // https://github.com/InjectiveLabs/injective-core/blob/2e3b4f9a9ef3d2c5c0758dd2e3a2b23c35458190/proto/injective/txfees/v1beta1/query.proto + "/injective.txfees.v1beta1.Query/GetEipBaseFee": {}, + "/injective.txfees.v1beta1.Query/Params": {}, + // https://github.com/InjectiveLabs/injective-core/blob/2e3b4f9a9ef3d2c5c0758dd2e3a2b23c35458190/proto/injective/wasmx/v1/query.proto + "/injective.wasmx.v1.Query/ContractRegistrationInfo": {}, + "/injective.wasmx.v1.Query/WasmxModuleState": {}, + "/injective.wasmx.v1.Query/WasmxParams": {}, + // https://github.com/InjectiveLabs/wasmd/blob/e0e9de04a62d8405a85abd6e8ef3fdad9dd5461e/proto/cosmwasm/wasm/v1/query.proto + "/cosmwasm.wasm.v1.Query/AllContractState": {}, + "/cosmwasm.wasm.v1.Query/BuildAddress": {}, + "/cosmwasm.wasm.v1.Query/Code": {}, + "/cosmwasm.wasm.v1.Query/Codes": {}, + "/cosmwasm.wasm.v1.Query/ContractHistory": {}, + "/cosmwasm.wasm.v1.Query/ContractInfo": {}, + "/cosmwasm.wasm.v1.Query/ContractsByCode": {}, + "/cosmwasm.wasm.v1.Query/ContractsByCreator": {}, + "/cosmwasm.wasm.v1.Query/Params": {}, + "/cosmwasm.wasm.v1.Query/PinnedCodes": {}, + "/cosmwasm.wasm.v1.Query/RawContractState": {}, + "/cosmwasm.wasm.v1.Query/SmartContractState": {}, +} + +// Retry only schema-verified unary reads; names alone cannot establish +// whether a future Query method streams or has side effects. +func historicalReadMethod(method string) bool { + _, ok := historicalUnaryMethods[method] + return ok +} diff --git a/internal/server/cosmos_grpc/history_methods_review_test.go b/internal/server/cosmos_grpc/history_methods_review_test.go new file mode 100644 index 0000000..ed0ba8d --- /dev/null +++ b/internal/server/cosmos_grpc/history_methods_review_test.go @@ -0,0 +1,57 @@ +package cosmos_grpc + +import "testing" + +func TestHistoricalReadContractsCoverChainModules(t *testing.T) { + for _, method := range []string{ + "/cosmos.bank.v1beta1.Query/Balance", + "/cosmos.staking.v1beta1.Query/Delegation", + "/cosmos.distribution.v1beta1.Query/DelegationRewards", + "/cosmos.authz.v1beta1.Query/Grants", + "/cosmos.gov.v1.Query/Proposal", + "/ibc.core.client.v1.Query/ClientState", + "/ibc.core.channel.v1.Query/Channel", + "/ibc.applications.transfer.v1.Query/DenomTrace", + "/injective.exchange.v1beta1.Query/SpotMarkets", + "/injective.exchange.v2.Query/SpotMarkets", + "/injective.oracle.v1beta1.Query/Params", + "/injective.evm.v1.Query/Balance", + "/cosmwasm.wasm.v1.Query/SmartContractState", + } { + if !historicalReadMethod(method) { + t.Errorf("verified chain query excluded: %s", method) + } + } +} + +func TestHistoricalRetryNeverInfersReadContractFromNamespace(t *testing.T) { + for _, method := range []string{ + "/cosmos.bank.v1beta1.Query/FutureMethod", + "/cosmos.unknown.v1.Query/Watch", + "/injective.exchange.v2.Query/FutureStreamingMethod", + "/cosmos.bank.v1beta1.Msg/Send", + "/cosmos.tx.v1beta1.Service/BroadcastTx", + "/cosmos.tx.v1beta1.Service/Simulate", + "/ibc.core.channel.v1.Msg/RecvPacket", + "/injective.exchange.v2.Msg/CreateSpotMarketOrder", + "/injective.stream.v1beta1.Stream/Stream", + "/injective.stream.v2.Stream/StreamV2", + "/injective_exchange_rpc.InjectiveExchangeRPC/StreamOrders", + "/cosmwasm.wasm.v1.Msg/ExecuteContract", + "/cosmos.group.v1.Query/GroupInfo", + } { + if historicalReadMethod(method) { + t.Errorf("unverified or non-read RPC is retryable: %s", method) + } + } +} + +// Adding a protobuf body-height decoder must not silently confer retryability +// on a method whose unary read contract was never checked against its schema. +func TestBodyHeightManifestHasVerifiedUnaryContracts(t *testing.T) { + for method := range Manifest { + if !historicalReadMethod(method) { + t.Errorf("body-height method needs schema verification: %s", method) + } + } +} diff --git a/internal/server/cosmos_grpc/history_test.go b/internal/server/cosmos_grpc/history_test.go new file mode 100644 index 0000000..2b78bf8 --- /dev/null +++ b/internal/server/cosmos_grpc/history_test.go @@ -0,0 +1,222 @@ +package cosmos_grpc + +import ( + "context" + "io" + "net" + "strings" + "sync/atomic" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/emptypb" + "google.golang.org/protobuf/types/known/wrapperspb" + + "github.com/InjectiveLabs/stitch/internal/backend" + "github.com/InjectiveLabs/stitch/internal/circuit" + "github.com/InjectiveLabs/stitch/internal/pool" + "github.com/InjectiveLabs/stitch/internal/types" +) + +const historicalMissing = "failed to load state at height 105504992; version mismatch on immutable IAVL tree; version does not exist" +const historicalMethod = "/cosmos.bank.v1beta1.Query/Params" + +type orderedHistoricalSelector struct{ backends []*backend.Backend } + +func (s orderedHistoricalSelector) Candidates(types.RouteKey) []*backend.Backend { return s.backends } + +func historicalRig(t *testing.T, handlers ...func(grpc.ServerStream, *emptypb.Empty) error) (*grpc.ClientConn, *circuit.Manager, *Director) { + t.Helper() + var backends []*backend.Backend + for i, h := range handlers { + lis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + srv := grpc.NewServer(grpc.UnknownServiceHandler(func(_ any, ss grpc.ServerStream) error { + var req emptypb.Empty + if err := ss.RecvMsg(&req); err != nil { + return err + } + return h(ss, &req) + })) + go func() { _ = srv.Serve(lis) }() + t.Cleanup(srv.Stop) + backends = append(backends, &backend.Backend{Name: string(rune('a' + i)), Endpoints: map[types.Protocol]string{types.ProtoGRPC: lis.Addr().String()}}) + } + cm := circuit.NewManager(circuit.Policy{MinRequests: 1, ErrorThreshold: .5, OpenDuration: time.Minute}) + p := pool.NewGRPCPool(time.Minute) + t.Cleanup(p.CloseAll) + d := NewDirector(orderedHistoricalSelector{backends}, cm, p) + front, err := New("127.0.0.1:0", d) + if err != nil { + t.Fatal(err) + } + go func() { _ = front.Start(context.Background()) }() + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + _ = front.Shutdown(ctx) + }) + conn, err := grpc.NewClient(front.Addr(), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = conn.Close() }) + return conn, cm, d +} + +func TestGRPCHistoricalRetryPreservesRequestAndFinalMetadata(t *testing.T) { + var hits [2]atomic.Int32 + want := wrapperspb.String("opaque query payload") + check := func(ss grpc.ServerStream, req *emptypb.Empty) { + md, _ := metadata.FromIncomingContext(ss.Context()) + if strings.Join(md.Get(HeightHeader), "") != "105504992" { + t.Error("historical height lost") + } + if strings.Join(md.Get("authorization"), "") != "test-token" { + t.Error("metadata lost") + } + data, _ := proto.Marshal(want) + if string(req.ProtoReflect().GetUnknown()) != string(data) { + t.Error("protobuf payload changed") + } + } + conn, cm, _ := historicalRig(t, func(ss grpc.ServerStream, req *emptypb.Empty) error { + hits[0].Add(1) + check(ss, req) + ss.SetTrailer(metadata.Pairs("attempt", "missing")) + return status.Error(codes.Unknown, historicalMissing) + }, func(ss grpc.ServerStream, req *emptypb.Empty) error { + hits[1].Add(1) + check(ss, req) + _ = ss.SetHeader(metadata.Pairs(HeightHeader, "105504992")) + ss.SetTrailer(metadata.Pairs("attempt", "success")) + return ss.SendMsg(wrapperspb.String("historical value")) + }) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + ctx = metadata.AppendToOutgoingContext(ctx, HeightHeader, "105504992", "authorization", "test-token") + var got wrapperspb.StringValue + var header, trailer metadata.MD + if err := conn.Invoke(ctx, historicalMethod, want, &got, grpc.Header(&header), grpc.Trailer(&trailer)); err != nil { + t.Fatal(err) + } + if got.Value != "historical value" || hits[0].Load() != 1 || hits[1].Load() != 1 { + t.Fatalf("response=%q hits=%d,%d", got.Value, hits[0].Load(), hits[1].Load()) + } + if strings.Join(header.Get(HeightHeader), "") != "105504992" || strings.Join(trailer.Get("attempt"), "") != "success" { + t.Fatalf("metadata leaked or lost: %v %v", header, trailer) + } + if cm.State("a", types.ProtoGRPC) != circuit.StateClosed { + t.Fatal("retention failure opened circuit") + } +} + +func TestGRPCHistoricalRetryLimitsAndErrors(t *testing.T) { + for _, tc := range []struct { + name, method string + header bool + code codes.Code + message string + max int + wantHits int + }{ + {"all_missing", historicalMethod, true, codes.Unknown, historicalMissing, 3, 2}, + {"one_attempt", historicalMethod, true, codes.Unknown, historicalMissing, 1, 1}, + {"ordinary_error", historicalMethod, true, codes.Internal, "permission denied", 3, 1}, + {"permission_status", historicalMethod, true, codes.PermissionDenied, historicalMissing, 3, 1}, + {"latest", historicalMethod, false, codes.Unknown, historicalMissing, 3, 1}, + {"broadcast", "/cosmos.tx.v1beta1.Service/BroadcastTx", true, codes.Unknown, historicalMissing, 3, 1}, + {"unknown_method", "/custom.Service/Write", true, codes.Unknown, historicalMissing, 3, 1}, + } { + t.Run(tc.name, func(t *testing.T) { + var hits atomic.Int32 + h := func(ss grpc.ServerStream, _ *emptypb.Empty) error { + hits.Add(1) + ss.SetTrailer(metadata.Pairs("attempt", "final")) + return status.Error(tc.code, tc.message) + } + conn, _, dir := historicalRig(t, h, h) + dir.SetFailoverPolicy(tc.max, time.Second) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if tc.header { + ctx = metadata.AppendToOutgoingContext(ctx, HeightHeader, "105504992") + } + var trailer metadata.MD + err := conn.Invoke(ctx, tc.method, &emptypb.Empty{}, &emptypb.Empty{}, grpc.Trailer(&trailer)) + if status.Code(err) != tc.code || status.Convert(err).Message() != tc.message || int(hits.Load()) != tc.wantHits { + t.Fatalf("error=%v hits=%d", err, hits.Load()) + } + if strings.Join(trailer.Get("attempt"), "") != "final" { + t.Fatalf("trailers=%v", trailer) + } + }) + } +} + +func TestGRPCHistoricalNeverRetriesAfterResponse(t *testing.T) { + var fallback atomic.Int32 + conn, _, _ := historicalRig(t, func(ss grpc.ServerStream, _ *emptypb.Empty) error { + if err := ss.SendMsg(&emptypb.Empty{}); err != nil { + return err + } + return status.Error(codes.Unknown, historicalMissing) + }, func(ss grpc.ServerStream, _ *emptypb.Empty) error { + fallback.Add(1) + return ss.SendMsg(&emptypb.Empty{}) + }) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + ctx = metadata.AppendToOutgoingContext(ctx, HeightHeader, "105504992") + s, err := conn.NewStream(ctx, &grpc.StreamDesc{ServerStreams: true}, historicalMethod) + if err != nil { + t.Fatal(err) + } + if err = s.SendMsg(&emptypb.Empty{}); err != nil { + t.Fatal(err) + } + if err = s.CloseSend(); err != nil { + t.Fatal(err) + } + if err = s.RecvMsg(&emptypb.Empty{}); err != nil { + t.Fatal(err) + } + if err = s.RecvMsg(&emptypb.Empty{}); status.Code(err) != codes.Unknown { + t.Fatalf("got %v", err) + } + if fallback.Load() != 0 { + t.Fatal("partial response retried") + } +} + +func TestGRPCHistoricalBodyHeightFallback(t *testing.T) { + var first atomic.Int32 + conn, _, _ := historicalRig(t, func(_ grpc.ServerStream, _ *emptypb.Empty) error { + first.Add(1) + return status.Error(codes.Unknown, historicalMissing) + }, func(ss grpc.ServerStream, _ *emptypb.Empty) error { + md, _ := metadata.FromIncomingContext(ss.Context()) + if strings.Join(md.Get(HeightHeader), "") != "105504992" { + t.Error("body height not forwarded") + } + return ss.SendMsg(&emptypb.Empty{}) + }) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + req := &emptypb.Empty{} + req.ProtoReflect().SetUnknown(heightPayload(1, 105504992)) + if err := conn.Invoke(ctx, "/cosmos.base.tendermint.v1beta1.Service/GetBlockByHeight", req, &emptypb.Empty{}); err != nil && err != io.EOF { + t.Fatal(err) + } + if first.Load() != 1 { + t.Fatalf("first hits=%d", first.Load()) + } +} diff --git a/internal/server/cosmos_grpc/server.go b/internal/server/cosmos_grpc/server.go index 6d4e95c..821ec04 100644 --- a/internal/server/cosmos_grpc/server.go +++ b/internal/server/cosmos_grpc/server.go @@ -9,8 +9,9 @@ // failover-by-redial). // - Reuses pooled *grpc.ClientConn per backend with idle eviction. // +// WebHandler optionally exposes the same proxy over gRPC-Web. +// // What is deferred: -// - gRPC-Web wrapping (small wrapper; phase 2b). // - Streaming-RPC failover after partial response (the subscription hub // in phase 5 covers this for ChainStream specifically). package cosmos_grpc diff --git a/internal/server/cosmos_grpc/web.go b/internal/server/cosmos_grpc/web.go new file mode 100644 index 0000000..6781f0c --- /dev/null +++ b/internal/server/cosmos_grpc/web.go @@ -0,0 +1,134 @@ +package cosmos_grpc + +import ( + "mime" + "net/http" + "strings" + + "github.com/rs/cors" + "github.com/traefik/grpc-web/go/grpcweb" +) + +// WebOptions applies only to the optional HTTP gRPC-Web handler. It does not +// change the native gRPC listener or the REST listener. +type WebOptions struct { + // AllowedOrigins contains exact browser origins. An explicit "*" permits + // any origin; an empty list denies requests carrying an Origin header. + // Requests without Origin remain usable by non-browser clients. + AllowedOrigins []string + // AllowedRequestHeaders extends the standard gRPC-Web/Cosmos/auth headers. + AllowedRequestHeaders []string + // MaxRequestBytes bounds the encoded HTTP body, including base64 overhead. + // Zero uses 90 MiB; the native gRPC server separately caps messages at 64 MiB. + MaxRequestBytes int64 +} + +const defaultWebMaxRequestBytes int64 = 90 << 20 + +// WebHandler exposes the same routing, metadata and protobuf handling as the +// native listener. The upstream library owns binary/text framing and status +// trailers; native gRPC over HTTP/2 is served directly, and ordinary HTTP +// requests are sent to fallback (normally Cosmos REST). +// +// Mount this on a separate optional HTTP listener with HTTP/2 (TLS or h2c) +// enabled for native clients. Only unary and server-streaming browser RPCs are +// supported; the nonstandard gRPC-Web WebSocket transport is deliberately +// disabled. Closing a request cancels its native proxy stream. +func (s *Server) WebHandler(opts WebOptions, fallback http.Handler) http.Handler { + if fallback == nil { + fallback = http.NotFoundHandler() + } + allowed := make(map[string]bool, len(opts.AllowedOrigins)) + for _, origin := range opts.AllowedOrigins { + allowed[origin] = true + } + allowedOrigin := func(origin string) bool { return allowed["*"] || allowed[origin] } + headers := []string{ + "Content-Type", "X-Grpc-Web", "X-User-Agent", "Grpc-Timeout", + "Grpc-Encoding", "Grpc-Accept-Encoding", "Authorization", "X-API-Key", + "X-Cosmos-Block-Height", "Accept", "Accept-Language", + } + headers = append(headers, opts.AllowedRequestHeaders...) + corsHandler := cors.New(cors.Options{ + AllowOriginFunc: allowedOrigin, + AllowedMethods: []string{http.MethodGet, http.MethodHead, http.MethodPost, http.MethodOptions}, + AllowedHeaders: headers, + ExposedHeaders: []string{"Grpc-Status", "Grpc-Message", "Grpc-Status-Details-Bin", HeightHeader, "X-Request-ID"}, + MaxAge: 600, + }) + wrapped := grpcweb.WrapServer(s.srv, grpcweb.WithWebsockets(false)) + maxBytes := opts.MaxRequestBytes + if maxBytes <= 0 { + maxBytes = defaultWebMaxRequestBytes + } + route := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mediaType, _, err := mime.ParseMediaType(r.Header.Get("Content-Type")) + if mediaType == "application/grpc" || mediaType == "application/grpc+proto" { + if err != nil { + http.Error(w, "invalid gRPC content type", http.StatusUnsupportedMediaType) + return + } + if r.ProtoMajor != 2 { + http.Error(w, "native gRPC requires HTTP/2", http.StatusHTTPVersionNotSupported) + return + } + // Preserve the native transport's content type, metadata, trailers, + // message limits and streaming behavior without web conversion. + s.srv.ServeHTTP(w, r) + return + } + if !strings.HasPrefix(mediaType, "application/grpc-web") { + fallback.ServeHTTP(w, r) + return + } + if r.Method != http.MethodPost { + w.Header().Set("Allow", http.MethodPost) + http.Error(w, "gRPC-Web requires POST", http.StatusMethodNotAllowed) + return + } + switch mediaType { + case "application/grpc-web", "application/grpc-web+proto", "application/grpc-web-text", "application/grpc-web-text+proto": + // The transparent backend uses protobuf; do not pass other codecs + // or prefix matches such as application/grpc-web-invalid to it. + default: + http.Error(w, "unsupported gRPC-Web content type", http.StatusUnsupportedMediaType) + return + } + if err != nil { + http.Error(w, "invalid gRPC-Web content type", http.StatusUnsupportedMediaType) + return + } + if r.ContentLength > maxBytes { + http.Error(w, "gRPC-Web request too large", http.StatusRequestEntityTooLarge) + return + } + // The protocol adapter rewrites request headers for grpc.Server. Clone + // first so middleware and the caller retain the original HTTP request. + r = r.Clone(r.Context()) + // HTTP/1 connection headers are invalid HTTP/2 metadata. Without + // removing them, a transparent upstream rejects even a normal + // Connection: close request with RST_STREAM(PROTOCOL_ERROR). + for _, value := range r.Header.Values("Connection") { + for _, name := range strings.Split(value, ",") { + r.Header.Del(strings.TrimSpace(name)) + } + } + for _, name := range []string{"Connection", "Proxy-Connection", "Keep-Alive", "Transfer-Encoding", "Upgrade", "TE", "Trailer"} { + r.Header.Del(name) + } + r.Header.Set("Content-Type", mediaType) + r.Body = http.MaxBytesReader(w, r.Body, maxBytes) + // CORS is handled once outside the adapter so REST fallback and + // preflights without x-grpc-web use the same explicit origin policy. + wrapped.HandleGrpcWebRequest(w, r) + }) + withCORS := corsHandler.Handler(route) + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if origin := r.Header.Get("Origin"); origin != "" && !allowedOrigin(origin) { + w.Header().Add("Vary", "Origin") + http.Error(w, "origin is not allowed", http.StatusForbidden) + return + } + withCORS.ServeHTTP(w, r) + }) +} diff --git a/internal/server/cosmos_grpc/web_test.go b/internal/server/cosmos_grpc/web_test.go new file mode 100644 index 0000000..dd91414 --- /dev/null +++ b/internal/server/cosmos_grpc/web_test.go @@ -0,0 +1,635 @@ +package cosmos_grpc + +import ( + "bufio" + "bytes" + "context" + "encoding/base64" + "encoding/binary" + "html" + "io" + "math" + "net" + "net/http" + "net/http/httptest" + "net/textproto" + "strconv" + "strings" + "sync/atomic" + "testing" + "time" + + "golang.org/x/net/http2" + "golang.org/x/net/http2/h2c" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/emptypb" + + "github.com/InjectiveLabs/stitch/internal/backend" + "github.com/InjectiveLabs/stitch/internal/circuit" + healthreg "github.com/InjectiveLabs/stitch/internal/health" + "github.com/InjectiveLabs/stitch/internal/pool" + "github.com/InjectiveLabs/stitch/internal/selector" + "github.com/InjectiveLabs/stitch/internal/types" +) + +const webBlockMethod = "/cosmos.base.tendermint.v1beta1.Service/GetBlockByHeight" + +func webFrame(payload []byte) []byte { + size := uint64(len(payload)) + if size > math.MaxUint32 { + panic("test payload exceeds gRPC frame size") + } + out := make([]byte, 5, 5+len(payload)) + binary.BigEndian.PutUint32(out[1:5], uint32(size)) + return append(out, payload...) +} + +func webRequest(t *testing.T, url, format string, payload []byte) *http.Request { + t.Helper() + body := webFrame(payload) + if strings.Contains(format, "text") { + body = []byte(base64.StdEncoding.EncodeToString(body)) + } + req, err := http.NewRequest(http.MethodPost, url, bytes.NewReader(body)) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Content-Type", format) + req.Header.Set("X-Grpc-Web", "1") + return req +} + +// Text gRPC-Web can flush independently padded base64 chunks. Decode in four-byte +// units to inspect both unary responses and flushed streams without assuming the +// whole body is a single base64 entity. +type webTextReader struct { + in io.Reader + pending []byte +} + +func (r *webTextReader) Read(dst []byte) (int, error) { + if len(dst) == 0 { + return 0, nil + } + if len(r.pending) == 0 { + var encoded [4]byte + if _, err := io.ReadFull(r.in, encoded[:]); err != nil { + return 0, err + } + decoded, err := base64.StdEncoding.DecodeString(string(encoded[:])) + if err != nil { + return 0, err + } + r.pending = decoded + } + n := copy(dst, r.pending) + r.pending = r.pending[n:] + return n, nil +} + +func webReader(resp *http.Response) io.Reader { + if strings.Contains(resp.Header.Get("Content-Type"), "text") { + return &webTextReader{in: resp.Body} + } + return resp.Body +} + +func readWebFrame(t *testing.T, in io.Reader) (byte, []byte) { + t.Helper() + var header [5]byte + if _, err := io.ReadFull(in, header[:]); err != nil { + t.Fatal(err) + } + size := binary.BigEndian.Uint32(header[1:5]) + if size > 1<<20 { + t.Fatalf("unexpected response frame size %d", size) + } + payload := make([]byte, size) + if _, err := io.ReadFull(in, payload); err != nil { + t.Fatal(err) + } + return header[0], payload +} + +func readWebResponse(t *testing.T, resp *http.Response) ([][]byte, http.Header) { + t.Helper() + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + t.Fatalf("HTTP %d: %s", resp.StatusCode, body) + } + in := webReader(resp) + var messages [][]byte + trailers := resp.Header.Clone() // Trailers-only errors can live in headers. + for { + var header [5]byte + if _, err := io.ReadFull(in, header[:]); err == io.EOF { + break + } else if err != nil { + t.Fatal(err) + } + size := binary.BigEndian.Uint32(header[1:5]) + if size > 1<<20 { + t.Fatalf("unexpected response frame size %d", size) + } + body := make([]byte, size) + if _, err := io.ReadFull(in, body); err != nil { + t.Fatal(err) + } + if header[0] == 0 { + messages = append(messages, body) + continue + } + if header[0] != 0x80 { + t.Fatalf("unexpected frame flag %x", header[0]) + } + for _, line := range strings.Split(string(body), "\r\n") { + key, _, ok := strings.Cut(line, ":") + if ok && strings.ToLower(key) != key { + t.Fatalf("trailer name must be lowercase: %q", key) + } + } + md, err := textproto.NewReader(bufio.NewReader(strings.NewReader(string(body) + "\r\n"))).ReadMIMEHeader() + if err != nil { + t.Fatal(err) + } + for k, v := range md { + trailers[k] = v + } + rest, err := io.ReadAll(in) + if err != nil || len(rest) != 0 { + t.Fatalf("trailers must be final frame: rest=%x err=%v", rest, err) + } + break + } + return messages, trailers +} + +func TestWebHistoricalRouting(t *testing.T) { + for _, format := range []string{"application/grpc-web+proto", "application/grpc-web-text+proto"} { + for _, tc := range []struct { + name, height, wantHeight string + payload []byte + wantShard bool + }{ + {"metadata", "1234", "1234", nil, true}, + {"body", "", "1234", heightPayload(1, 1234), true}, + {"metadata wins", "90000", "90000", heightPayload(1, 1234), false}, + } { + t.Run(format+"/"+tc.name, func(t *testing.T) { + rig := setupGRPC(t) + defer rig.close() + httpServer := httptest.NewServer(rig.front.WebHandler(WebOptions{}, nil)) + defer httpServer.Close() + req := webRequest(t, httpServer.URL+webBlockMethod, format, tc.payload) + req.Header.Set("Connection", "close, X-Hop-Only") + req.Header.Set("X-Hop-Only", "must-not-forward") + if tc.height != "" { + req.Header.Set(HeightHeader, tc.height) + } + resp, err := httpServer.Client().Do(req) + if err != nil { + t.Fatal(err) + } + messages, trailers := readWebResponse(t, resp) + if len(messages) != 1 || trailers.Get("Grpc-Status") != "0" { + t.Fatalf("messages=%d trailers=%v", len(messages), trailers) + } + chosen, other := rig.archive, rig.shard + if tc.wantShard { + chosen, other = rig.shard, rig.archive + } + if chosen.hits.Load() != 1 || other.hits.Load() != 0 { + t.Fatalf("routing: chosen=%d other=%d", chosen.hits.Load(), other.hits.Load()) + } + if chosen.heightHeader.Load() != tc.wantHeight || !bytes.Equal(chosen.body.Load().([]byte), tc.payload) { + t.Fatalf("request changed: height=%v body=%x", chosen.heightHeader.Load(), chosen.body.Load()) + } + // The same grpc.Server must keep its native transport usable. + if err := callHealth(t, rig.frontConn, "1234"); err != nil { + t.Fatal(err) + } + }) + } + } +} + +func newWebBackend(t *testing.T, opts WebOptions, fallback http.Handler, handler func(grpc.ServerStream) error) *httptest.Server { + t.Helper() + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + upstream := grpc.NewServer(grpc.UnknownServiceHandler(func(_ any, stream grpc.ServerStream) error { + var request emptypb.Empty + if err := stream.RecvMsg(&request); err != nil { + return err + } + return handler(stream) + })) + go func() { _ = upstream.Serve(listener) }() + t.Cleanup(upstream.Stop) + reg := backend.NewRegistry([]*backend.Backend{{Name: "archive", Weight: 100, + Coverage: backend.Coverage{Kind: backend.CovArchive}, + Endpoints: map[types.Protocol]string{types.ProtoGRPC: listener.Addr().String()}, + }}) + h := healthreg.NewRegistry() + h.Update(healthreg.Snapshot{Backend: "archive", Protocol: types.ProtoRPC, Healthy: true, LatestHeight: 200000000}) + cm := circuit.NewManager(circuit.Policy{ErrorThreshold: 0.5, MinRequests: 10, OpenDuration: time.Minute}) + connections := pool.NewGRPCPool(time.Minute) + t.Cleanup(connections.CloseAll) + front, err := New("127.0.0.1:0", NewDirector(selector.NewRangeSelector(reg, h, cm, 0), cm, connections)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = front.lis.Close(); front.srv.Stop() }) + httpServer := httptest.NewServer(front.WebHandler(opts, fallback)) + httpServer.Client().Timeout = 4 * time.Second + t.Cleanup(httpServer.Close) + return httpServer +} + +func TestWebMetadataTrailersAndErrors(t *testing.T) { + for _, format := range []string{"application/grpc-web", "application/grpc-web-text"} { + t.Run(format, func(t *testing.T) { + srv := newWebBackend(t, WebOptions{AllowedOrigins: []string{"https://client.example"}}, nil, func(stream grpc.ServerStream) error { + md, _ := metadata.FromIncomingContext(stream.Context()) + for _, name := range []string{"connection", "x-hop-only", "keep-alive", "proxy-connection", "upgrade"} { + if len(md.Get(name)) != 0 { + return status.Error(codes.InvalidArgument, "HTTP hop metadata leaked upstream") + } + } + if md.Get("x-api-key")[0] != "test-key" || md.Get(HeightHeader)[0] != "1234" { + return status.Error(codes.InvalidArgument, "metadata changed") + } + if err := stream.SendHeader(metadata.Pairs(HeightHeader, "1234")); err != nil { + return err + } + if err := stream.SendMsg(&emptypb.Empty{}); err != nil { + return err + } + stream.SetTrailer(metadata.Pairs("x-test-trailer", "kept")) + return status.Error(codes.NotFound, "missing test state") + }) + req := webRequest(t, srv.URL+webBlockMethod, format, nil) + req.Header.Set("Origin", "https://client.example") + req.Header.Set("X-API-Key", "test-key") + req.Header.Set("Connection", "close, X-Hop-Only") + req.Header.Set("X-Hop-Only", "must-not-forward") + req.Header.Set("Keep-Alive", "timeout=5") + req.Header.Set("Proxy-Connection", "keep-alive") + req.Header.Set("Upgrade", "unwanted") + req.Header.Set(HeightHeader, "1234") + resp, err := srv.Client().Do(req) + if err != nil { + t.Fatal(err) + } + messages, trailers := readWebResponse(t, resp) + if len(messages) != 1 || trailers.Get("Grpc-Status") != "5" || trailers.Get("Grpc-Message") != "missing test state" || trailers.Get("X-Test-Trailer") != "kept" { + t.Fatalf("messages=%d trailers=%v", len(messages), trailers) + } + if resp.Header.Get(HeightHeader) != "1234" || resp.Header.Get("Access-Control-Allow-Origin") != "https://client.example" { + t.Fatalf("response headers=%v", resp.Header) + } + }) + } +} + +func TestWebCORSAndRESTFallback(t *testing.T) { + var hits atomic.Int64 + fallback := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits.Add(1) + w.Header().Set("Content-Type", "text/plain") + w.Header().Set(HeightHeader, r.Header.Get(HeightHeader)) + _, _ = io.WriteString(w, html.EscapeString(r.Method+" "+r.URL.RequestURI())) + }) + srv := newWebBackend(t, WebOptions{AllowedOrigins: []string{"https://client.example"}}, fallback, func(stream grpc.ServerStream) error { + return stream.SendMsg(&emptypb.Empty{}) + }) + for _, requested := range []string{"content-type,x-api-key,x-cosmos-block-height,x-grpc-web", "content-type,x-cosmos-block-height"} { + req, _ := http.NewRequest(http.MethodOptions, srv.URL+webBlockMethod, nil) + req.Header.Set("Origin", "https://client.example") + req.Header.Set("Access-Control-Request-Method", "POST") + req.Header.Set("Access-Control-Request-Headers", requested) + resp, err := srv.Client().Do(req) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + if resp.StatusCode != http.StatusNoContent || resp.Header.Get("Access-Control-Allow-Origin") != "https://client.example" || resp.Header.Get("Access-Control-Allow-Headers") == "" { + t.Fatalf("preflight: %d %v", resp.StatusCode, resp.Header) + } + } + req, _ := http.NewRequest(http.MethodGet, srv.URL+"/cosmos/bank/v1beta1/supply?height=1234", nil) + req.Header.Set("Origin", "https://client.example") + req.Header.Set(HeightHeader, "1234") + resp, err := srv.Client().Do(req) + if err != nil { + t.Fatal(err) + } + body, _ := io.ReadAll(resp.Body) + resp.Body.Close() + if string(body) != "GET /cosmos/bank/v1beta1/supply?height=1234" || resp.Header.Get(HeightHeader) != "1234" || resp.Header.Get("Access-Control-Allow-Origin") != "https://client.example" { + t.Fatalf("REST fallback changed: %s %v", body, resp.Header) + } + for _, method := range []string{http.MethodGet, http.MethodPost, http.MethodOptions} { + req, _ := http.NewRequest(method, srv.URL+webBlockMethod, nil) + req.Header.Set("Origin", "https://other.example") + resp, err := srv.Client().Do(req) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + if resp.StatusCode != http.StatusForbidden || resp.Header.Get("Access-Control-Allow-Origin") != "" { + t.Fatalf("denied origin: %d %v", resp.StatusCode, resp.Header) + } + } + if hits.Load() != 1 { + t.Fatalf("preflight or rejected origin reached fallback: %d", hits.Load()) + } +} + +func TestWebStreamsAndCancellation(t *testing.T) { + for _, format := range []string{"application/grpc-web+proto", "application/grpc-web-text+proto"} { + t.Run(format, func(t *testing.T) { + canceled := make(chan struct{}) + srv := newWebBackend(t, WebOptions{}, nil, func(stream grpc.ServerStream) error { + if err := stream.SendMsg(&emptypb.Empty{}); err != nil { + return err + } + <-stream.Context().Done() + close(canceled) + return stream.Context().Err() + }) + req := webRequest(t, srv.URL+"/test.Stream/Watch", format, nil) + ctx, cancel := context.WithCancel(req.Context()) + defer cancel() + resp, err := srv.Client().Do(req.WithContext(ctx)) + if err != nil { + t.Fatal(err) + } + // This must arrive while the upstream is still open, not after it ends. + flag, payload := readWebFrame(t, webReader(resp)) + if flag != 0 || len(payload) != 0 { + t.Fatalf("first frame: %x %x", flag, payload) + } + cancel() + resp.Body.Close() + select { + case <-canceled: + case <-time.After(3 * time.Second): + t.Fatal("browser cancellation did not reach upstream") + } + }) + } +} + +func TestWebLimitsAndContentTypes(t *testing.T) { + var hits atomic.Int64 + srv := newWebBackend(t, WebOptions{MaxRequestBytes: 16}, nil, func(stream grpc.ServerStream) error { + hits.Add(1) + return stream.SendMsg(&emptypb.Empty{}) + }) + for _, tc := range []struct { + name, format, method string + payload []byte + want int + }{ + {"body limit", "application/grpc-web+proto", http.MethodPost, make([]byte, 20), http.StatusRequestEntityTooLarge}, + {"text body limit", "application/grpc-web-text+proto", http.MethodPost, make([]byte, 10), http.StatusRequestEntityTooLarge}, + {"unsupported codec", "application/grpc-web+json", http.MethodPost, nil, http.StatusUnsupportedMediaType}, + {"invalid suffix", "application/grpc-web-invalid", http.MethodPost, nil, http.StatusUnsupportedMediaType}, + {"wrong method", "application/grpc-web+proto", http.MethodGet, nil, http.StatusMethodNotAllowed}, + } { + t.Run(tc.name, func(t *testing.T) { + req := webRequest(t, srv.URL+webBlockMethod, tc.format, tc.payload) + req.Method = tc.method + resp, err := srv.Client().Do(req) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + if resp.StatusCode != tc.want { + t.Fatalf("status %d; want %d", resp.StatusCode, tc.want) + } + }) + } + if hits.Load() != 0 { + t.Fatalf("rejected request reached upstream: %d", hits.Load()) + } +} + +func TestWebRejectsMalformedAndChunkedOversizeBodies(t *testing.T) { + var hits atomic.Int64 + srv := newWebBackend(t, WebOptions{MaxRequestBytes: 16}, nil, func(stream grpc.ServerStream) error { + hits.Add(1) + return stream.SendMsg(&emptypb.Empty{}) + }) + for _, tc := range []struct { + name, format string + body []byte + }{ + {"chunked binary limit", "application/grpc-web+proto", webFrame(make([]byte, 20))}, + {"chunked text limit", "application/grpc-web-text+proto", []byte(base64.StdEncoding.EncodeToString(webFrame(make([]byte, 20))))}, + {"invalid base64", "application/grpc-web-text+proto", []byte("$$$$")}, + {"truncated frame", "application/grpc-web+proto", []byte{0, 0, 0}}, + {"declared message too large", "application/grpc-web+proto", []byte{0, 4, 0, 0, 1}}, + } { + t.Run(tc.name, func(t *testing.T) { + req, _ := http.NewRequest(http.MethodPost, srv.URL+webBlockMethod, bytes.NewReader(tc.body)) + req.ContentLength = -1 // Force HTTP/1 chunked input: no length to trust. + req.Header.Set("Content-Type", tc.format) + resp, err := srv.Client().Do(req) + if err != nil { + t.Fatal(err) + } + messages, trailers := readWebResponse(t, resp) + if len(messages) != 0 || trailers.Get("Grpc-Status") == "" || trailers.Get("Grpc-Status") == "0" { + t.Fatalf("malformed request accepted: messages=%d trailers=%v", len(messages), trailers) + } + }) + } + if hits.Load() != 0 { + t.Fatalf("malformed request reached upstream: %d", hits.Load()) + } +} + +func TestWebDeadlineAndTrailersOnly(t *testing.T) { + for _, tc := range []struct{ name, timeout, wantStatus string }{ + {"deadline", "25m", "4"}, + {"trailers only", "", "7"}, + } { + t.Run(tc.name, func(t *testing.T) { + finished := make(chan struct{}) + srv := newWebBackend(t, WebOptions{}, nil, func(stream grpc.ServerStream) error { + defer close(finished) + if tc.timeout == "" { + return status.Error(codes.PermissionDenied, "denied test request") + } + <-stream.Context().Done() + return stream.Context().Err() + }) + req := webRequest(t, srv.URL+webBlockMethod, "application/grpc-web-text+proto", nil) + if tc.timeout != "" { + req.Header.Set("Grpc-Timeout", tc.timeout) + } + resp, err := srv.Client().Do(req) + if err != nil { + t.Fatal(err) + } + messages, trailers := readWebResponse(t, resp) + if len(messages) != 0 || trailers.Get("Grpc-Status") != tc.wantStatus { + t.Fatalf("messages=%d trailers=%v", len(messages), trailers) + } + select { + case <-finished: + case <-time.After(3 * time.Second): + t.Fatal("upstream still running after request ended") + } + }) + } +} + +func TestWebOriginDefaultsAndCustomHeaders(t *testing.T) { + for _, tc := range []struct { + name string + origins []string + wantStatus int + }{ + {"default deny", nil, http.StatusForbidden}, + {"explicit wildcard", []string{"*"}, http.StatusNoContent}, + {"explicit origin", []string{"https://client.example"}, http.StatusNoContent}, + } { + t.Run(tc.name, func(t *testing.T) { + srv := newWebBackend(t, WebOptions{AllowedOrigins: tc.origins, AllowedRequestHeaders: []string{"X-Custom-Client"}}, nil, func(_ grpc.ServerStream) error { + t.Error("preflight reached upstream") + return nil + }) + for _, requestHeaders := range []string{"content-type,x-custom-client,x-grpc-web", "content-type,x-disallowed"} { + req, _ := http.NewRequest(http.MethodOptions, srv.URL+webBlockMethod, nil) + req.Header.Set("Origin", "https://client.example") + req.Header.Set("Access-Control-Request-Method", "POST") + req.Header.Set("Access-Control-Request-Headers", requestHeaders) + resp, err := srv.Client().Do(req) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + if resp.StatusCode != tc.wantStatus { + t.Fatalf("status %d; want %d", resp.StatusCode, tc.wantStatus) + } + wantCORS := tc.wantStatus == http.StatusNoContent && !strings.Contains(requestHeaders, "disallowed") + if got := resp.Header.Get("Access-Control-Allow-Origin") != ""; got != wantCORS { + t.Fatalf("preflight headers=%v; permitted=%v", resp.Header, wantCORS) + } + } + }) + } +} + +func TestWebListenerAlsoServesNativeGRPC(t *testing.T) { + for _, subtype := range []string{"", "proto"} { + for _, byBody := range []bool{false, true} { + t.Run("subtype="+subtype+"/body="+strconv.FormatBool(byBody), func(t *testing.T) { + rig := setupGRPC(t) + defer rig.close() + var fallbackHits atomic.Int64 + fallback := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + fallbackHits.Add(1) + _, _ = io.WriteString(w, "rest fallback") + }) + handler := rig.front.WebHandler(WebOptions{AllowedOrigins: []string{"https://client.example"}}, fallback) + srv := httptest.NewServer(h2c.NewHandler(handler, &http2.Server{})) + defer srv.Close() + conn, err := grpc.NewClient(strings.TrimPrefix(srv.URL, "http://"), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + request := &emptypb.Empty{} + var payload []byte + if byBody { + payload = heightPayload(1, 1234) + request.ProtoReflect().SetUnknown(payload) + } else { + ctx = metadata.AppendToOutgoingContext(ctx, HeightHeader, "1234") + } + var headers metadata.MD + opts := []grpc.CallOption{grpc.Header(&headers)} + if subtype != "" { + opts = append(opts, grpc.CallContentSubtype(subtype)) + } + if err := conn.Invoke(ctx, webBlockMethod, request, &emptypb.Empty{}, opts...); err != nil { + t.Fatal(err) + } + wantType := "application/grpc" + if subtype != "" { + wantType += "+" + subtype + } + if got := headers.Get("content-type"); len(got) != 1 || got[0] != wantType { + t.Fatalf("native content type changed: %v; want %q", got, wantType) + } + if rig.shard.hits.Load() != 1 || rig.archive.hits.Load() != 0 || rig.shard.heightHeader.Load() != "1234" || !bytes.Equal(rig.shard.body.Load().([]byte), payload) { + t.Fatalf("native historical route changed: shard=%d archive=%d height=%v body=%x", rig.shard.hits.Load(), rig.archive.hits.Load(), rig.shard.heightHeader.Load(), rig.shard.body.Load()) + } + // The browser formats and REST remain available on this listener. + for _, format := range []string{"application/grpc-web+proto", "application/grpc-web-text+proto"} { + req := webRequest(t, srv.URL+webBlockMethod, format, heightPayload(1, 1234)) + resp, err := srv.Client().Do(req) + if err != nil { + t.Fatal(err) + } + messages, trailers := readWebResponse(t, resp) + if len(messages) != 1 || trailers.Get("Grpc-Status") != "0" { + t.Fatalf("web failed alongside native: %v", trailers) + } + } + resp, err := srv.Client().Get(srv.URL + "/cosmos/bank/v1beta1/supply") + if err != nil { + t.Fatal(err) + } + body, _ := io.ReadAll(resp.Body) + resp.Body.Close() + if string(body) != "rest fallback" || fallbackHits.Load() != 1 { + t.Fatalf("unexpected REST fallback: %q hits=%d", body, fallbackHits.Load()) + } + // Native dispatch must not bypass the shared origin restriction. + deniedCtx := metadata.AppendToOutgoingContext(ctx, "origin", "https://other.example") + err = conn.Invoke(deniedCtx, webBlockMethod, request, &emptypb.Empty{}) + if status.Code(err) != codes.PermissionDenied { + t.Fatalf("native disallowed origin: %v", err) + } + if rig.shard.hits.Load() != 3 { + t.Fatalf("denied native origin reached upstream: %d", rig.shard.hits.Load()) + } + }) + } + } +} + +func TestWebListenerRejectsNativeGRPCOverHTTP1(t *testing.T) { + srv := newWebBackend(t, WebOptions{}, http.HandlerFunc(func(http.ResponseWriter, *http.Request) { + t.Error("native request reached REST fallback") + }), func(grpc.ServerStream) error { + t.Error("HTTP/1 native request reached backend") + return nil + }) + for _, contentType := range []string{"application/grpc", "application/grpc+proto"} { + req := webRequest(t, srv.URL+webBlockMethod, contentType, nil) + resp, err := srv.Client().Do(req) + if err != nil { + t.Fatal(err) + } + body, _ := io.ReadAll(resp.Body) + resp.Body.Close() + if resp.StatusCode != http.StatusHTTPVersionNotSupported || !strings.Contains(string(body), "requires HTTP/2") { + t.Fatalf("native HTTP/1 response: status=%d body=%q", resp.StatusCode, body) + } + } +} diff --git a/internal/server/http.go b/internal/server/http.go new file mode 100644 index 0000000..fa56f1c --- /dev/null +++ b/internal/server/http.go @@ -0,0 +1,31 @@ +package server + +import ( + "context" + "errors" + "net/http" + "time" + + "golang.org/x/net/http2" + "golang.org/x/net/http2/h2c" +) + +// HTTP wraps an HTTP handler in the shared server lifecycle. +type HTTP struct { + name string + server *http.Server +} + +func NewHTTP(name, addr string, handler http.Handler) *HTTP { + return &HTTP{name: name, server: &http.Server{Addr: addr, Handler: h2c.NewHandler(handler, &http2.Server{}), ReadHeaderTimeout: 10 * time.Second}} +} + +func (s *HTTP) Name() string { return s.name } +func (s *HTTP) Start(context.Context) error { + err := s.server.ListenAndServe() + if errors.Is(err, http.ErrServerClosed) { + return nil + } + return err +} +func (s *HTTP) Shutdown(ctx context.Context) error { return s.server.Shutdown(ctx) } diff --git a/tools/generate-history-methods.py b/tools/generate-history-methods.py new file mode 100644 index 0000000..98e6603 --- /dev/null +++ b/tools/generate-history-methods.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +"""Generate exact historical unary query methods from pinned chain schemas. + +Requires an authenticated gh CLI for the Injective source repositories. Runtime +requests never use reflection or perform source discovery. Run from repo root: + python3 tools/generate-history-methods.py +""" +import concurrent.futures +import re +import subprocess +from pathlib import Path + +# Dependency revisions come from this chain revision's go.mod; module selection +# comes from injective-chain/app/app.go at the same revision. +CORE = "2e3b4f9a9ef3d2c5c0758dd2e3a2b23c35458190" +COSMOS = "3361beeef240618d6dd70ca35259880bb73e7cda" # v0.50.14-inj.11 +IBC = "dbee2d797b8a34179ece0082eebe96bfe023de1e" # v8.7.0-inj.4 +WASM = "e0e9de04a62d8405a85abd6e8ef3fdad9dd5461e" # v0.53.3-inj.3 + +COSMOS_MODULES = ["auth/v1beta1", "authz/v1beta1", "bank/v1beta1", "consensus/v1", + "distribution/v1beta1", "evidence/v1beta1", "feegrant/v1beta1", "gov/v1", "gov/v1beta1", + "mint/v1beta1", "params/v1beta1", "slashing/v1beta1", "staking/v1beta1", "upgrade/v1beta1"] +INJECTIVE_MODULES = ["auction/v1beta1", "downtimedetector/v1beta1", "erc20/v1beta1", "evm/v1", + "exchange/v1beta1", "exchange/v2", "insurance/v1beta1", "oracle/v1beta1", "peggy/v1", + "permissions/v1beta1", "tokenfactory/v1beta1", "txfees/v1beta1", "wasmx/v1"] +IBC_MODULES = ["applications/fee/v1", "applications/interchain_accounts/host/v1", + "applications/transfer/v1", "core/channel/v1", "core/client/v1", "core/connection/v1"] +# These body-height reads use Service instead of Query. Never admit every +# Service method: cosmos.tx Service also contains BroadcastTx. +SERVICE_READS = { + "/cosmos.base.tendermint.v1beta1.Service/GetBlockByHeight", + "/cosmos.base.tendermint.v1beta1.Service/GetValidatorSetByHeight", + "/cosmos.base.tendermint.v1beta1.Service/ABCIQuery", + "/cosmos.tx.v1beta1.Service/GetBlockWithTxs", +} +SOURCES = ( + [("InjectiveLabs/injective-core", CORE, f"proto/injective/{m}/query.proto") for m in INJECTIVE_MODULES] + + [("InjectiveLabs/cosmos-sdk", COSMOS, f"proto/cosmos/{m}/query.proto") for m in COSMOS_MODULES] + + [("InjectiveLabs/cosmos-sdk", COSMOS, "proto/cosmos/base/tendermint/v1beta1/query.proto"), + ("InjectiveLabs/cosmos-sdk", COSMOS, "proto/cosmos/tx/v1beta1/service.proto")] + + [("InjectiveLabs/ibc-go", IBC, f"proto/ibc/{m}/query.proto") for m in IBC_MODULES] + + [("InjectiveLabs/wasmd", WASM, "proto/cosmwasm/wasm/v1/query.proto")] +) + + +def parse_methods(source): + # Mask strings and comments before balancing service braces. Text inside + # options/comments must never create a fake service or RPC declaration. + source = re.sub(r'//[^\n]*|/\*[\s\S]*?\*/|"(?:\\.|[^"\\])*"', ' ', source) + package = re.search(r'\bpackage\s+([\w.]+)\s*;', source) + if not package: + raise ValueError("missing protobuf package") + methods = [] + for service in re.finditer(r'\bservice\s+(\w+)\s*\{', source): + depth, end = 1, service.end() + while depth and end < len(source): + depth += (source[end] == '{') - (source[end] == '}') + end += 1 + if depth: + raise ValueError("unclosed service body") + for rpc in re.finditer(r'\brpc\s+(\w+)\s*\(\s*(stream\s+)?[\w.]+\s*\)\s*returns\s*\(\s*(stream\s+)?[\w.]+\s*\)', source[service.end():end-1]): + name = f"/{package.group(1)}.{service.group(1)}/{rpc.group(1)}" + if (service.group(1) == "Query" or name in SERVICE_READS) and not (rpc.group(2) or rpc.group(3)): + methods.append(name) + return methods + + +def fetch(item): + repo, revision, path = item + source = subprocess.check_output(["gh", "api", "-H", "Accept: application/vnd.github.raw+json", + f"repos/{repo}/contents/{path}?ref={revision}"], text=True) + methods = parse_methods(source) + if not methods: + raise ValueError(f"no eligible unary reads in {repo}/{path}") + return item, methods + + +def main(): + with concurrent.futures.ThreadPoolExecutor(max_workers=6) as pool: + entries = sorted(pool.map(fetch, SOURCES)) + methods = {method for _, group in entries for method in group} + if not SERVICE_READS <= methods: + raise ValueError("body-height Service schema changed") + lines = ["// Code generated by tools/generate-history-methods.py; DO NOT EDIT.", + "// Source versions match Injective core " + CORE + ".", + "// Cosmos: v0.50.14-inj.11; IBC: v8.7.0-inj.4; Wasmd: v0.53.3-inj.3.", + "// Modules are selected from injective-chain/app/app.go; Query RPCs are", + "// read-only module contracts. Streaming methods, Msg services, unknown", + "// future methods, and off-chain indexer services are excluded.", + "package cosmos_grpc", "", "var historicalUnaryMethods = map[string]struct{}{"] + seen = set() + for (repo, revision, path), group in entries: + lines.append(f"\t// https://github.com/{repo}/blob/{revision}/{path}") + for method in sorted(group): + if method not in seen: + lines.append(f'\t"{method}": {{}},') + seen.add(method) + lines += ["}", "", "// Retry only schema-verified unary reads; names alone cannot establish", + "// whether a future Query method streams or has side effects.", + "func historicalReadMethod(method string) bool {", + "\t_, ok := historicalUnaryMethods[method]", "\treturn ok", "}", ""] + output = Path(__file__).resolve().parents[1] / "internal/server/cosmos_grpc/history_methods.go" + output.write_text("\n".join(lines)) + subprocess.run(["gofmt", "-w", str(output)], check=True) + print(f"Generated {len(methods)} exact unary query methods from {len(entries)} pinned schemas") + + +if __name__ == "__main__": + main() diff --git a/tools/test_generate_history_methods.py b/tools/test_generate_history_methods.py new file mode 100644 index 0000000..40e46c0 --- /dev/null +++ b/tools/test_generate_history_methods.py @@ -0,0 +1,51 @@ +import importlib.util +from pathlib import Path +import unittest + +spec = importlib.util.spec_from_file_location("history_generator", Path(__file__).with_name("generate-history-methods.py")) +generator = importlib.util.module_from_spec(spec) +spec.loader.exec_module(generator) + +class SchemaSelectionTests(unittest.TestCase): + def test_unary_query_only_not_streams_or_writes(self): + proto = ''' + syntax = "proto3"; + package cosmos.review.v1; + service Query { + rpc Balance(Request) returns (Response) {} + rpc Watch(Request) returns (stream Response) {} + rpc Upload(stream Request) returns (Response) {} + rpc Chat(stream Request) returns (stream Response) {} + } + service Msg { rpc Send(Request) returns (Response) {} } + service Stream { rpc Watch(Request) returns (Response) {} } + ''' + self.assertEqual(generator.parse_methods(proto), ["/cosmos.review.v1.Query/Balance"]) + + def test_comments_and_option_strings_cannot_create_methods(self): + proto = '''package cosmos.review.v1; + // service Query { rpc Fake(A) returns (B) {} } + /* service Query { rpc AlsoFake(A) returns (B) {} } */ + service Query { + option description = "} rpc Sneaky(A) returns (B) {"; + rpc Balance(A) returns (B) { option description = "// /* { }"; } + } + ''' + self.assertEqual(generator.parse_methods(proto), ["/cosmos.review.v1.Query/Balance"]) + + def test_service_reads_require_exact_inventory(self): + proto = '''package cosmos.tx.v1beta1; + service Service { + rpc BroadcastTx(A) returns (B) {} + rpc Simulate(A) returns (B) {} + rpc GetBlockWithTxs(A) returns (B) {} + }''' + self.assertEqual(generator.parse_methods(proto), ["/cosmos.tx.v1beta1.Service/GetBlockWithTxs"]) + + def test_even_explicit_service_read_must_remain_unary(self): + proto = '''package cosmos.tx.v1beta1; + service Service { rpc GetBlockWithTxs(A) returns (stream B) {} }''' + self.assertEqual(generator.parse_methods(proto), []) + +if __name__ == "__main__": + unittest.main()