fix Sui fund account with retry

framework/components/blockchain/sui.go

@@ -4,14 +4,17 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
+	"net/http"
 	"path/filepath"
 	"runtime"
 	"strings"
 	"time"
 
 	"net/netip"
 
+	"github.com/avast/retry-go/v4"
 	"github.com/block-vision/sui-go-sdk/models"
 
 	"github.com/go-resty/resty/v2"
@@ -46,24 +49,118 @@ type SuiWalletInfo struct {
 	SuiAddress      string  `toml:"sui_address" json:"suiAddress" comment:"Sui key address"`                     // Sui address is a 0x prefixed hex string
 }
 
+// faucetFundTimeout bounds the total time spent retrying Sui faucet /gas funding.
+// The faucet is served by the same container that just started, and the container
+// readiness gate only waits for the TCP port to listen — not for the faucet's HTTP
+// handler to be ready. So the first /gas request can race the faucet's own readiness
+// and fail with a connection reset. Retry with backoff until the faucet accepts it.
+const faucetFundTimeout = 2 * time.Minute
+
+// faucetRequestTimeout bounds a single faucet /gas HTTP request so one hung attempt
+// does not consume the entire faucetFundTimeout budget, leaving room for retries.
+const faucetRequestTimeout = 10 * time.Second
+
+// faucetFundAttempts is the maximum number of /gas funding attempts before giving up.
+// The faucetFundTimeout context still bounds the wall-clock total.
+const faucetFundAttempts = uint(15)
+
 // funds provided key using local faucet
 // we can't use the best client available - block-vision/sui-go-sdk for that, since some versions have old API and it is hardcoded
 // https://github.com/block-vision/sui-go-sdk/blob/main/sui/faucet_api.go#L16
-func fundAccount(url string, address string) error {
-	r := resty.New().SetBaseURL(url)
+func fundAccount(ctx context.Context, url string, address string) error {
+	// Bound the overall retry. A child of the caller's context wins on the earlier
+	// deadline, so callers can tighten it further.
+	ctx, cancel := context.WithTimeout(ctx, faucetFundTimeout)
+	defer cancel()
+
+	r := resty.New().
+		SetBaseURL(url).
+		SetTimeout(faucetRequestTimeout)
+
 	b := &models.FaucetRequest{
 		FixedAmountRequest: &models.FaucetFixedAmountRequest{
 			Recipient: address,
 		},
 	}
-	resp, err := r.R().SetBody(b).SetHeader("Content-Type", "application/json").Post("/gas")
+	_, err := retry.DoWithData(func() (*resty.Response, error) {
+		resp, perr := r.R().
+			SetContext(ctx).
+			SetBody(b).
+			SetHeader("Content-Type", "application/json").
+			Post("/gas")
+		if perr != nil {
+			return nil, perr
+		}
+		if resp.IsError() {
+			return nil, &faucetStatusError{status: resp.StatusCode(), body: resp.Body()}
+		}
+		return resp, nil
+	},
+		retry.Context(ctx),
+		retry.Attempts(faucetFundAttempts),
+		retry.Delay(time.Second),
+		retry.DelayType(retry.BackOffDelay),
+		retry.RetryIf(isRetryableFaucetErr),
+		retry.LastErrorOnly(true),
+		retry.OnRetry(func(n uint, err error) {
+			framework.L.Warn().Err(err).Uint("attempt", n+1).Uint("attempts", faucetFundAttempts).
+				Str("recipient", address).Msg("Retrying Sui faucet /gas funding")
+		}),
+	)
 	if err != nil {
-		return err
+		return fmt.Errorf("fund account via Sui faucet: %w", err)
 	}
-	framework.L.Info().Any("Resp", resp).Msg("Address is funded!")
+	framework.L.Info().Str("recipient", address).Msg("Address is funded!")
 	return nil
 }
 
+// faucetStatusError carries the HTTP status and body of a non-2xx faucet response so
+// isRetryableFaucetErr can classify it without re-issuing the request.
+type faucetStatusError struct {
+	status int
+	body   []byte
+}
+
+func (e *faucetStatusError) Error() string {
+	if trimmed := bytes.TrimSpace(e.body); len(trimmed) > 0 {
+		return fmt.Sprintf("faucet returned status %d: %s", e.status, trimmed)
+	}
+	return fmt.Sprintf("faucet returned status %d", e.status)
+}
+
+// isRetryableFaucetErr classifies faucet /gas errors. Transient failures (faucet still
+// warming up, brief network blips, rate limiting, 5xx) are retried; failures that retrying
+// cannot fix (a malformed request, an already-cancelled context, 4xx other than 429) stop
+// immediately so they don't burn the retry budget and prolong startup/teardown.
+func isRetryableFaucetErr(err error) bool {
+	if err == nil {
+		return false
+	}
+	// A cancelled/deadline-exceeded context means the caller (or the total budget) has
+	// given up; never retry, just propagate.
+	if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
+		return false
+	}
+	var se *faucetStatusError
+	if errors.As(err, &se) {
+		switch {
+		case se.status == http.StatusTooManyRequests:
+			// Rate limited; back off and try again.
+			return true
+		case se.status >= 400 && se.status < 500:
+			// Client-side error (bad recipient, unauthorized, not found, ...).
+			// Retrying an identical request won't fix it.
+			return false
+		case se.status >= 500:
+			// Server-side error / faucet not yet ready.
+			return true
+		}
+	}
+	// Transport-level failures (connection refused/reset, EOF, DNS, timeouts) are
+	// treated as transient while the faucet container comes up.
+	return true
+}
+
 // demuxDockerExecOutput converts Docker exec attach output to plain text when it uses the
 // multiplexed stream format (first byte 1=stdout / 2=stderr). Must run before stripping 0x01,
 // which appears in stream headers and would corrupt the stream if removed globally.
@@ -249,7 +346,7 @@ func newSui(ctx context.Context, in *Input) (*Output, error) {
 	if err != nil {
 		return nil, err
 	}
-	if err := fundAccount(fmt.Sprintf("http://%s:%s", "127.0.0.1", in.FaucetPort), suiAccount.SuiAddress); err != nil {
+	if err := fundAccount(ctx, fmt.Sprintf("http://%s:%s", "127.0.0.1", in.FaucetPort), suiAccount.SuiAddress); err != nil {
 		return nil, err
 	}
 	return &Output{

framework/components/blockchain/sui_faucet_test.go

@@ -0,0 +1,139 @@
+package blockchain
+
+import (
+	"context"
+	"errors"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestIsRetryableFaucetErr(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name string
+		err  error
+		want bool
+	}{
+		{"nil", nil, false},
+		{"context canceled", context.Canceled, false},
+		{"context deadline exceeded", context.DeadlineExceeded, false},
+		{"4xx client error not retryable", &faucetStatusError{status: http.StatusBadRequest}, false},
+		{"404 not retryable", &faucetStatusError{status: http.StatusNotFound}, false},
+		{"401 unauthorized not retryable", &faucetStatusError{status: http.StatusUnauthorized}, false},
+		{"429 rate limited retryable", &faucetStatusError{status: http.StatusTooManyRequests}, true},
+		{"500 server error retryable", &faucetStatusError{status: http.StatusInternalServerError}, true},
+		{"503 service unavailable retryable", &faucetStatusError{status: http.StatusServiceUnavailable}, true},
+		{"bare error retryable (transport-classified)", errors.New("connection reset by peer"), true},
+		{"io EOF retryable (transport-classified)", io.EOF, true},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+			assert.Equal(t, tt.want, isRetryableFaucetErr(tt.err))
+		})
+	}
+}
+
+// faucetStub is a configurable /gas handler that can fail the first N requests with
+// either an HTTP 5xx or a connection reset (hijack+close), then succeed.
+type faucetStub struct {
+	failN      int32 // number of requests to fail
+	resetMode  bool  // true: hijack+close (transport reset); false: 503
+	hits       atomic.Int32
+	successOK  atomic.Bool
+}
+
+func (f *faucetStub) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	n := f.hits.Add(1)
+	if n <= f.failN {
+		if f.resetMode {
+			// Hijack and close the TCP conn mid-response → client sees a
+			// connection reset / EOF (the readiness race we are fixing).
+			if hj, ok := w.(http.Hijacker); ok {
+				if conn, _, err := hj.Hijack(); err == nil {
+					_ = conn.Close()
+					return
+				}
+			}
+			// Fallback if hijack is unavailable: force a transport-level abort.
+			panic("faucetStub: hijack unavailable, cannot simulate connection reset")
+		}
+		w.WriteHeader(http.StatusServiceUnavailable)
+		return
+	}
+	f.successOK.Store(true)
+	w.WriteHeader(http.StatusOK)
+	_, _ = w.Write([]byte(`{"task":"ok"}`))
+}
+
+// shortCtx keeps the retry budget tight so a misbehaving stub can't hang the test.
+// fundAccount wraps this in a 2min child, but the earlier parent deadline wins.
+// BackOffDelay is 1s<<n (2s, 4s, ...) computed after each increment, so a test that
+// fails twice needs ~6s before its third (success) attempt — 20s gives headroom.
+func shortCtx(t *testing.T) context.Context {
+	t.Helper()
+	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+	t.Cleanup(cancel)
+	return ctx
+}
+
+func TestFundAccount_RetriesOn503ThenSucceeds(t *testing.T) {
+	t.Parallel()
+
+	stub := &faucetStub{failN: 2, resetMode: false}
+	srv := httptest.NewServer(stub)
+	t.Cleanup(srv.Close)
+
+	err := fundAccount(shortCtx(t), srv.URL, "0xdeadbeef")
+	require.NoError(t, err)
+	assert.Equal(t, int32(3), stub.hits.Load(), "should fail twice then succeed on third")
+	assert.True(t, stub.successOK.Load(), "success path should have run")
+}
+
+func TestFundAccount_RetriesOnConnectionResetThenSucceeds(t *testing.T) {
+	t.Parallel()
+
+	stub := &faucetStub{failN: 2, resetMode: true}
+	srv := httptest.NewServer(stub)
+	t.Cleanup(srv.Close)
+
+	err := fundAccount(shortCtx(t), srv.URL, "0xdeadbeef")
+	require.NoError(t, err)
+	assert.Equal(t, int32(3), stub.hits.Load(), "should absorb two resets then succeed")
+	assert.True(t, stub.successOK.Load(), "success path should have run")
+}
+
+func TestFundAccount_FastFailsOnNonRetryable4xx(t *testing.T) {
+	t.Parallel()
+
+	// A 4xx must stop immediately instead of burning the retry budget. A 5xx stub would
+	// loop many times; this stub always returns 400 and must be hit exactly once.
+	stub := &badRequestStub{}
+	srv := httptest.NewServer(stub)
+	t.Cleanup(srv.Close)
+
+	err := fundAccount(shortCtx(t), srv.URL, "0xdeadbeef")
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "fund account via Sui faucet")
+	assert.Contains(t, err.Error(), "status 400")
+	assert.Equal(t, int32(1), stub.hits.Load(), "non-retryable 4xx must not be retried")
+}
+
+// badRequestStub always returns 400 to verify non-retryable fast-fail.
+type badRequestStub struct {
+	hits atomic.Int32
+}
+
+func (b *badRequestStub) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	b.hits.Add(1)
+	w.WriteHeader(http.StatusBadRequest)
+	_, _ = w.Write([]byte(`{"error":"bad recipient"}`))
+}