prod-mainnet load test

build/devenv/env-prod-mainnet.toml

@@ -29,6 +29,7 @@ out.chain_id = "98866"
 out.family = "evm"
 out.container_name = "plume-mainnet"
 out.nodes = [
+  { http_url = "https://rpcs.cldev.sh/17912061998839310979", internal_http_url = "https://rpcs.cldev.sh/17912061998839310979", internal_ws_url = "wss://rpcs.cldev.sh/17912061998839310979", ws_url = "wss://rpcs.cldev.sh/17912061998839310979" },
   { http_url = "https://spectrum-cl-02.simplystaking.xyz/Y2hhaW5saW5rLTAyLWVhNDQ2OTRlLWNoYWlubGluaw/TmH0l6uLUmf8WQ/plume/mainnet/", internal_http_url = "https://spectrum-cl-02.simplystaking.xyz/Y2hhaW5saW5rLTAyLWVhNDQ2OTRlLWNoYWlubGluaw/TmH0l6uLUmf8WQ/plume/mainnet/", internal_ws_url = "wss://spectrum-cl-02.simplystaking.xyz/Y2hhaW5saW5rLTAyLWVhNDQ2OTRlLWNoYWlubGluaw/6HDlgzRTAK0xBg/plume/mainnet/", ws_url = "wss://spectrum-cl-02.simplystaking.xyz/Y2hhaW5saW5rLTAyLWVhNDQ2OTRlLWNoYWlubGluaw/6HDlgzRTAK0xBg/plume/mainnet/" },
   { http_url = "https://plume-mainnet-cll.public.linkpool.io/0/3MEVodcpxuMeCmqas7Fh3Fs73oF78EN5", internal_http_url = "https://plume-mainnet-cll.public.linkpool.io/0/3MEVodcpxuMeCmqas7Fh3Fs73oF78EN5", internal_ws_url = "wss://plume-mainnet-cll.public.linkpool.io/0/ws/3MEVodcpxuMeCmqas7Fh3Fs73oF78EN5", ws_url = "wss://plume-mainnet-cll.public.linkpool.io/0/ws/3MEVodcpxuMeCmqas7Fh3Fs73oF78EN5" },
   { http_url = "https://plume-mainnet-cll.public.linkpool.io/1/3MEVodcpxuMeCmqas7Fh3Fs73oF78EN5", internal_http_url = "https://plume-mainnet-cll.public.linkpool.io/1/3MEVodcpxuMeCmqas7Fh3Fs73oF78EN5", internal_ws_url = "wss://plume-mainnet-cll.public.linkpool.io/1/ws/3MEVodcpxuMeCmqas7Fh3Fs73oF78EN5", ws_url = "wss://plume-mainnet-cll.public.linkpool.io/1/ws/3MEVodcpxuMeCmqas7Fh3Fs73oF78EN5" },
@@ -37,7 +38,6 @@ out.nodes = [
   { http_url = "https://mainnet.plume.private.validationcloud.io:8547/?api-key=Lp7FKOSJrY2WQT2VyT8Ytv5R5eaeD0ipMm86hGIfc5YS", internal_http_url = "https://mainnet.plume.private.validationcloud.io:8547/?api-key=Lp7FKOSJrY2WQT2VyT8Ytv5R5eaeD0ipMm86hGIfc5YS", internal_ws_url = "wss://mainnet.plume.private.validationcloud.io:8548/?api-key=Lp7FKOSJrY2WQT2VyT8Ytv5R5eaeD0ipMm86hGIfc5YS", ws_url = "wss://mainnet.plume.private.validationcloud.io:8548/?api-key=Lp7FKOSJrY2WQT2VyT8Ytv5R5eaeD0ipMm86hGIfc5YS" },
   { http_url = "https://plume-mainnet-cll-extra.public.linkpool.io/kanEQFaGZaRGK62qpimiM9cCZhgWMTq6", internal_http_url = "https://plume-mainnet-cll-extra.public.linkpool.io/kanEQFaGZaRGK62qpimiM9cCZhgWMTq6", internal_ws_url = "wss://plume-mainnet-cll-extra.public.linkpool.io/ws/kanEQFaGZaRGK62qpimiM9cCZhgWMTq6", ws_url = "wss://plume-mainnet-cll-extra.public.linkpool.io/ws/kanEQFaGZaRGK62qpimiM9cCZhgWMTq6" },
   { http_url = "https://nd-620-061-642.p2pify.com/7afa04341fed9c432081c2fd953337f9", internal_http_url = "https://nd-620-061-642.p2pify.com/7afa04341fed9c432081c2fd953337f9", internal_ws_url = "wss://ws-nd-620-061-642.p2pify.com/7afa04341fed9c432081c2fd953337f9", ws_url = "wss://ws-nd-620-061-642.p2pify.com/7afa04341fed9c432081c2fd953337f9" },
-  { http_url = "https://rpcs.cldev.sh/17912061998839310979", internal_http_url = "https://rpcs.cldev.sh/17912061998839310979", internal_ws_url = "wss://rpcs.cldev.sh/17912061998839310979", ws_url = "wss://rpcs.cldev.sh/17912061998839310979" },
 ]
 
 [cldf]

build/devenv/evm/event_poller.go

@@ -28,6 +28,7 @@ type pollerResult[T any] struct {
 type eventPoller[T any] struct {
 	ethClient          *ethclient.Client
 	logger             zerolog.Logger
+	chainName          string
 	eventName          string
 	lastScannedBlock   uint64
 	waitersBySeqNum    map[eventKey]chan pollerResult[T]
@@ -43,6 +44,7 @@ type eventPoller[T any] struct {
 func newEventPoller[T any](
 	ethClient *ethclient.Client,
 	logger zerolog.Logger,
+	chainName string,
 	eventName string,
 	pollFn func(start, end uint64) (map[eventKey]T, error),
 ) *eventPoller[T] {
@@ -68,7 +70,8 @@ func (p *eventPoller[T]) registerByMessageID(ctx context.Context, key eventKey)
 		resultCh := make(chan pollerResult[T], 1)
 		p.logger.Debug().
 			Uint64("chainSelector", key.chainSelector).
-			Bytes("messageID", key.messageID[:]).
+			Str("messageID", key.messageID.String()).
+			Str("chain", p.chainName).
 			Str("event", p.eventName).
 			Msg("Cache hit")
 		resultCh <- cachedResult
@@ -179,7 +182,7 @@ func (p *eventPoller[T]) poll() {
 
 	events, err := p.pollFn(lastScanned+1, latestBlock)
 	if err != nil {
-		p.logger.Warn().Err(err).Str("event", p.eventName).Msg("Failed to poll events")
+		p.logger.Warn().Err(err).Str("event", p.eventName).Str("chain", p.chainName).Msg("Failed to poll events")
 		return
 	}
 
@@ -208,7 +211,7 @@ func (p *eventPoller[T]) poll() {
 				delete(p.waitersByMessageID, msgIDKey)
 				p.logger.Info().
 					Uint64("chainSelector", key.chainSelector).
-					Bytes("messageID", key.messageID[:]).
+					Str("messageID", key.messageID.String()).
 					Str("event", p.eventName).
 					Msg("Event received")
 				ch <- result

build/devenv/evm/impl.go

@@ -246,7 +246,7 @@ func (m *CCIP17EVM) getOrCreateOnRampPoller() (*eventPoller[cciptestinterfaces.M
 		return events, nil
 	}
 
-	poller := newEventPoller(ethClient, m.logger, "CCIPMessageSent", pollFn)
+	poller := newEventPoller(ethClient, m.logger, m.chain.Name(), "CCIPMessageSent", pollFn)
 	m.onRampPoller = poller
 	return poller, nil
 }
@@ -294,7 +294,7 @@ func (m *CCIP17EVM) getOrCreateOffRampPoller() (*eventPoller[cciptestinterfaces.
 		return events, nil
 	}
 
-	poller := newEventPoller(ethClient, m.logger, "ExecutionStateChanged", pollFn)
+	poller := newEventPoller(ethClient, m.logger, m.chain.Name(), "ExecutionStateChanged", pollFn)
 	m.offRampPoller = poller
 	return poller, nil
 }
@@ -404,7 +404,7 @@ func (m *CCIP17EVM) ConfirmSendOnSource(ctx context.Context, to uint64, key ccip
 	pollerKey := eventKey{chainSelector: to, msgNum: key.SeqNum, messageID: key.MessageID}
 	var resultCh <-chan pollerResult[cciptestinterfaces.MessageSentEvent]
 	if key.MessageID != (protocol.Bytes32{}) {
-		l.Info().Uint64("from", m.chainDetails.ChainSelector).Uint64("to", to).Bytes("messageID", key.MessageID[:]).Msg("Awaiting CCIPMessageSent event")
+		l.Info().Uint64("from", m.chainDetails.ChainSelector).Uint64("to", to).Str("messageID", key.MessageID.String()).Msg("Awaiting CCIPMessageSent event")
 		resultCh = poller.registerByMessageID(ctx, pollerKey)
 	} else {
 		l.Info().Uint64("from", m.chainDetails.ChainSelector).Uint64("to", to).Uint64("seq", key.SeqNum).Msg("Awaiting CCIPMessageSent event")
@@ -442,7 +442,7 @@ func (m *CCIP17EVM) ConfirmExecOnDest(ctx context.Context, from uint64, key ccip
 	pollerKey := eventKey{chainSelector: from, msgNum: key.SeqNum, messageID: key.MessageID}
 	var resultCh <-chan pollerResult[cciptestinterfaces.ExecutionStateChangedEvent]
 	if key.MessageID != (protocol.Bytes32{}) {
-		l.Info().Uint64("from", from).Bytes("messageID", key.MessageID[:]).Msg("Awaiting ExecutionStateChanged event")
+		l.Info().Uint64("from", from).Str("messageID", key.MessageID.String()).Msg("Awaiting ExecutionStateChanged event")
 		resultCh = poller.registerByMessageID(ctx, pollerKey)
 	} else {
 		l.Info().Uint64("from", from).Uint64("to", m.chainDetails.ChainSelector).Uint64("seq", key.SeqNum).Msg("Awaiting ExecutionStateChanged event")
@@ -599,6 +599,7 @@ func (m *CCIP17EVM) validateTokenBalances(
 	}
 
 	if !haveEnoughFeeTokens {
+		l.Error().Str("chain", srcChain.Name()).Str("sender", srcChain.DeployerKey.From.Hex()).Msg("lol")
 		return nil, fmt.Errorf("not enough tokens to send message, feeToken: %s, fee: %s, msgValue: %s", feeToken.Hex(), fee.String(), msgValue.String())
 	}
 
@@ -1987,6 +1988,7 @@ func (m *CCIP17EVM) SendChainMessage(ctx context.Context, destChain uint64, msg
 
 	tx, err := rout.CcipSend(senderKeyCopy, destChain, message)
 	if err != nil {
+		l.Info().Uint64("dest_chain", destChain).Str("src_chain", srcChain.Name()).Str("router", rout.Address().Hex()).Msg("lol")
 		return cciptestinterfaces.MessageSentEvent{}, protocol.ByteSlice{}, fmt.Errorf("failed to send CCIP message: %w", err)
 	}
 	txHash := tx.Hash()

build/devenv/evm/interface.go

@@ -24,6 +24,7 @@ type SendOptions struct {
 	Sender                       *bind.TransactOpts
 	UseTestRouter                bool
 	DisableTokenAmountValidation bool
+	NoWaitConfirm                bool // submit tx to mempool and return immediately without waiting for inclusion
 }
 
 // SendOptionsAccessor is the evm side of the ChainSendOption interface.

build/devenv/local-load.toml

@@ -0,0 +1,61 @@
+message_profiles=[
+    {name="data only", hasData=true, data_size_bytes=32, fee_token="native"},
+    {name="PTT", hasData=true, hasToken=true, fee_token="native"},
+    {name="token only", hasToken=true, fee_token="native"},
+]
+
+# steady: 1msg/10s for 10min, both directions
+[[test_profiles]]
+    enabled=true
+    chains_as_source=[
+        {selector="3379446385462418246", ratio=1},  # 0G mainnet
+        {selector="4793464827907405086", ratio=1}, # plume mainnet
+    ]
+    chains_as_dest=[
+        {selector="3379446385462418246", ratio=1},  # 0G mainnet
+        {selector="4793464827907405086", ratio=1}, # plume mainnet
+    ]
+    messages=[
+        {ratio=1, message_profile="PTT"},
+    ]
+    test_duration = "10m"
+    # Should be "integer/duration" like "1/1s" or "10/5m"
+    message_rate = "1/10s"
+    load_duration = "10m"
+
+
+# surge: 1/10s for 5min, 10/s burst for 3min, 1/10s cooldown for 2min — 0G→Plume only
+# schedule is built in code (wasp.Combine); this profile is reference only
+[[test_profiles]]
+    enabled=false
+    chains_as_source=[
+        {selector="3379446385462418246", ratio=1},  # 0G mainnet
+    ]
+    chains_as_dest=[
+        {selector="4793464827907405086", ratio=1}, # plume mainnet
+    ]
+    messages=[
+        {ratio=1, message_profile="PTT"},
+    ]
+    test_duration = "10m"
+    # Should be "integer/duration" like "1/1s" or "10/5m"
+    message_rate = "1/10s"
+    load_duration = "10m"
+
+
+# threshold: 1000 msgs fire-and-forget — 0G→Plume only
+[[test_profiles]]
+    enabled=false
+    chains_as_source=[
+        {selector="3379446385462418246", ratio=1},  # 0G mainnet
+    ]
+    chains_as_dest=[
+        {selector="4793464827907405086", ratio=1}, # plume mainnet
+    ]
+    messages=[
+        {ratio=1, message_profile="PTT"},
+    ]
+    test_duration = "5m"
+    # Should be "integer/duration" like "1/1s" or "10/5m"
+    message_rate = "10/1s"
+    load_duration = "5m"

build/devenv/prod-load.toml

@@ -0,0 +1,65 @@
+message_profiles=[
+    {name="data only", hasData=true, data_size_bytes=32, finality=10, fee_token="native"},
+    {name="PTT", hasData=true, hasToken=true, finality=10, fee_token="native"},
+    {name="token only", hasToken=true, finality=10, fee_token="native"},
+]
+
+# steady: 1msg/10s for 10min, both directions
+[[test_profiles]]
+    enabled=true
+    chains_as_source=[
+        {selector="4426351306075016396", ratio=1},  # 0G mainnet
+        {selector="17912061998839310979", ratio=1}, # plume mainnet
+    ]
+    chains_as_dest=[
+        {selector="4426351306075016396", ratio=1},  # 0G mainnet
+        {selector="17912061998839310979", ratio=1}, # plume mainnet
+    ]
+    messages=[
+        {ratio=1, message_profile="PTT"},
+    ]
+    test_duration = "10m"
+    # Should be "integer/duration" like "1/1s" or "10/5m"
+    message_rate = "1/10s"
+    load_duration = "10m"
+
+
+# surge: 1/10s for 5min, 10/s burst for 3min, 1/10s cooldown for 2min — 0G→Plume only
+# schedule is built in code (wasp.Combine); this profile is reference only
+[[test_profiles]]
+    enabled=false
+    chains_as_source=[
+        {selector="4426351306075016396", ratio=1},  # 0G mainnet
+        {selector="17912061998839310979", ratio=1}, # plume mainnet
+    ]
+    chains_as_dest=[
+        {selector="4426351306075016396", ratio=1},  # 0G mainnet
+        {selector="17912061998839310979", ratio=1}, # plume mainnet
+    ]
+    messages=[
+        {ratio=1, message_profile="PTT"},
+    ]
+    test_duration = "10m"
+    # Should be "integer/duration" like "1/1s" or "10/5m"
+    message_rate = "1/10s"
+    load_duration = "10m"
+
+
+# threshold: 1000 msgs fire-and-forget — both directions
+[[test_profiles]]
+    enabled=false
+    chains_as_source=[
+        {selector="4426351306075016396", ratio=1},  # 0G mainnet
+        {selector="17912061998839310979", ratio=1}, # plume mainnet
+    ]
+    chains_as_dest=[
+        {selector="4426351306075016396", ratio=1},  # 0G mainnet
+        {selector="17912061998839310979", ratio=1}, # plume mainnet
+    ]
+    messages=[
+        {ratio=1, message_profile="PTT"},
+    ]
+    test_duration = "5m"
+    # Should be "integer/duration" like "1/1s" or "10/5m"
+    message_rate = "10/1s"
+    load_duration = "5m"

build/devenv/tests/e2e/gun.go

@@ -220,8 +220,9 @@ func (m *EVMTXGun) Call(_ *wasp.Generator) *wasp.Response {
 
 	// WETH fees need msgValue=0; DisableTokenAmountValidation sets msgValue=fee and reverts.
 	sentEvent, _, err := chainAsSource.SendChainMessage(ctx, destSelector, srcMessage, evm.SendOptions{
-		Nonce:  &currentNonce,
-		Sender: sender,
+		Nonce:         &currentNonce,
+		Sender:        sender,
+		UseTestRouter: true,
 	})
 	if err != nil {
 		return &wasp.Response{Error: fmt.Errorf("failed to send message: %w", err).Error(), Failed: true}
@@ -332,32 +333,29 @@ func (m *EVMTXGun) buildExtraArgs(srcSelector uint64, dest destLoadInfo, opts cc
 func (m *EVMTXGun) selectMessageProfile(srcSelector uint64, dest destLoadInfo) (cciptestinterfaces.MessageFields, cciptestinterfaces.MessageOptions, error) {
 	receiver := dest.receiver
 
-	wethContract, err := m.e.DataStore.Addresses().Get(
-		datastore.NewAddressRefKey(
-			srcSelector,
-			datastore.ContractType(weth.ContractType),
-			semver.MustParse(weth.Deploy.Version()),
-			""))
-	if err != nil {
-		return cciptestinterfaces.MessageFields{}, cciptestinterfaces.MessageOptions{}, fmt.Errorf("could not find WETH address in datastore: %w", err)
-	}
-
-	committeeVerifierProxyRef, err := m.e.DataStore.Addresses().Get(
-		datastore.NewAddressRefKey(
-			srcSelector,
-			datastore.ContractType(versioned_verifier_resolver.CommitteeVerifierResolverType),
-			versioned_verifier_resolver.Version,
-			devenvcommon.DefaultCommitteeVerifierQualifier))
-	if err != nil {
-		return cciptestinterfaces.MessageFields{}, cciptestinterfaces.MessageOptions{}, fmt.Errorf("could not find committee verifier proxy address in datastore: %w", err)
-	}
-
-	// generate a random finality between 0 (chain default finality) and 1 (custom finality)
-	finality, err := rand.Int(rand.Reader, big.NewInt(2))
-	if err != nil {
-		return cciptestinterfaces.MessageFields{}, cciptestinterfaces.MessageOptions{}, fmt.Errorf("failed to generate finality: %w", err)
-	}
 	if m.testConfig == nil || m.testConfig.Messages == nil {
+		wethContract, err := m.e.DataStore.Addresses().Get(
+			datastore.NewAddressRefKey(
+				srcSelector,
+				datastore.ContractType(weth.ContractType),
+				semver.MustParse(weth.Deploy.Version()),
+				""))
+		if err != nil {
+			return cciptestinterfaces.MessageFields{}, cciptestinterfaces.MessageOptions{}, fmt.Errorf("could not find WETH address in datastore: %w", err)
+		}
+		committeeVerifierProxyRef, err := m.e.DataStore.Addresses().Get(
+			datastore.NewAddressRefKey(
+				srcSelector,
+				datastore.ContractType(versioned_verifier_resolver.CommitteeVerifierResolverType),
+				versioned_verifier_resolver.Version,
+				devenvcommon.DefaultCommitteeVerifierQualifier))
+		if err != nil {
+			return cciptestinterfaces.MessageFields{}, cciptestinterfaces.MessageOptions{}, fmt.Errorf("could not find committee verifier proxy address in datastore: %w", err)
+		}
+		finality, err := rand.Int(rand.Reader, big.NewInt(2))
+		if err != nil {
+			return cciptestinterfaces.MessageFields{}, cciptestinterfaces.MessageOptions{}, fmt.Errorf("failed to generate finality: %w", err)
+		}
 		return cciptestinterfaces.MessageFields{
 				Receiver: receiver,
 				Data:     []byte{},
@@ -374,14 +372,32 @@ func (m *EVMTXGun) selectMessageProfile(srcSelector uint64, dest destLoadInfo) (
 			},
 			nil
 	}
+
 	messageProfile, err := load.GetMessageByRatio(m.testConfig.Messages, m.messageProfiles)
 	if err != nil {
 		return cciptestinterfaces.MessageFields{}, cciptestinterfaces.MessageOptions{}, fmt.Errorf("failed to get message profile: %w", err)
 	}
+
+	var feeToken protocol.UnknownAddress
+	if messageProfile.FeeToken == "native" {
+		feeToken = make([]byte, 20)
+	} else { // "weth" or empty → WETH
+		wethContract, err := m.e.DataStore.Addresses().Get(
+			datastore.NewAddressRefKey(
+				srcSelector,
+				datastore.ContractType(weth.ContractType),
+				semver.MustParse(weth.Deploy.Version()),
+				""))
+		if err != nil {
+			return cciptestinterfaces.MessageFields{}, cciptestinterfaces.MessageOptions{}, fmt.Errorf("could not find WETH address in datastore: %w", err)
+		}
+		feeToken = common.HexToAddress(wethContract.Address).Bytes()
+	}
+
 	fields := cciptestinterfaces.MessageFields{
 		Receiver: receiver,
 		Data:     []byte{},
-		FeeToken: protocol.UnknownAddress(common.HexToAddress(wethContract.Address).Bytes()),
+		FeeToken: feeToken,
 	}
 	opts := cciptestinterfaces.MessageOptions{
 		FinalityConfig: protocol.Finality(messageProfile.Finality),

build/devenv/tests/e2e/load/types.go

@@ -17,6 +17,7 @@ type MessageProfileConfig struct {
 	HasData       bool   `toml:"has_data"`        // e.g., true
 	DataSizeBytes int    `toml:"data_size_bytes"` // payload size when has_data is true; 0 uses default
 	HasToken      bool   `toml:"has_token"`       // e.g., true
+	FeeToken      string `toml:"fee_token"`       // "native" for native gas token, "weth" or empty for WETH
 }
 
 // MessageDataSizeBytes returns the arbitrary-message payload size for a profile.