diff --git a/crates/bin/aggregator/README.md b/crates/bin/aggregator/README.md index 26563d5..099a6a4 100644 --- a/crates/bin/aggregator/README.md +++ b/crates/bin/aggregator/README.md @@ -23,15 +23,17 @@ As described in the [gateway README section on Graph Tally](https://github.com/e A JSON-RPC service for Graph Tally that lets clients request an aggregate receipt from a list of individual receipts. -Usage: graph_tally_aggregator [OPTIONS] --private-key +Usage: graph_tally_aggregator [OPTIONS] Options: --port Port to listen on for JSON-RPC requests [env: GRAPH_TALLY_PORT=] [default: 8080] - --private-key - Sender private key for signing Receipt Aggregate Vouchers, as a hex string [env: GRAPH_TALLY_PRIVATE_KEY=] - --public-keys - Signer public keys for incoming receipts/RAVs [env: GRAPH_TALLY_PUBLIC_KEYS=] + --signers + Signing key per payer, as `;`-separated `=` entries + [env: GRAPH_TALLY_SIGNERS=] + --accepted-signers + Extra accepted receipt signers per payer, as `;`-separated `=` entries + [env: GRAPH_TALLY_ACCEPTED_SIGNERS=] --max-request-body-size Maximum request body size in bytes. Defaults to 10MB [env: GRAPH_TALLY_MAX_REQUEST_BODY_SIZE=] [default: 10485760] --max-response-body-size @@ -56,6 +58,61 @@ Options: Print version ``` +## Signing keys and payers + +A RAV carries the `payer` named in the receipts it aggregates, and `GraphTallyCollector` recovers the +RAV's signer and requires it to be authorized for **that** payer. A RAV signed by a key belonging to a +different payer is rejected by indexers and uncollectable on chain. + +Because the collector binds each signer to exactly one authorizer, one key cannot serve two payers. +Serving more than one payer therefore takes one signing key per payer: + +```sh +GRAPH_TALLY_SIGNERS="0xPAYER_A=0xSIGNER_KEY_A;0xPAYER_B=0xSIGNER_KEY_B" +``` + +Left of `=` is a payer **address**; right of `=` is the private key of a **signer** authorized on chain +for that payer. It is not the payer's own key -- that controls the escrow balance and belongs with the +escrow manager, not with a public-facing service. + +The aggregator picks the signing key from the payer on the incoming receipts, and refuses requests for +a payer it holds no key for rather than signing with whatever key is at hand. + +### Rotating a signer within one payer + +Receipts already issued under the previous signer stay valid as long as that address remains accepted +**for its own payer**. List it explicitly, repeating the payer for several: + +```sh +GRAPH_TALLY_ACCEPTED_SIGNERS="0xPAYER_A=0xOLD_SIGNER_A;0xPAYER_A=0xOLDER_SIGNER_A" +``` + +Each payer's own signing address is always accepted, so it need not be listed. Accepted signers are +scoped per payer: a signer accepted for payer A does not vouch for payer B's receipts. + +The old signer must also stay authorized on chain until the last RAV it signed has been collected. +Revocation is `thawSigner` → wait out `REVOKE_AUTHORIZATION_THAWING_PERIOD` → `revokeAuthorizedSigner`. + +### Migrating to a new payer + +A payer change is not a signer rotation. Keep a `GRAPH_TALLY_SIGNERS` entry for the old payer, with a +key authorized to *it*, until every receipt issued under that payer has been aggregated and collected. +Dropping the old entry early leaves those receipts unaggregatable. + +### Single-payer deployments + +Use `GRAPH_TALLY_SIGNERS` with one entry. There is no separate single-payer mode: naming the payer is +what lets receipts for any *other* payer be refused, and a key on its own cannot say which payer it +belongs to. + +`GRAPH_TALLY_PRIVATE_KEY` and `GRAPH_TALLY_PUBLIC_KEYS` were removed. Nothing reads them any more, so +a deployment that still sets them starts normally and ignores them -- remove them from the manifest +rather than relying on an error. Startup fails only if `GRAPH_TALLY_SIGNERS` is unset. + +To find the payer for an existing key, read the `authorizer` field of +`GraphTallyCollector.authorizations()`, or take it from the escrow manager's +`payer = 0x…` startup line. + Please refer to [GraphTallyCollector](https://github.com/graphprotocol/contracts/blob/main/packages/horizon/contracts/payments/collectors/GraphTallyCollector.sol) for more information about Receipt Aggregate Voucher signing keys. ## Operational recommendations diff --git a/crates/bin/aggregator/src/aggregator.rs b/crates/bin/aggregator/src/aggregator.rs index 7cbfc52..2972511 100644 --- a/crates/bin/aggregator/src/aggregator.rs +++ b/crates/bin/aggregator/src/aggregator.rs @@ -1,26 +1,49 @@ use std::collections::HashSet; -use anyhow::{bail, Ok, Result}; +use anyhow::{anyhow, bail, Ok, Result}; use graph_tally_core::{receipt::WithUniqueId, signed_message::Eip712SignedMessage}; use graph_tally_graph::{Receipt, ReceiptAggregateVoucher}; use rayon::prelude::*; use thegraph_core::alloy::{ dyn_abi::Eip712Domain, primitives::{Address, FixedBytes}, - signers::local::PrivateKeySigner, sol_types::SolStruct, }; +use crate::signers::SignerRegistry; + pub fn check_and_aggregate_receipts( domain_separator: &Eip712Domain, receipts: &[Eip712SignedMessage], previous_rav: Option>, - wallet: &PrivateKeySigner, - accepted_addresses: &HashSet
, + signers: &SignerRegistry, ) -> Result> { check_signatures_unique(receipts)?; - // Check that the receipts are signed by an accepted signer address + // Get the allocation id from the first receipt, return error if there are no receipts + let (collection_id, payer, data_service, service_provider) = match receipts.first() { + Some(receipt) => ( + receipt.message.collection_id, + receipt.message.payer, + receipt.message.data_service, + receipt.message.service_provider, + ), + None => return Err(graph_tally_core::Error::NoValidReceiptsForRavRequest.into()), + }; + + // The payer is read before any signature is checked because it decides both halves of + // the check: which key signs the RAV, and which signers are acceptable on the way in. + // `check_collection_id` below proves the remaining receipts carry this same payer. + let (wallet, accepted_addresses) = signers.resolve(payer).ok_or_else(|| { + anyhow!( + "no signing key configured for payer {payer}; \ + signing with another payer's key would produce an uncollectable RAV" + ) + })?; + + // Check that the receipts are signed by a signer accepted for *this payer*. A signer + // accepted for some other payer is not interchangeable: the RAV carries this payer, and + // the collector requires its signer to be authorized for it. receipts.par_iter().try_for_each(|receipt| { check_signature_is_from_one_of_addresses(receipt, domain_separator, accepted_addresses) })?; @@ -37,17 +60,6 @@ pub fn check_and_aggregate_receipts( // Check that the receipts timestamp is greater than the previous rav check_receipt_timestamps(receipts, previous_rav.as_ref())?; - // Get the allocation id from the first receipt, return error if there are no receipts - let (collection_id, payer, data_service, service_provider) = match receipts.first() { - Some(receipt) => ( - receipt.message.collection_id, - receipt.message.payer, - receipt.message.data_service, - receipt.message.service_provider, - ), - None => return Err(graph_tally_core::Error::NoValidReceiptsForRavRequest.into()), - }; - // Check that the receipts all have the same collection id check_collection_id( receipts, @@ -193,6 +205,8 @@ mod tests { signers::local::PrivateKeySigner, }; + use crate::signers::SignerRegistry; + #[fixture] fn keys() -> (PrivateKeySigner, Address) { let wallet = PrivateKeySigner::random(); @@ -229,6 +243,126 @@ mod tests { graph_tally_eip712_domain(1, Address::from([0x11u8; 20])) } + /// Two payers, each with its own signing key -- the shape that a payer migration needs. + fn two_payer_registry( + payer_a: Address, + signer_a: &PrivateKeySigner, + payer_b: Address, + signer_b: &PrivateKeySigner, + ) -> SignerRegistry { + SignerRegistry::build( + [(payer_a, signer_a.clone()), (payer_b, signer_b.clone())], + [], + ) + .unwrap() + } + + fn receipt_for( + domain_separator: &Eip712Domain, + payer: Address, + signer: &PrivateKeySigner, + value: u128, + ) -> Eip712SignedMessage { + Eip712SignedMessage::new( + domain_separator, + Receipt::new( + collection_id(), + payer, + data_service(), + service_provider(), + value, + ) + .unwrap(), + signer, + ) + .unwrap() + } + + #[rstest] + #[test] + /// The RAV must be signed by the key belonging to the payer named in the receipts. + /// + /// The collector recovers the RAV signer and requires it to be authorized for the RAV's + /// payer, so signing payer A's receipts with payer B's key yields a RAV that is rejected + /// by the indexer and uncollectable on chain. + fn signs_each_payer_with_its_own_key(domain_separator: Eip712Domain) { + let (payer_a, payer_b) = (Address::repeat_byte(0xa1), Address::repeat_byte(0xb2)); + let (signer_a, signer_b) = (PrivateKeySigner::random(), PrivateKeySigner::random()); + let registry = two_payer_registry(payer_a, &signer_a, payer_b, &signer_b); + + for (payer, expected) in [(payer_a, &signer_a), (payer_b, &signer_b)] { + let receipts = vec![receipt_for(&domain_separator, payer, expected, 42)]; + let rav = + super::check_and_aggregate_receipts(&domain_separator, &receipts, None, ®istry) + .unwrap(); + assert_eq!(rav.message.payer, payer); + assert_eq!( + rav.recover_signer(&domain_separator).unwrap(), + expected.address(), + ); + } + } + + #[rstest] + #[test] + /// A payer with no configured key is refused rather than signed with whatever key is at + /// hand. Refusing is a loud failure; signing anyway is a silent one discovered days later. + fn refuses_a_payer_it_holds_no_key_for(domain_separator: Eip712Domain) { + let (payer_a, payer_b) = (Address::repeat_byte(0xa1), Address::repeat_byte(0xb2)); + let (signer_a, signer_b) = (PrivateKeySigner::random(), PrivateKeySigner::random()); + let registry = two_payer_registry(payer_a, &signer_a, payer_b, &signer_b); + + let unknown = Address::repeat_byte(0xcc); + let receipts = vec![receipt_for(&domain_separator, unknown, &signer_a, 42)]; + let err = + super::check_and_aggregate_receipts(&domain_separator, &receipts, None, ®istry) + .unwrap_err(); + assert!(err.to_string().contains("no signing key configured")); + } + + #[rstest] + #[test] + /// Accepted signers are scoped per payer, so one payer's signer cannot vouch for another's + /// receipts. This is the case that turns a payer migration into uncollectable RAVs: the + /// receipts are accepted, the RAV carries the old payer, and the new key signs it. + fn rejects_a_signer_belonging_to_a_different_payer(domain_separator: Eip712Domain) { + let (payer_a, payer_b) = (Address::repeat_byte(0xa1), Address::repeat_byte(0xb2)); + let (signer_a, signer_b) = (PrivateKeySigner::random(), PrivateKeySigner::random()); + let registry = two_payer_registry(payer_a, &signer_a, payer_b, &signer_b); + + // Receipts claiming payer A, signed by payer B's signer. + let receipts = vec![receipt_for(&domain_separator, payer_a, &signer_b, 42)]; + let err = + super::check_and_aggregate_receipts(&domain_separator, &receipts, None, ®istry) + .unwrap_err(); + assert!(err.to_string().contains(&signer_b.address().to_string())); + } + + #[rstest] + #[test] + /// A rotation *within* one payer is still supported: receipts from the payer's previous + /// signer aggregate into a RAV signed by its current one. Both keys are authorized for + /// that same payer on chain, so the result is valid. + fn accepts_a_previous_signer_of_the_same_payer(domain_separator: Eip712Domain) { + let payer = Address::repeat_byte(0xa1); + let (old_signer, new_signer) = (PrivateKeySigner::random(), PrivateKeySigner::random()); + let registry = SignerRegistry::build( + [(payer, new_signer.clone())], + [(payer, old_signer.address())], + ) + .unwrap(); + + let receipts = vec![receipt_for(&domain_separator, payer, &old_signer, 42)]; + let rav = + super::check_and_aggregate_receipts(&domain_separator, &receipts, None, ®istry) + .unwrap(); + assert_eq!(rav.message.payer, payer); + assert_eq!( + rav.recover_signer(&domain_separator).unwrap(), + new_signer.address(), + ); + } + #[rstest] #[test] fn check_signatures_unique_fail( diff --git a/crates/bin/aggregator/src/lib.rs b/crates/bin/aggregator/src/lib.rs index 7f98a9f..574119a 100644 --- a/crates/bin/aggregator/src/lib.rs +++ b/crates/bin/aggregator/src/lib.rs @@ -5,3 +5,4 @@ pub mod grpc; pub mod jsonrpsee_helpers; pub mod metrics; pub mod server; +pub mod signers; diff --git a/crates/bin/aggregator/src/main.rs b/crates/bin/aggregator/src/main.rs index e283700..5d23028 100644 --- a/crates/bin/aggregator/src/main.rs +++ b/crates/bin/aggregator/src/main.rs @@ -1,10 +1,10 @@ #![doc = include_str!("../README.md")] -use std::{collections::HashSet, str::FromStr, time::Duration}; +use std::{sync::Arc, time::Duration}; -use anyhow::Result; +use anyhow::{anyhow, Context as _, Result}; use clap::Parser; -use graph_tally_aggregator::{metrics, server}; +use graph_tally_aggregator::{metrics, server, signers::SignerRegistry}; use graph_tally_core::graph_tally_eip712_domain; use log::{debug, info}; use thegraph_core::alloy::{ @@ -19,17 +19,22 @@ struct Args { #[arg(long, default_value_t = 8080, env = "GRAPH_TALLY_PORT")] port: u16, - /// Signer private key for signing Receipt Aggregate Vouchers, as a hex string. - #[arg(long, env = "GRAPH_TALLY_PRIVATE_KEY")] - private_key: String, + /// Signing key per payer, as `;`-separated `=` entries. + /// + /// The key is a signer authorized on chain for that payer, not the payer's own key. A RAV + /// carries the payer named in the receipts it aggregates, and the collector requires the + /// RAV's signer to be authorized for that payer -- so each payer served needs its own key. + #[arg(long, env = "GRAPH_TALLY_SIGNERS")] + signers: Option, - /// Signer public keys. Not the counterpart of the signer private key. Signers that are allowed - /// for the incoming receipts / RAV to aggregate. Useful when needing to accept receipts that - /// were signed with a different key (e.g. a recent key rotation, or receipts coming from a - /// different gateway / aggregator that use a different signing key). - /// Expects a comma-separated list of Ethereum addresses. - #[arg(long, env = "GRAPH_TALLY_PUBLIC_KEYS")] - public_keys: Option>, + /// Additional accepted receipt signers per payer, as `;`-separated + /// `=` entries. Repeat a payer to list several. + /// + /// Each payer's own signing key is already accepted and need not be listed. Every address + /// listed here must be authorized for the payer it is listed under -- a signer accepted + /// for one payer does not vouch for another's receipts. + #[arg(long, env = "GRAPH_TALLY_ACCEPTED_SIGNERS")] + accepted_signers: Option, /// Maximum request body size in bytes. /// Defaults to 10MB. @@ -74,8 +79,8 @@ impl std::fmt::Debug for Args { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("Args") .field("port", &self.port) - .field("public_keys", &self.public_keys) - .field("private_key", &"[REDACTED]") + .field("signers", &"[REDACTED]") + .field("accepted_signers", &self.accepted_signers) .field("max_request_body_size", &self.max_request_body_size) .field("max_response_body_size", &self.max_response_body_size) .field("max_connections", &self.max_connections) @@ -104,21 +109,26 @@ async fn main() -> Result<()> { // We just let it gracelessly get killed at the end of main() tokio::spawn(metrics::run_server(args.metrics_port)); - // Create a wallet from the mnemonic. - let wallet = PrivateKeySigner::from_str(&args.private_key)?; - - info!("Wallet address: {:#40x}", wallet.address()); + let signers = Arc::new(signer_registry( + // The payer is the piece an older single-key config never recorded, so say where to + // find it rather than leaving the reader to work out which payer their key serves. + args.signers.as_deref().context( + "GRAPH_TALLY_SIGNERS is required: `;`-separated `=` entries. The payer for an existing signing key is the `authorizer` returned \ + by GraphTallyCollector.authorizations().", + )?, + args.accepted_signers.as_deref(), + )?); + // Logged per payer because a signer bound to the wrong payer produces RAVs that are + // rejected downstream with nothing in this process's logs to say why. + for (payer, signer, accepted) in signers.summary() { + info!("payer {payer:#40x} signs with {signer:#40x}"); + info!(" accepted receipt signers: {accepted:?}"); + } // Create the EIP-712 domain separator. let domain_separator = create_eip712_domain(&args)?; - // Create HashSet of *all* allowed signers - let mut accepted_addresses: HashSet
= std::collections::HashSet::new(); - accepted_addresses.insert(wallet.address().0.into()); - if let Some(public_keys) = &args.public_keys { - accepted_addresses.extend(public_keys.iter().cloned()); - } - let kafka = match args.kafka_config { None => None, Some(config) => { @@ -134,8 +144,7 @@ async fn main() -> Result<()> { // This await is non-blocking let (handle, _) = server::run_server( args.port, - wallet, - accepted_addresses, + signers, domain_separator, args.max_request_body_size, args.max_response_body_size, @@ -153,6 +162,62 @@ async fn main() -> Result<()> { Ok(()) } +/// Split `;`-separated `key=value` pairs, as `--kafka-config` does. +/// +/// A segment with no `=` is an error rather than a skip. Dropping one would leave a payer +/// unserved or a signer unaccepted, and neither shows up until a request arrives and is +/// refused -- by which point the failure is the indexer's, not the operator's. +/// +/// Empty segments are still ignored, so a trailing or doubled `;` stays valid. +fn pairs(raw: &str) -> Result> { + raw.split(';') + .map(str::trim) + .filter(|entry| !entry.is_empty()) + .map(|entry| { + entry + .split_once('=') + .map(|(key, value)| (key.trim(), value.trim())) + .ok_or_else(|| anyhow!("malformed entry {entry:?}, expected `key=value`")) + }) + .collect() +} + +/// Parse `--signers` / `--accepted-signers` into a [`SignerRegistry`]. +fn signer_registry(signers: &str, accepted_signers: Option<&str>) -> Result { + let signing_keys = pairs(signers) + .context("GRAPH_TALLY_SIGNERS")? + .into_iter() + .map(|(payer, key)| { + let payer: Address = payer + .parse() + .with_context(|| format!("GRAPH_TALLY_SIGNERS: parse payer address {payer:?}"))?; + let key: PrivateKeySigner = key + .parse() + .with_context(|| format!("GRAPH_TALLY_SIGNERS: parse signing key for {payer}"))?; + Ok((payer, key)) + }) + .collect::>>()?; + + let accepted = accepted_signers + .map(pairs) + .transpose() + .context("GRAPH_TALLY_ACCEPTED_SIGNERS")? + .unwrap_or_default() + .into_iter() + .map(|(payer, signer)| { + let payer: Address = payer.parse().with_context(|| { + format!("GRAPH_TALLY_ACCEPTED_SIGNERS: parse payer address {payer:?}") + })?; + let signer: Address = signer.parse().with_context(|| { + format!("GRAPH_TALLY_ACCEPTED_SIGNERS: parse signer address for {payer}") + })?; + Ok((payer, signer)) + }) + .collect::>>()?; + + SignerRegistry::build(signing_keys, accepted).context("GRAPH_TALLY_SIGNERS") +} + /// Creates the Graph Tally EIP-712 domain separator based on the provided arguments fn create_eip712_domain(args: &Args) -> Result { if args.domain_chain_id.is_some() { @@ -172,3 +237,74 @@ fn create_eip712_domain(args: &Args) -> Result { verifying_contract.unwrap_or_default(), )) } + +#[cfg(test)] +mod tests { + use super::{pairs, signer_registry}; + + const PAYER_A: &str = "0x1111111111111111111111111111111111111111"; + const PAYER_B: &str = "0x2222222222222222222222222222222222222222"; + const KEY_A: &str = "0x0000000000000000000000000000000000000000000000000000000000000001"; + const KEY_B: &str = "0x0000000000000000000000000000000000000000000000000000000000000002"; + const SIGNER: &str = "0x3333333333333333333333333333333333333333"; + + #[test] + fn splits_pairs_and_trims() { + assert_eq!(pairs("a=1;b=2").unwrap(), vec![("a", "1"), ("b", "2")]); + assert_eq!( + pairs(" a = 1 ; b = 2 ").unwrap(), + vec![("a", "1"), ("b", "2")] + ); + // Only the first `=` separates, so a value may contain one. + assert_eq!(pairs("a=1=2").unwrap(), vec![("a", "1=2")]); + } + + #[test] + fn empty_segments_are_ignored() { + // Trailing and doubled separators stay valid. + assert_eq!(pairs("a=1;").unwrap(), vec![("a", "1")]); + assert_eq!(pairs("a=1;;b=2").unwrap(), vec![("a", "1"), ("b", "2")]); + assert_eq!(pairs(" ; ").unwrap(), vec![]); + } + + #[test] + fn a_segment_without_an_equals_is_an_error() { + // Skipping it would drop one payer from a multi-payer config and only surface when + // that payer sent a request. + let err = pairs("a=1;oops;b=2").unwrap_err().to_string(); + assert!(err.contains("oops"), "{err}"); + assert!(pairs("oops").is_err()); + } + + #[test] + fn a_typo_in_one_signers_entry_fails_startup() { + let good = format!("{PAYER_A}={KEY_A};{PAYER_B}={KEY_B}"); + assert!(signer_registry(&good, None).is_ok()); + + // Same config with the second `=` fat-fingered into a space. + let typo = format!("{PAYER_A}={KEY_A};{PAYER_B} {KEY_B}"); + // `.err()` rather than `unwrap_err()`: the Ok type holds private keys and is + // deliberately not Debug. + let err = signer_registry(&typo, None) + .err() + .expect("malformed entry should fail"); + assert!( + format!("{err:#}").contains("GRAPH_TALLY_SIGNERS"), + "{err:#}" + ); + } + + #[test] + fn a_typo_in_accepted_signers_fails_startup() { + let signers = format!("{PAYER_A}={KEY_A}"); + assert!(signer_registry(&signers, Some(&format!("{PAYER_A}={SIGNER}"))).is_ok()); + + let err = signer_registry(&signers, Some(&format!("{PAYER_A} {SIGNER}"))) + .err() + .expect("malformed entry should fail"); + assert!( + format!("{err:#}").contains("GRAPH_TALLY_ACCEPTED_SIGNERS"), + "{err:#}" + ); + } +} diff --git a/crates/bin/aggregator/src/server.rs b/crates/bin/aggregator/src/server.rs index 21b1b40..a9ec906 100644 --- a/crates/bin/aggregator/src/server.rs +++ b/crates/bin/aggregator/src/server.rs @@ -1,4 +1,4 @@ -use std::{collections::HashSet, fmt::Debug, str::FromStr, time::Duration}; +use std::{fmt::Debug, str::FromStr, sync::Arc, time::Duration}; use anyhow::Result; use axum::{error_handling::HandleError, routing::post_service, BoxError, Router}; @@ -12,9 +12,7 @@ use jsonrpsee::{ use lazy_static::lazy_static; use log::{error, info}; use prometheus::{register_counter, register_int_counter, Counter, IntCounter}; -use thegraph_core::alloy::{ - dyn_abi::Eip712Domain, primitives::Address, signers::local::PrivateKeySigner, -}; +use thegraph_core::alloy::{dyn_abi::Eip712Domain, primitives::Address}; use tokio::{net::TcpListener, signal, task::JoinHandle}; use tonic::{codec::CompressionEncoding, service::Routes, Request, Response, Status}; use tower::{layer::util::Identity, make::Shared}; @@ -31,6 +29,7 @@ use crate::{ error_codes::{JsonRpcErrorCode, JsonRpcWarningCode}, grpc::graph_tally, jsonrpsee_helpers::{JsonRpcError, JsonRpcResponse, JsonRpcResult, JsonRpcWarning}, + signers::SignerRegistry, }; // Register the metrics into the global metrics registry. @@ -108,8 +107,9 @@ pub trait Rpc { #[derive(Clone)] struct RpcImpl { - wallet: PrivateKeySigner, - accepted_addresses: HashSet
, + // Shared rather than cloned per request: it holds private keys, and cloning them into + // every connection handler serves no purpose. + signers: Arc, domain_separator: Eip712Domain, kafka: Option>, } @@ -145,8 +145,7 @@ fn check_api_version_deprecation(api_version: &GraphTallyRpcApiVersion) -> Optio fn aggregate_receipts_( api_version: String, - wallet: &PrivateKeySigner, - accepted_addresses: &HashSet
, + signers: &SignerRegistry, domain_separator: &Eip712Domain, receipts: Vec>, previous_rav: Option>, @@ -183,8 +182,7 @@ fn aggregate_receipts_( domain_separator, &receipts, previous_rav, - wallet, - accepted_addresses, + signers, ); // Handle aggregation error @@ -226,8 +224,7 @@ impl v2::tap_aggregator_server::TapAggregator for RpcImpl { &self.domain_separator, receipts.as_slice(), previous_rav, - &self.wallet, - &self.accepted_addresses, + &self.signers, ) { Ok(res) => { TOTAL_GRT_AGGREGATED.inc_by(receipts_grt as f64); @@ -282,8 +279,7 @@ impl graph_tally::graph_tally_aggregator_server::GraphTallyAggregator for RpcImp &self.domain_separator, receipts.as_slice(), previous_rav, - &self.wallet, - &self.accepted_addresses, + &self.signers, ) { Ok(res) => { TOTAL_GRT_AGGREGATED.inc_by(receipts_grt as f64); @@ -332,8 +328,7 @@ impl RpcServer for RpcImpl { match aggregate_receipts_( api_version, - &self.wallet, - &self.accepted_addresses, + &self.signers, &self.domain_separator, receipts, previous_rav, @@ -345,7 +340,7 @@ impl RpcServer for RpcImpl { if let Some(kafka) = &self.kafka { produce_kafka_records( kafka, - &self.wallet.address(), + &res.data.message.payer, &res.data.message.collectionId, res.data.message.valueAggregate, ); @@ -363,8 +358,7 @@ impl RpcServer for RpcImpl { #[allow(clippy::too_many_arguments)] pub async fn run_server( port: u16, - wallet: PrivateKeySigner, - accepted_addresses: HashSet
, + signers: Arc, domain_separator: Eip712Domain, max_request_body_size: u32, max_response_body_size: u32, @@ -374,8 +368,7 @@ pub async fn run_server( ) -> Result<(JoinHandle<()>, std::net::SocketAddr)> { // Setting up the JSON RPC server let rpc_impl = RpcImpl { - wallet, - accepted_addresses, + signers, domain_separator, kafka, }; @@ -544,7 +537,7 @@ fn produce_kafka_records( #[cfg(test)] #[allow(clippy::too_many_arguments)] mod tests { - use std::{collections::HashSet, str::FromStr, time::Duration}; + use std::{str::FromStr, sync::Arc, time::Duration}; use graph_tally_core::{graph_tally_eip712_domain, signed_message::Eip712SignedMessage}; use graph_tally_graph::{Receipt, ReceiptAggregateVoucher}; @@ -556,7 +549,7 @@ mod tests { signers::local::PrivateKeySigner, }; - use crate::server; + use crate::{server, signers::SignerRegistry}; #[derive(Clone)] struct Keys { @@ -618,6 +611,7 @@ mod tests { #[rstest] #[tokio::test] async fn protocol_version( + payer: Address, domain_separator: Eip712Domain, http_request_size_limit: u32, http_response_size_limit: u32, @@ -630,8 +624,7 @@ mod tests { // Start the JSON-RPC server. let (handle, local_addr) = server::run_server( 0, - keys_main.wallet, - HashSet::from([keys_main.address]), + Arc::new(SignerRegistry::build([(payer, keys_main.wallet.clone())], []).unwrap()), domain_separator, http_request_size_limit, http_response_size_limit, @@ -687,8 +680,13 @@ mod tests { // Start the JSON-RPC server. let (handle, local_addr) = server::run_server( 0, - keys_main.wallet.clone(), - HashSet::from([keys_main.address, keys_0.address, keys_1.address]), + Arc::new( + SignerRegistry::build( + [(payer, keys_main.wallet.clone())], + [(payer, keys_0.address), (payer, keys_1.address)], + ) + .unwrap(), + ), domain_separator.clone(), http_request_size_limit, http_response_size_limit, @@ -782,8 +780,13 @@ mod tests { // Start the JSON-RPC server. let (handle, local_addr) = server::run_server( 0, - keys_main.wallet.clone(), - HashSet::from([keys_main.address, keys_0.address, keys_1.address]), + Arc::new( + SignerRegistry::build( + [(payer, keys_main.wallet.clone())], + [(payer, keys_0.address), (payer, keys_1.address)], + ) + .unwrap(), + ), domain_separator.clone(), http_request_size_limit, http_response_size_limit, @@ -869,8 +872,7 @@ mod tests { // Start the JSON-RPC server. let (handle, local_addr) = server::run_server( 0, - keys_main.wallet.clone(), - HashSet::from([keys_main.address]), + Arc::new(SignerRegistry::build([(payer, keys_main.wallet.clone())], []).unwrap()), domain_separator.clone(), http_request_size_limit, http_response_size_limit, @@ -966,8 +968,7 @@ mod tests { // Start the JSON-RPC server. let (handle, local_addr) = server::run_server( 0, - keys_main.wallet.clone(), - HashSet::from([keys_main.address]), + Arc::new(SignerRegistry::build([(payer, keys_main.wallet.clone())], []).unwrap()), domain_separator.clone(), http_request_size_limit, http_response_size_limit, diff --git a/crates/bin/aggregator/src/signers.rs b/crates/bin/aggregator/src/signers.rs new file mode 100644 index 0000000..8d51c20 --- /dev/null +++ b/crates/bin/aggregator/src/signers.rs @@ -0,0 +1,239 @@ +//! Per-payer signing keys. +//! +//! A GraphTally payer can accept a number of signers to manage receipts/RAVs on its behalf. +//! These signers must be authorized on chain for that specific payer. +//! +//! The aggregator process will accept receipts and RAVs from any of those signers, process +//! them and sign with a configured signer for the target payer. + +use std::collections::{HashMap, HashSet}; + +use anyhow::{anyhow, bail}; +use thegraph_core::alloy::{primitives::Address, signers::local::PrivateKeySigner}; + +/// For any given GraphTally payer, one key signs but several signers may be accepted +/// - `signing_key`, used to sign RAVs for this payer +/// - `accepted_signers`, list of signer addresses for which to accept receipts/RAVs for this payer +pub struct PayerKeys { + signing_key: PrivateKeySigner, + accepted_signers: HashSet
, +} + +impl PayerKeys { + /// Keys for one payer, accepting `also_accept` on top of `signing_key`'s own address. + /// + /// Private: a `PayerKeys` on its own carries no cross-payer context, so building the map + /// by hand is what would let one address end up under two payers. + fn new(signing_key: PrivateKeySigner, also_accept: impl IntoIterator) -> Self { + let mut accepted_signers = HashSet::from([signing_key.address()]); + accepted_signers.extend(also_accept); + Self { + signing_key, + accepted_signers, + } + } +} + +/// Resolves a payer to the key that may sign RAVs on its behalf. +pub struct SignerRegistry(HashMap); + +impl SignerRegistry { + /// Build a registry from already-parsed `(payer, signing key)` and `(payer, accepted signer)` + /// pairs. A signer is bound to exactly one authorizer on chain, so an address may appear + /// under one payer only, whichever way it arrives. + /// + /// This is the only way to obtain a `SignerRegistry`, so the uniqueness rule cannot be + /// sidestepped by assembling the map yourself and handing it to [`crate::server::run_server`]. + pub fn build( + signing_keys: impl IntoIterator, + accepted: impl IntoIterator, + ) -> anyhow::Result { + let mut payers: HashMap = HashMap::new(); + // Every signer address seen so far and the payer it belongs to + let mut signer_payer: HashMap = HashMap::new(); + for (payer, signing_key) in signing_keys { + if let Some(other) = signer_payer.insert(signing_key.address(), payer) { + bail!( + "signing key {} is listed for both {other} and {payer}, but a signer can only \ + be authorized for one payer", + signing_key.address(), + ); + } + if payers + .insert(payer, PayerKeys::new(signing_key, [])) + .is_some() + { + bail!("duplicate entry for payer {payer}"); + } + } + if payers.is_empty() { + bail!("no payers configured"); + } + + for (payer, accepted) in accepted { + let keys = payers.get_mut(&payer).ok_or_else(|| { + anyhow!("accepted signer {accepted} names payer {payer}, which has no signing key") + })?; + if let Some(other) = signer_payer.insert(accepted, payer) { + if other != payer { + bail!( + "signer {accepted} is listed for both {other} and {payer}, but a signer \ + can only be authorized for one payer" + ); + } + } + keys.accepted_signers.insert(accepted); + } + + Ok(Self(payers)) + } + + /// The key that may sign for `payer`, and the signers whose receipts are accepted for it. + /// + /// `None` means no key is held for that payer. Signing anyway would produce a RAV that + /// neither the indexer nor the collector accepts, so refusing is the only safe answer. + pub fn resolve(&self, payer: Address) -> Option<(&PrivateKeySigner, &HashSet
)> { + self.0 + .get(&payer) + .map(|keys| (&keys.signing_key, &keys.accepted_signers)) + } + + /// `(payer, signing address, accepted signers)` per entry, for logging at startup. + pub fn summary(&self) -> Vec<(Address, Address, Vec
)> { + let mut out: Vec<_> = self + .0 + .iter() + .map(|(payer, keys)| { + let mut accepted: Vec
= keys.accepted_signers.iter().copied().collect(); + accepted.sort(); + (*payer, keys.signing_key.address(), accepted) + }) + .collect(); + out.sort_by_key(|(payer, _, _)| *payer); + out + } +} + +#[cfg(test)] +mod tests { + use thegraph_core::alloy::{primitives::Address, signers::local::PrivateKeySigner}; + + use super::SignerRegistry; + + fn payer(n: u8) -> Address { + Address::repeat_byte(n) + } + + fn key(n: u8) -> PrivateKeySigner { + PrivateKeySigner::from_bytes(&Address::repeat_byte(n).into_word()).unwrap() + } + + #[test] + fn selects_the_key_for_the_payer() { + let (a, b) = (key(1), key(2)); + let registry = + SignerRegistry::build([(payer(1), a.clone()), (payer(2), b.clone())], []).unwrap(); + assert_eq!(registry.resolve(payer(1)).unwrap().0.address(), a.address()); + assert_eq!(registry.resolve(payer(2)).unwrap().0.address(), b.address()); + } + + #[test] + fn unknown_payer_is_refused() { + let registry = SignerRegistry::build([(payer(1), key(1))], []).unwrap(); + // Signing here would produce a RAV no indexer or collector would accept. + assert!(registry.resolve(payer(3)).is_none()); + } + + #[test] + fn a_payers_own_signer_is_accepted_without_being_listed() { + let a = key(1); + let registry = SignerRegistry::build([(payer(1), a.clone())], []).unwrap(); + // Needed for `previous_rav`: the RAV this process signs comes back as input, and is + // checked against this same set. + assert!(registry.resolve(payer(1)).unwrap().1.contains(&a.address())); + } + + #[test] + fn accepted_signers_are_scoped_to_their_payer() { + let other = payer(7); + let registry = SignerRegistry::build( + [(payer(1), key(1)), (payer(2), key(2))], + [(payer(1), other)], + ) + .unwrap(); + // A signer accepted for one payer must not be accepted for the other -- accepting it + // across payers is what turns a migration into uncollectable RAVs. + assert!(registry.resolve(payer(1)).unwrap().1.contains(&other)); + assert!(!registry.resolve(payer(2)).unwrap().1.contains(&other)); + } + + #[test] + fn accepted_signers_accumulate_for_one_payer() { + let registry = SignerRegistry::build( + [(payer(1), key(1))], + [(payer(1), payer(7)), (payer(1), payer(8))], + ) + .unwrap(); + let (_, accepted) = registry.resolve(payer(1)).unwrap(); + assert_eq!(accepted.len(), 3); // own signer + two listed + } + + #[test] + fn an_accepted_signer_cannot_be_shared_across_payers() { + let shared = payer(9); + // Accepting one address for two payers is a config the chain cannot honour, and it + // would let receipts signed by a key authorized elsewhere draw on the other's escrow. + assert!(SignerRegistry::build( + [(payer(1), key(1)), (payer(2), key(2))], + [(payer(1), shared), (payer(2), shared)], + ) + .is_err()); + // Same, when the shared address is another payer's signing key. + let a_key = key(1); + assert!(SignerRegistry::build( + [(payer(1), a_key.clone()), (payer(2), key(2))], + [(payer(2), a_key.address())], + ) + .is_err()); + } + + #[test] + fn repeating_a_signer_for_its_own_payer_is_allowed() { + let extra = payer(9); + // Listing the same pair twice is idempotent, not a conflict. + assert!(SignerRegistry::build( + [(payer(1), key(1))], + [(payer(1), extra), (payer(1), extra)] + ) + .is_ok()); + // Nor is redundantly listing a payer's own signing key, which is accepted anyway. + let a_key = key(1); + assert!( + SignerRegistry::build([(payer(1), a_key.clone())], [(payer(1), a_key.address())]) + .is_ok() + ); + } + + #[test] + fn rejects_configs_the_chain_could_not_honour() { + // One key cannot be authorized for two payers. + assert!(SignerRegistry::build([(payer(1), key(1)), (payer(2), key(1))], []).is_err()); + // Two keys for one payer: only one can sign, so the second is silently lost. + assert!(SignerRegistry::build([(payer(1), key(1)), (payer(1), key(2))], []).is_err()); + // Accepted signer naming a payer with no signing key. + assert!(SignerRegistry::build([(payer(1), key(1))], [(payer(5), payer(7))]).is_err()); + // Nothing configured. + assert!(SignerRegistry::build([], []).is_err()); + } + + #[test] + fn summary_reports_every_payer() { + let a = key(1); + let summary = SignerRegistry::build([(payer(1), a.clone()), (payer(2), key(2))], []) + .unwrap() + .summary(); + assert_eq!(summary.len(), 2); + assert_eq!(summary[0].0, payer(1)); + assert_eq!(summary[0].1, a.address()); + } +} diff --git a/crates/bin/aggregator/tests/aggregate_test.rs b/crates/bin/aggregator/tests/aggregate_test.rs index 1a9a657..0851fcf 100644 --- a/crates/bin/aggregator/tests/aggregate_test.rs +++ b/crates/bin/aggregator/tests/aggregate_test.rs @@ -1,9 +1,10 @@ -use std::{collections::HashSet, str::FromStr, time::Duration}; +use std::{str::FromStr, sync::Arc, time::Duration}; use graph_tally_aggregator::{ grpc::graph_tally::{graph_tally_aggregator_client::GraphTallyAggregatorClient, RavRequest}, jsonrpsee_helpers::JsonRpcResponse, server, + signers::SignerRegistry, }; use graph_tally_core::{graph_tally_eip712_domain, signed_message::Eip712SignedMessage}; use graph_tally_graph::{Receipt, ReceiptAggregateVoucher}; @@ -19,17 +20,16 @@ async fn aggregation_test() { let domain_separator = graph_tally_eip712_domain(1, Address::ZERO); let wallet = PrivateKeySigner::random(); + // The registry is keyed by payer, so it has to be known before the server starts. + let payer = Address::from_str("0xabababababababababababababababababababab").unwrap(); let max_request_body_size = 1024 * 100; let max_response_body_size = 1024 * 100; let max_concurrent_connections = 1; - let accepted_addresses = HashSet::from([wallet.address()]); - let (join_handle, local_addr) = server::run_server( 0, - wallet.clone(), - accepted_addresses, + Arc::new(SignerRegistry::build([(payer, wallet.clone())], []).unwrap()), domain_separator.clone(), max_request_body_size, max_response_body_size, @@ -48,7 +48,6 @@ async fn aggregation_test() { .send_compressed(CompressionEncoding::Zstd); let collection_id = FixedBytes::from([0xab; 32]); - let payer = Address::from_str("0xabababababababababababababababababababab").unwrap(); let data_service = Address::from_str("0xdeaddeaddeaddeaddeaddeaddeaddeaddeaddead").unwrap(); let service_provider = Address::from_str("0xbeefbeefbeefbeefbeefbeefbeefbeefbeefbeef").unwrap(); diff --git a/crates/integration_tests/tests/showcase.rs b/crates/integration_tests/tests/showcase.rs index dc9a09f..906cdba 100644 --- a/crates/integration_tests/tests/showcase.rs +++ b/crates/integration_tests/tests/showcase.rs @@ -2,7 +2,7 @@ // The tests use a mock Indexer server running a Manager instance and a graph_tally_aggregator to handle RAV requests. // An Indexer checks and stores receipts. After receiving a specific number of receipts, the Indexer sends a RAV request to the aggregator. use std::{ - collections::{HashMap, HashSet}, + collections::HashMap, convert::TryInto, net::{SocketAddr, TcpListener}, str::FromStr, @@ -11,7 +11,7 @@ use std::{ }; use anyhow::{Error, Result}; -use graph_tally_aggregator::{jsonrpsee_helpers, server as agg_server}; +use graph_tally_aggregator::{jsonrpsee_helpers, server as agg_server, signers::SignerRegistry}; use graph_tally_core::{ graph_tally_eip712_domain, manager::context::memory::{checks::get_full_list_of_checks, *}, @@ -882,12 +882,13 @@ async fn start_sender_aggregator( listener.local_addr()?.port() }; - let accepted_addresses = HashSet::from([keys.address()]); + // The registry is keyed by payer, and every receipt these tests build carries the `payer` + // fixture, so that is the one payer this aggregator serves. + let signers = Arc::new(SignerRegistry::build([(payer(), keys)], []).unwrap()); let (server_handle, socket_addr) = agg_server::run_server( http_port, - keys, - accepted_addresses, + signers, domain_separator, http_request_size_limit, http_response_size_limit, diff --git a/grafana/aggregator.json b/grafana/aggregator.json new file mode 100644 index 0000000..ad3d02e --- /dev/null +++ b/grafana/aggregator.json @@ -0,0 +1,2758 @@ +{ + "apiVersion": "dashboard.grafana.app/v2", + "kind": "Dashboard", + "metadata": { + "name": "graph-tally-aggregator", + "labels": {}, + "annotations": {} + }, + "spec": { + "annotations": [ + { + "kind": "AnnotationQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana", + "version": "v0", + "spec": {}, + "labels": { + "grafana.app/export-label": "grafana-1" + } + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "builtIn": true + } + } + ], + "cursorSync": "Crosshair", + "editable": true, + "elements": { + "panel-1": { + "kind": "Panel", + "spec": { + "id": 1, + "title": "Aggregations / sec", + "description": "Successful RAV aggregations per second, all instances.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "${datasource}" + }, + "spec": { + "expr": "(sum(rate(aggregation_success_count{job=~\"$job\"}[$__rate_interval])) or vector(0))", + "legendFormat": "aggregations/s" + }, + "labels": { + "grafana.app/export-label": "prometheus-1", + "grafana.app/export-datasource-name": "VM graph-mainnet" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "13.3.0-34942711622", + "spec": { + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "decimals": 3, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "green" + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + } + } + } + } + }, + "panel-2": { + "kind": "Panel", + "spec": { + "id": 2, + "title": "Failures / sec", + "description": "Failed aggregation requests per second. Covers every rejection reason -- bad signer, timestamp, collection id mismatch.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "${datasource}" + }, + "spec": { + "expr": "(sum(rate(aggregation_failure_count{job=~\"$job\"}[$__rate_interval])) or vector(0))", + "legendFormat": "failures/s" + }, + "labels": { + "grafana.app/export-label": "prometheus-1", + "grafana.app/export-datasource-name": "VM graph-mainnet" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "13.3.0-34942711622", + "spec": { + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "decimals": 3, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "green" + }, + { + "value": 1e-09, + "color": "red" + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + } + } + } + } + }, + "panel-3": { + "kind": "Panel", + "spec": { + "id": 3, + "title": "Receipts / sec", + "description": "Individual receipts folded into RAVs per second.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "${datasource}" + }, + "spec": { + "expr": "(sum(rate(total_aggregated_receipts{job=~\"$job\"}[$__rate_interval])) or vector(0))", + "legendFormat": "receipts/s" + }, + "labels": { + "grafana.app/export-label": "prometheus-1", + "grafana.app/export-datasource-name": "VM graph-mainnet" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "13.3.0-34942711622", + "spec": { + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "decimals": 1, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "green" + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + } + } + } + } + }, + "panel-4": { + "kind": "Panel", + "spec": { + "id": 4, + "title": "GRT Aggregated / hr", + "description": "total_aggregated_grt is an f64 counter in wei (server.rs:63-68); above ~2^53 wei it loses integer precision. Good for trend, not for reconciliation.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "${datasource}" + }, + "spec": { + "expr": "(sum(rate(total_aggregated_grt{job=~\"$job\"}[$__rate_interval])) or vector(0)) * 3600 / 1e18", + "legendFormat": "GRT/hr" + }, + "labels": { + "grafana.app/export-label": "prometheus-1", + "grafana.app/export-datasource-name": "VM graph-mainnet" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "13.3.0-34942711622", + "spec": { + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "fieldConfig": { + "defaults": { + "unit": "none", + "decimals": 2, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "green" + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + } + } + } + } + }, + "panel-5": { + "kind": "Panel", + "spec": { + "id": 5, + "title": "Success Ratio", + "description": "Reads 0% when there is no traffic at all.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "${datasource}" + }, + "spec": { + "expr": "100 * (sum(rate(aggregation_success_count{job=~\"$job\"}[$__rate_interval])) or vector(0)) / clamp_min((sum(rate(aggregation_success_count{job=~\"$job\"}[$__rate_interval])) or vector(0)) + (sum(rate(aggregation_failure_count{job=~\"$job\"}[$__rate_interval])) or vector(0)), 1e-9)", + "legendFormat": "success %" + }, + "labels": { + "grafana.app/export-label": "prometheus-1", + "grafana.app/export-datasource-name": "VM graph-mainnet" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "13.3.0-34942711622", + "spec": { + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "decimals": 2, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "red" + }, + { + "value": 99, + "color": "green" + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + } + } + } + } + }, + "panel-6": { + "kind": "Panel", + "spec": { + "id": 6, + "title": "Receipts per RAV", + "description": "Batching efficiency. A collapse toward 1 means indexers are requesting aggregation far more often than they need to.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "${datasource}" + }, + "spec": { + "expr": "(sum(rate(total_aggregated_receipts{job=~\"$job\"}[$__rate_interval])) or vector(0)) / clamp_min((sum(rate(aggregation_success_count{job=~\"$job\"}[$__rate_interval])) or vector(0)), 1e-9)", + "legendFormat": "receipts / RAV" + }, + "labels": { + "grafana.app/export-label": "prometheus-1", + "grafana.app/export-datasource-name": "VM graph-mainnet" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "13.3.0-34942711622", + "spec": { + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "fieldConfig": { + "defaults": { + "unit": "none", + "decimals": 1, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "green" + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + } + } + } + } + }, + "panel-7": { + "kind": "Panel", + "spec": { + "id": 7, + "title": "Kafka Publish Failures / sec", + "description": "Every failure here means a RAV was returned to the indexer but never reached the gateway_ravs topic, so escrow debt accounting drifts (server.rs:536).", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "${datasource}" + }, + "spec": { + "expr": "(sum(rate(kafka_publish_failure_total{job=~\"$job\"}[$__rate_interval])) or vector(0))", + "legendFormat": "kafka failures/s" + }, + "labels": { + "grafana.app/export-label": "prometheus-1", + "grafana.app/export-datasource-name": "VM graph-mainnet" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "13.3.0-34942711622", + "spec": { + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "decimals": 3, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "green" + }, + { + "value": 1e-09, + "color": "red" + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + } + } + } + } + }, + "panel-8": { + "kind": "Panel", + "spec": { + "id": 8, + "title": "Instances Up", + "description": "Scrape target count. Does not prove the RPC port is serving -- the metrics server binds at main.rs:105, before the private key is parsed at line 108.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "${datasource}" + }, + "spec": { + "expr": "sum(up{job=~\"$job\"})", + "legendFormat": "instances up" + }, + "labels": { + "grafana.app/export-label": "prometheus-1", + "grafana.app/export-datasource-name": "VM graph-mainnet" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "13.3.0-34942711622", + "spec": { + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "fieldConfig": { + "defaults": { + "unit": "none", + "decimals": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "green" + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + } + } + } + } + }, + "panel-9": { + "kind": "Panel", + "spec": { + "id": 9, + "title": "Aggregation Rate", + "description": "Failures climbing while successes stay flat is the signature of a signer rotation gone wrong -- previous_rav rejected by aggregator.rs:29-35.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "${datasource}" + }, + "spec": { + "expr": "sum by (pod) (rate(aggregation_success_count{job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "success {{pod}}" + }, + "labels": { + "grafana.app/export-label": "prometheus-1", + "grafana.app/export-datasource-name": "VM graph-mainnet" + } + }, + "refId": "A", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "${datasource}" + }, + "spec": { + "expr": "sum by (pod) (rate(aggregation_failure_count{job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "failure {{pod}}" + }, + "labels": { + "grafana.app/export-label": "prometheus-1", + "grafana.app/export-datasource-name": "VM graph-mainnet" + } + }, + "refId": "B", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "13.3.0-34942711622", + "spec": { + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "green" + } + ] + }, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + } + }, + "overrides": [] + } + } + } + } + }, + "panel-10": { + "kind": "Panel", + "spec": { + "id": 10, + "title": "Success Ratio", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "${datasource}" + }, + "spec": { + "expr": "100 * (sum(rate(aggregation_success_count{job=~\"$job\"}[$__rate_interval])) or vector(0)) / clamp_min((sum(rate(aggregation_success_count{job=~\"$job\"}[$__rate_interval])) or vector(0)) + (sum(rate(aggregation_failure_count{job=~\"$job\"}[$__rate_interval])) or vector(0)), 1e-9)", + "legendFormat": "success %" + }, + "labels": { + "grafana.app/export-label": "prometheus-1", + "grafana.app/export-datasource-name": "VM graph-mainnet" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "13.3.0-34942711622", + "spec": { + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "red" + }, + { + "value": 99, + "color": "green" + } + ] + }, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + } + }, + "overrides": [] + } + } + } + } + }, + "panel-11": { + "kind": "Panel", + "spec": { + "id": 11, + "title": "Receipts Aggregated", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "${datasource}" + }, + "spec": { + "expr": "sum by (pod) (rate(total_aggregated_receipts{job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "{{pod}}" + }, + "labels": { + "grafana.app/export-label": "prometheus-1", + "grafana.app/export-datasource-name": "VM graph-mainnet" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "13.3.0-34942711622", + "spec": { + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "green" + } + ] + }, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + } + }, + "overrides": [] + } + } + } + } + }, + "panel-12": { + "kind": "Panel", + "spec": { + "id": 12, + "title": "Receipts per RAV", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "${datasource}" + }, + "spec": { + "expr": "(sum(rate(total_aggregated_receipts{job=~\"$job\"}[$__rate_interval])) or vector(0)) / clamp_min((sum(rate(aggregation_success_count{job=~\"$job\"}[$__rate_interval])) or vector(0)), 1e-9)", + "legendFormat": "receipts / RAV" + }, + "labels": { + "grafana.app/export-label": "prometheus-1", + "grafana.app/export-datasource-name": "VM graph-mainnet" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "13.3.0-34942711622", + "spec": { + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "green" + } + ] + }, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + } + }, + "overrides": [] + } + } + } + } + }, + "panel-13": { + "kind": "Panel", + "spec": { + "id": 13, + "title": "GRT Aggregated Rate", + "description": "GRT per hour.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "${datasource}" + }, + "spec": { + "expr": "sum by (pod) (rate(total_aggregated_grt{job=~\"$job\"}[$__rate_interval])) * 3600 / 1e18", + "legendFormat": "{{pod}}" + }, + "labels": { + "grafana.app/export-label": "prometheus-1", + "grafana.app/export-datasource-name": "VM graph-mainnet" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "13.3.0-34942711622", + "spec": { + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "green" + } + ] + }, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2 + }, + "overrides": [] + } + } + } + } + }, + "panel-14": { + "kind": "Panel", + "spec": { + "id": 14, + "title": "GRT Aggregated per Hour", + "description": "increase() rather than the raw counter, so process restarts do not read as a cliff.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "${datasource}" + }, + "spec": { + "expr": "sum(increase(total_aggregated_grt{job=~\"$job\"}[1h])) / 1e18", + "legendFormat": "GRT" + }, + "labels": { + "grafana.app/export-label": "prometheus-1", + "grafana.app/export-datasource-name": "VM graph-mainnet" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "13.3.0-34942711622", + "spec": { + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "green" + } + ] + }, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2 + }, + "overrides": [] + } + } + } + } + }, + "panel-15": { + "kind": "Panel", + "spec": { + "id": 15, + "title": "Kafka Publish Rate", + "description": "Absent entirely when --kafka-config is unset: the publish path is guarded by `if let Some(kafka)` and the counters never register.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "${datasource}" + }, + "spec": { + "expr": "sum by (pod) (rate(kafka_publish_success_total{job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "success {{pod}}" + }, + "labels": { + "grafana.app/export-label": "prometheus-1", + "grafana.app/export-datasource-name": "VM graph-mainnet" + } + }, + "refId": "A", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "${datasource}" + }, + "spec": { + "expr": "sum by (pod) (rate(kafka_publish_failure_total{job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "failure {{pod}}" + }, + "labels": { + "grafana.app/export-label": "prometheus-1", + "grafana.app/export-datasource-name": "VM graph-mainnet" + } + }, + "refId": "B", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "13.3.0-34942711622", + "spec": { + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "green" + } + ] + }, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + } + }, + "overrides": [] + } + } + } + } + }, + "panel-16": { + "kind": "Panel", + "spec": { + "id": 16, + "title": "Kafka Publish Failure Ratio", + "description": "Sustained non-zero means escrow debt is being under-counted and deposits will lag real usage.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "${datasource}" + }, + "spec": { + "expr": "100 * (sum(rate(kafka_publish_failure_total{job=~\"$job\"}[$__rate_interval])) or vector(0)) / clamp_min((sum(rate(kafka_publish_success_total{job=~\"$job\"}[$__rate_interval])) or vector(0)) + (sum(rate(kafka_publish_failure_total{job=~\"$job\"}[$__rate_interval])) or vector(0)), 1e-9)", + "legendFormat": "failure %" + }, + "labels": { + "grafana.app/export-label": "prometheus-1", + "grafana.app/export-datasource-name": "VM graph-mainnet" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "13.3.0-34942711622", + "spec": { + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "green" + }, + { + "value": 1e-09, + "color": "red" + } + ] + }, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + } + }, + "overrides": [] + } + } + } + } + }, + "panel-17": { + "kind": "Panel", + "spec": { + "id": 17, + "title": "Version Errors", + "description": "JSON-RPC only -- incremented at server.rs:158, which the gRPC handlers never reach. Flat zero is expected if all your traffic is gRPC.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "${datasource}" + }, + "spec": { + "expr": "(sum(rate(version_error_count{job=~\"$job\"}[$__rate_interval])) or vector(0))", + "legendFormat": "version errors" + }, + "labels": { + "grafana.app/export-label": "prometheus-1", + "grafana.app/export-datasource-name": "VM graph-mainnet" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "13.3.0-34942711622", + "spec": { + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "green" + }, + { + "value": 1e-09, + "color": "red" + } + ] + }, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + } + }, + "overrides": [] + } + } + } + } + }, + "panel-18": { + "kind": "Panel", + "spec": { + "id": 18, + "title": "Deprecation Warnings", + "description": "JSON-RPC only (server.rs:167). Also stays zero while GRAPH_TALLY_RPC_API_VERSIONS_DEPRECATED is empty.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "${datasource}" + }, + "spec": { + "expr": "(sum(rate(deprecation_warning_count{job=~\"$job\"}[$__rate_interval])) or vector(0))", + "legendFormat": "deprecation warnings" + }, + "labels": { + "grafana.app/export-label": "prometheus-1", + "grafana.app/export-datasource-name": "VM graph-mainnet" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "13.3.0-34942711622", + "spec": { + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "green" + } + ] + }, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + } + }, + "overrides": [] + } + } + } + } + }, + "panel-19": { + "kind": "Panel", + "spec": { + "id": 19, + "title": "Scrape Up", + "description": "The prometheus crate is built without the `process` feature, so there are no process_cpu/rss/start_time series to chart here.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "${datasource}" + }, + "spec": { + "expr": "up{job=~\"$job\"}", + "legendFormat": "{{pod}}" + }, + "labels": { + "grafana.app/export-label": "prometheus-1", + "grafana.app/export-datasource-name": "VM graph-mainnet" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "13.3.0-34942711622", + "spec": { + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "green" + } + ] + }, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + } + }, + "overrides": [] + } + } + } + } + }, + "panel-20": { + "kind": "Panel", + "spec": { + "id": 20, + "title": "Detected Restarts", + "description": "Counter resets as a restart proxy. There is no uptime metric, and the aggregator logs nothing after its three startup lines, so this is the cheapest restart signal available.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "${datasource}" + }, + "spec": { + "expr": "sum by (pod) (resets(aggregation_success_count{job=~\"$job\"}[1h]))", + "legendFormat": "{{pod}}" + }, + "labels": { + "grafana.app/export-label": "prometheus-1", + "grafana.app/export-datasource-name": "VM graph-mainnet" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "13.3.0-34942711622", + "spec": { + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "green" + }, + { + "value": 1e-09, + "color": "red" + } + ] + }, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + } + }, + "overrides": [] + } + } + } + } + }, + "panel-21": { + "kind": "Panel", + "spec": { + "id": 21, + "title": "Graph Tally aggregator logs", + "description": "Expect three lines at startup and silence thereafter -- there is no per-request logging on the gRPC path.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "victoriametrics-logs-datasource", + "version": "v0", + "spec": { + "direction": "desc", + "editorMode": "code", + "expr": "app: \"tap-aggregator-mainnet\" AND cluster: \"network\" | replace_regexp(\"\\\\x1b\\\\[[0-9;]*m\", \"\")", + "queryType": "instant" + }, + "labels": { + "grafana.app/export-label": "victoriametrics-logs-datasource-1", + "grafana.app/export-datasource-name": "vmetrics-logs" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "logs", + "version": "13.3.0-34942711622", + "spec": { + "options": { + "dedupStrategy": "none", + "enableInfiniteScrolling": false, + "enableLogDetails": true, + "prettifyLogMessage": true, + "showControls": false, + "showFieldSelector": false, + "showLevel": true, + "showLogAttributes": true, + "showTime": false, + "sortOrder": "Descending", + "timestampResolution": "ms", + "unwrappedColumns": false, + "wrapLogMessage": false + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + } + } + } + }, + "layout": { + "kind": "RowsLayout", + "spec": { + "rows": [ + { + "kind": "RowsLayoutRow", + "spec": { + "title": "Overview", + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 6, + "height": 4, + "element": { + "kind": "ElementReference", + "name": "panel-1" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 6, + "y": 0, + "width": 6, + "height": 4, + "element": { + "kind": "ElementReference", + "name": "panel-2" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 12, + "y": 0, + "width": 6, + "height": 4, + "element": { + "kind": "ElementReference", + "name": "panel-3" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 18, + "y": 0, + "width": 6, + "height": 4, + "element": { + "kind": "ElementReference", + "name": "panel-4" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 4, + "width": 6, + "height": 4, + "element": { + "kind": "ElementReference", + "name": "panel-5" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 6, + "y": 4, + "width": 6, + "height": 4, + "element": { + "kind": "ElementReference", + "name": "panel-6" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 12, + "y": 4, + "width": 6, + "height": 4, + "element": { + "kind": "ElementReference", + "name": "panel-7" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 18, + "y": 4, + "width": 6, + "height": 4, + "element": { + "kind": "ElementReference", + "name": "panel-8" + } + } + } + ] + } + } + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "title": "Aggregation", + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-9" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 12, + "y": 0, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-10" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 8, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-11" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 12, + "y": 8, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-12" + } + } + } + ] + } + } + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "title": "Value", + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-13" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 12, + "y": 0, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-14" + } + } + } + ] + } + } + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "title": "Kafka", + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-15" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 12, + "y": 0, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-16" + } + } + } + ] + } + } + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "title": "API Versions (JSON-RPC only)", + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-17" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 12, + "y": 0, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-18" + } + } + } + ] + } + } + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "title": "Health", + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-19" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 12, + "y": 0, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-20" + } + } + } + ] + } + } + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "title": "Logs", + "collapse": false, + "layout": { + "kind": "AutoGridLayout", + "spec": { + "maxColumnCount": 3, + "columnWidthMode": "standard", + "rowHeightMode": "standard", + "items": [ + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-21" + } + } + } + ] + } + } + } + } + ] + } + }, + "links": [], + "liveNow": false, + "preload": false, + "tags": [ + "graph-tally", + "aggregator" + ], + "timeSettings": { + "timezone": "browser", + "from": "now-6h", + "to": "now", + "autoRefresh": "", + "autoRefreshIntervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "hideTimepicker": false, + "fiscalYearStartMonth": 0 + }, + "title": "Graph Tally Aggregator", + "variables": [ + { + "kind": "DatasourceVariable", + "spec": { + "name": "datasource", + "pluginId": "prometheus", + "refresh": "onDashboardLoad", + "regex": "", + "current": { + "text": "", + "value": "" + }, + "options": [], + "multi": false, + "includeAll": false, + "label": "Datasource", + "hide": "dontHide", + "skipUrlSync": false, + "allowCustomValue": true + } + }, + { + "kind": "QueryVariable", + "spec": { + "name": "job", + "label": "Job", + "description": "Aggregator deployment to display. Keyed off `up` because every aggregator counter is lazy-registered and absent before first use; set the regex to narrow this list.", + "current": { + "text": "", + "value": "" + }, + "hide": "dontHide", + "refresh": "onDashboardLoad", + "skipUrlSync": false, + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "${datasource}" + }, + "spec": { + "qryType": 1, + "label": "job", + "metric": "up", + "query": "label_values(up,job)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "labels": { + "grafana.app/export-label": "prometheus-1", + "grafana.app/export-datasource-name": "VM graph-mainnet" + } + }, + "regex": "", + "sort": "alphabeticalAsc", + "definition": "label_values(up,job)", + "options": [], + "multi": false, + "includeAll": false, + "allowCustomValue": true + } + } + ] + } +}