diff --git a/README.md b/README.md index 2cbd375..0b61ca1 100644 --- a/README.md +++ b/README.md @@ -165,6 +165,31 @@ copy `.env.example` to `.env`, install Node.js 22 dependencies, apply Stop the stack with `docker compose down`. Persistent database and Redis data remain in Docker volumes; `docker compose down -v` removes them. + +### API and worker tiers + +The backend runs as two separately deployable processes: + +| Process | Entrypoint | `PROCESS_ROLE` | Does | +|---|---|---|---| +| API | `dist/main.js` | `api` | REST reads + the `/oracle` WebSocket feed | +| Worker | `dist/worker.js` | `worker` | oracle polling, consensus, on-chain publishing, claim settlement | + +The worker publishes readings to Redis (`REDIS_URL` is required for a split +deployment); the API relays them to WebSocket clients and serves the latest +reading per coverage type. Each tier can be scaled and restarted on its own, +and a burst of API traffic cannot delay a payout. + +- Run **one** worker per relayer key — the relayer's sequence numbers are + tracked in-process. +- The worker exposes `GET /health` (503 when a pipeline stage stops + succeeding for `WORKER_STALE_AFTER_MS`, default 10 min) and `GET /metrics` + on `WORKER_HEALTH_PORT` (default 4002). +- Leaving `PROCESS_ROLE` unset runs both tiers in one process (`all`). That + is the default so existing deployments keep paying out, and it is handy + locally, but deployed environments should split. + +`docker compose up` starts both tiers. ## Scripts | Command | Purpose | @@ -172,8 +197,11 @@ remain in Docker volumes; `docker compose down -v` removes them. | `npm run dev` | Hot-reloading dev server (ts-node + nodemon) | | `npm run build` | Compile TypeScript to `dist/` | | `npm start` | Run the compiled server | +| `npm run start:worker` / `start:worker:prod` | Run the worker tier (dev / compiled) | | `npm run typecheck` | `tsc --noEmit` | | `npm run lint` | ESLint over `src/` | +| `npm test` | Jest unit tests (includes property-based tests) | +| `npm run test:property` | Only the property-based / fuzz suites (`*.property.spec.ts`); set `FC_NUM_RUNS=20000` for a deeper soak, `FC_SEED=` to replay a failure | ## Operator CLI diff --git a/docker-compose.yml b/docker-compose.yml index 827d370..156bf2a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,3 +1,20 @@ +x-backend-env: &backend-env + FRONTEND_URL: ${FRONTEND_URL:-http://localhost:3000} + DATABASE_URL: postgresql://refract:${POSTGRES_PASSWORD:-refract_dev}@postgres:5432/refract + REDIS_URL: redis://redis:6379 + STELLAR_NETWORK: ${STELLAR_NETWORK:-testnet} + SOROBAN_RPC_URL: ${SOROBAN_RPC_URL:-https://soroban-testnet.stellar.org} + STELLAR_NETWORK_PASSPHRASE: ${STELLAR_NETWORK_PASSPHRASE:-Test SDF Network ; September 2015} + REFRACT_POOL_CONTRACT_ID: ${REFRACT_POOL_CONTRACT_ID:-} + REFRACT_POLICY_CONTRACT_ID: ${REFRACT_POLICY_CONTRACT_ID:-} + REFRACT_ORACLE_CONTRACT_ID: ${REFRACT_ORACLE_CONTRACT_ID:-} + ORACLE_RELAYER_SECRET: ${ORACLE_RELAYER_SECRET:-} + COINGECKO_BASE_URL: ${COINGECKO_BASE_URL:-https://api.coingecko.com/api/v3} + STELLAR_HORIZON_URL: ${STELLAR_HORIZON_URL:-https://horizon-testnet.stellar.org} + DEFILLAMA_BASE_URL: ${DEFILLAMA_BASE_URL:-https://api.llama.fi} + DEFILLAMA_PROTOCOL_SLUG: ${DEFILLAMA_PROTOCOL_SLUG:-aave} + ORACLE_HTTP_TIMEOUT_MS: ${ORACLE_HTTP_TIMEOUT_MS:-5000} + services: postgres: image: postgres:16-alpine @@ -61,22 +78,11 @@ services: # before Docker sends SIGKILL (the default is only 10s). stop_grace_period: 40s environment: + <<: *backend-env PORT: 4001 - FRONTEND_URL: ${FRONTEND_URL:-http://localhost:3000} - DATABASE_URL: postgresql://refract:${POSTGRES_PASSWORD:-refract_dev}@postgres:5432/refract - REDIS_URL: redis://redis:6379 - STELLAR_NETWORK: ${STELLAR_NETWORK:-testnet} - SOROBAN_RPC_URL: ${SOROBAN_RPC_URL:-https://soroban-testnet.stellar.org} - STELLAR_NETWORK_PASSPHRASE: ${STELLAR_NETWORK_PASSPHRASE:-Test SDF Network ; September 2015} - REFRACT_POOL_CONTRACT_ID: ${REFRACT_POOL_CONTRACT_ID:-} - REFRACT_POLICY_CONTRACT_ID: ${REFRACT_POLICY_CONTRACT_ID:-} - REFRACT_ORACLE_CONTRACT_ID: ${REFRACT_ORACLE_CONTRACT_ID:-} - ORACLE_RELAYER_SECRET: ${ORACLE_RELAYER_SECRET:-} - COINGECKO_BASE_URL: ${COINGECKO_BASE_URL:-https://api.coingecko.com/api/v3} - STELLAR_HORIZON_URL: ${STELLAR_HORIZON_URL:-https://horizon-testnet.stellar.org} - DEFILLAMA_BASE_URL: ${DEFILLAMA_BASE_URL:-https://api.llama.fi} - DEFILLAMA_PROTOCOL_SLUG: ${DEFILLAMA_PROTOCOL_SLUG:-aave} - ORACLE_HTTP_TIMEOUT_MS: ${ORACLE_HTTP_TIMEOUT_MS:-5000} + # API tier: serves reads and the WebSocket feed only. The payout + # pipeline runs in the `worker` service below. + PROCESS_ROLE: api SHUTDOWN_GRACE_PERIOD_MS: ${SHUTDOWN_GRACE_PERIOD_MS:-30000} ORACLE_SHUTDOWN_DRAIN_MS: ${ORACLE_SHUTDOWN_DRAIN_MS:-20000} ORACLE_EVIDENCE_SPOOL_DIR: /var/lib/refract/oracle-evidence @@ -98,6 +104,32 @@ services: retries: 3 start_period: 10s + worker: + # Oracle polling, consensus, on-chain publishing and claim settlement. + # Runs exactly one replica per relayer key (sequence numbers are managed + # in-process). Restarts independently of the API. + build: + context: . + dockerfile: Dockerfile + command: ["node", "dist/worker.js"] + environment: + <<: *backend-env + PROCESS_ROLE: worker + WORKER_HEALTH_PORT: 4002 + ports: + - "127.0.0.1:4002:4002" + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + healthcheck: + test: ["CMD", "node", "-e", "fetch('http://127.0.0.1:4002/health').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))"] + interval: 30s + timeout: 3s + retries: 3 + start_period: 30s + volumes: postgres-data: redis-data: diff --git a/docs/oracle-provenance.md b/docs/oracle-provenance.md new file mode 100644 index 0000000..faddd71 --- /dev/null +++ b/docs/oracle-provenance.md @@ -0,0 +1,69 @@ +# Oracle reading provenance + +Each oracle reading the relayer publishes carries a packed `u128` next to the +value, so contracts and auditors can tell how well-supported the value is +without leaving the ledger: + +``` +update_reading(coverage_type: u32, value: i128, provenance: u128) +``` + +The backend side lives in `src/oracle/oracle-encoding.ts` +(`packProvenance`, `unpackProvenance`, `provenanceFromDecision`, +`buildUpdateReadingArgs`). The contract must accept the third argument +before the relayer switches to it. + +## Layout + +Most significant bits first: + +| bits | width | field | +|----------|-------|-------------------------------------------| +| 127..124 | 4 | layout version (currently `1`) | +| 123..120 | 4 | trust tier | +| 119..112 | 8 | agreeing source count | +| 111..104 | 8 | reserved, always `0` | +| 103..64 | 40 | source timestamp, Unix seconds | +| 63..0 | 64 | consensus decision id | + +A provenance of `0` (version 0) means "no provenance" and is never produced. + +### Trust tiers + +| value | tier | meaning | +|-------|----------------|-----------------------------------------------------------------| +| 0 | `Unverified` | no consensus — never published | +| 1 | `SingleSource` | exactly one source agreed (only if policy allows one source) | +| 2 | `Corroborated` | two or more sources agreed and the weighted quorum was met | +| 3 | `Unanimous` | three or more sources, all fresh, valid and in agreement | + +The encoder rejects inconsistent combinations (e.g. `Corroborated` with one +source), so consumers can rely on `tier >= 2` meaning "at least two sources". + +### Source timestamp + +The earliest observation time among the agreeing sources, floored to whole +seconds. Using the earliest one means the value never looks fresher than its +oldest supporting input. + +### Decision id + +The first 64 bits of a SHA-256 over the round's sorted inputs, the consensus +policy, the outcome and the evaluation time (`computeDecisionId` in +`src/oracle/consensus/consensus.ts`). Re-running consensus over the logged +inputs reproduces the id, which ties an on-chain value back to the exact +observations behind it. + +## Decoding in a contract (Rust sketch) + +```rust +let version = (p >> 124) & 0xF; +let tier = (p >> 120) & 0xF; +let sources = (p >> 112) & 0xFF; +let ts_secs = ((p >> 64) & 0xFF_FFFF_FFFF) as u64; +let decision = (p & 0xFFFF_FFFF_FFFF_FFFF) as u64; +assert!(version == 1); +``` + +A consumer that only wants corroborated data can require `tier >= 2`; one +that cares about freshness can compare `ts_secs` with the ledger timestamp. diff --git a/package.json b/package.json index 9613b5e..7a1b752 100644 --- a/package.json +++ b/package.json @@ -1,27 +1,41 @@ { - "name": "refract", + "name": "refract-protocol-backend", "version": "0.1.0", - "private": true, - "description": "Refract oracle network monorepo", + "description": "Refract Protocol backend \u2014 oracle monitoring, claim processing, premium quoting", "main": "dist/main.js", + "private": true, "workspaces": [ "refract-backend", "refract-frontend", "refract-contracts" ], "scripts": { - "build": "npm run build --workspaces --if-present", - "dev": "nodemon --exec ts-node src/main.ts", - "start": "node dist/main.js", - "typecheck": "tsc -p tsconfig.json --noEmit", - "test": "npm run test:unit && npm run test:contracts", + "build": "nest build", + "start": "nest start", + "start:dev": "nest start --watch", + "dev": "nest start --watch", + "start:prod": "node dist/main", + "start:worker": "nest start --entryFile worker", + "start:worker:dev": "nest start --watch --entryFile worker", + "start:worker:prod": "node dist/worker", + "lint": "eslint \"src/**/*.ts\"", + "typecheck": "tsc --noEmit", + "test": "jest", + "test:watch": "jest --watch", + "test:cov": "jest --coverage", + "test:property": "jest --testRegex '.*\\.property\\.spec\\.ts$'", + "test:integration": "jest --config test/integration/jest.integration.config.ts --runInBand", "test:unit": "npm run test --workspaces --if-present", "test:contracts": "node test/contracts/verify.js", "test:contracts:update": "node test/contracts/verify.js --update", "attack-cost": "node scripts/attack-cost.js", "attack-cost:reference": "node scripts/attack-cost.js --fixture test/fixtures/attack-cost/reference.json", - "lint": "npm run lint --workspaces --if-present", - "ci": "npm run lint && npm run test" + "ci": "npm run lint && npm run test", + "migrate:up": "node-pg-migrate up --migrations-dir migrations --config-file database.json", + "migrate:down": "node-pg-migrate down --migrations-dir migrations --config-file database.json", + "migrate:create": "node-pg-migrate create --migrations-dir migrations --config-file database.json", + "migrate:status": "node-pg-migrate status --migrations-dir migrations --config-file database.json", + "regenerate:xdr-fixtures": "ts-node scripts/regenerate-xdr-fixtures.ts" }, "engines": { "node": ">=18" @@ -89,16 +103,28 @@ "testMatch": [ "**/src/**/*.spec.ts" ], - "moduleFileExtensions": ["js", "json", "ts"], + "moduleFileExtensions": [ + "js", + "json", + "ts" + ], "transform": { - "^.+\\.(t|j)s$": "ts-jest", "^.+\\.ts$": "ts-jest", - "^.+\\.js$": "babel-jest" + "^.+\\.js$": [ + "babel-jest", + { + "plugins": [ + "@babel/plugin-transform-modules-commonjs" + ] + } + ] }, "transformIgnorePatterns": [ - "/node_modules/(?!(@stellar|@noble|@scure|stellar-base|bignumber.js|crc|tweetnacl|base32.js|uint8array-extras|@adraffy|@noble/curves)/)" + "^(?!.*/node_modules/(@noble|uint8array-extras)/).*/node_modules/" + ], + "collectCoverageFrom": [ + "**/*.(t|j)s" ], - "collectCoverageFrom": ["**/*.(t|j)s"], "coverageDirectory": "../coverage" } } diff --git a/scripts/regenerate-xdr-fixtures.ts b/scripts/regenerate-xdr-fixtures.ts index d2bd2e7..e205d93 100644 --- a/scripts/regenerate-xdr-fixtures.ts +++ b/scripts/regenerate-xdr-fixtures.ts @@ -19,6 +19,7 @@ import { encodeProvideCapitalArgs, encodeWithdrawCapitalArgs, } from "../src/stellar/scval-encoders"; +import { TrustTier, buildUpdateReadingArgs, encodeReadingForChain } from "../src/oracle/oracle-encoding"; const outDir = path.join(__dirname, "../src/stellar/xdr-fixtures"); @@ -439,6 +440,16 @@ writeB64("oracle-publish-feed.xdr.b64", feed); writeB64("oracle-publish-value.xdr.b64", value); writeB64("oracle-publish-timestamp.xdr.b64", ts); +const [, , provenance] = buildUpdateReadingArgs( + encodeReadingForChain("StablecoinDepeg", 0.9998, { + trustTier: TrustTier.Corroborated, + agreeingSources: 3, + decisionId: "0123456789abcdef", + sourceTimestamp: 1_700_000_000, + }) +); +writeB64("oracle-update-reading-provenance.xdr.b64", provenance, "u128 provenance: Corroborated, 3 sources, ts=1700000000"); + // silence unused import if tree-shaken oddly void Address; void nativeToScVal; diff --git a/src/app.module.ts b/src/app.module.ts index dab8f12..026c30d 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -29,6 +29,8 @@ import { AuthModule } from './auth/auth.module'; import { ApiKeyGuard } from './auth/api-key.guard'; import { SecurityReportsModule } from './security-reports/security-reports.module'; import { DigestModule } from './digest/digest.module'; +import { ReadingBusModule } from './oracle/reading-bus.module'; +import { OracleFeedModule } from './oracle/oracle-feed.module'; import { OracleIngestionModule } from './oracle/sources/oracle-ingestion.module'; import { OracleRegionsModule } from './oracle/regions/oracle-regions.module'; import { ReevaluationModule } from './claim/reevaluation/reevaluation.module'; @@ -87,6 +89,9 @@ export const isSandboxMode = (): boolean => // Global: per-adapter endpoint lists with health-based failover. ProviderEndpointsModule, OracleModule, + // Read side of the oracle: relays readings the worker publishes. + ReadingBusModule, + OracleFeedModule, OracleBootstrapModule, OracleAdminModule, ConsensusModule, diff --git a/src/claim/claim.scheduler.ts b/src/claim/claim.scheduler.ts index 1252d35..e6e9ea4 100644 --- a/src/claim/claim.scheduler.ts +++ b/src/claim/claim.scheduler.ts @@ -1,9 +1,13 @@ -import { Injectable, Logger, OnApplicationShutdown, OnModuleInit } from '@nestjs/common'; +import { Injectable, Logger, OnApplicationShutdown, OnModuleInit, Optional } from '@nestjs/common'; import { Cron, CronExpression } from '@nestjs/schedule'; import { ClaimService, ScanOutcome } from './claim.service'; import { ProtocolParametersService } from '../parameters/protocol-parameters.service'; +import { runsPipeline } from '../runtime/process-role'; +import { PipelineHeartbeat } from '../runtime/pipeline-heartbeat'; import { LeaderElector } from '../leader/leader-elector.service'; +export const CLAIM_PIPELINE_STAGE = 'claim-settlement'; + /** * Upper bound (ms) we are willing to wait for an in-flight settlement run to * finish before letting the process shut down. Kept in sync with the @@ -42,10 +46,16 @@ export class ClaimScheduler implements OnModuleInit, OnApplicationShutdown { constructor( private readonly claimService: ClaimService, private readonly protocolParameters: ProtocolParametersService, + @Optional() private readonly heartbeat?: PipelineHeartbeat, private readonly leaderElector: LeaderElector, ) {} async onModuleInit(): Promise { + if (!runsPipeline()) { + this.logger.log('Claim settlement disabled in this process (PROCESS_ROLE=api)'); + return; + } + this.heartbeat?.expect(CLAIM_PIPELINE_STAGE); const interval = await this.getIntervalSeconds(); this.logger.log(`Claim scheduler initialized with interval ${interval}s`); } @@ -53,6 +63,10 @@ export class ClaimScheduler implements OnModuleInit, OnApplicationShutdown { async onApplicationShutdown(signal?: string): Promise { this.shuttingDown = true; + if (!this.inFlight) { + return; + } + this.logger.log( `ClaimScheduler shutdown (${signal ?? 'unknown'}): waiting up to ${SHUTDOWN_DRAIN_TIMEOUT_MS}ms for in-flight settlement run to finish.`, ); @@ -103,6 +117,11 @@ export class ClaimScheduler implements OnModuleInit, OnApplicationShutdown { @Cron(CronExpression.EVERY_MINUTE) async handleClaimSettlement(): Promise { + // Settlement triggers payouts, so it only runs in the worker tier. + if (!runsPipeline()) { + return; + } + if (this.shuttingDown) { this.logger.log( 'Skipping claim settlement run: scheduler is shutting down.', @@ -150,6 +169,7 @@ export class ClaimScheduler implements OnModuleInit, OnApplicationShutdown { if (outcome.settled.length > 0) { this.logger.log(`Settled ${outcome.settled.length} claims this cycle`); } + this.heartbeat?.beat(CLAIM_PIPELINE_STAGE); } catch (error) { this.logger.error("Claim scan failed", error instanceof Error ? error.stack : String(error)); } diff --git a/src/main.ts b/src/main.ts index e69de29..a903a6f 100644 --- a/src/main.ts +++ b/src/main.ts @@ -0,0 +1,45 @@ +import { NestFactory } from '@nestjs/core'; +import { Logger, INestApplication } from '@nestjs/common'; +import { AppModule } from './app.module'; +import { resolveProcessRole } from './runtime/process-role'; +import { installGracefulShutdown } from './runtime/graceful-shutdown'; + +async function bootstrap(): Promise { + const logger = new Logger('Bootstrap'); + + // API entrypoint. With PROCESS_ROLE=api this process only serves reads and + // the WebSocket feed; the payout pipeline runs in src/worker.ts. Unset + // (= "all") keeps the old single-process behaviour for local development. + const role = resolveProcessRole(); + if (role === 'worker') { + throw new Error('PROCESS_ROLE=worker is for src/worker.ts; start the API with PROCESS_ROLE=api or all'); + } + if (role === 'all') { + logger.warn( + 'PROCESS_ROLE is not set: running the API and the payout pipeline in one process. ' + + 'In deployed environments set PROCESS_ROLE=api here and run the worker separately (npm run start:worker).', + ); + } + + const app: INestApplication = await NestFactory.create(AppModule); + + // Let Nest propagate shutdown signals to every provider that implements + // onApplicationShutdown / beforeApplicationShutdown (schedulers, DB pool, + // Redis client, etc.). + app.enableShutdownHooks(); + + const port = process.env.PORT ?? 3000; + await app.listen(port); + logger.log(`Application listening on port ${port} (PROCESS_ROLE=${role})`); + + // app.close() stops accepting new connections, drains in-flight HTTP + // requests, and invokes onApplicationShutdown on all providers so + // schedulers finish their current run and pools close cleanly. + installGracefulShutdown(() => app.close(), logger); +} + +bootstrap().catch((err) => { + // eslint-disable-next-line no-console + console.error('Fatal error during bootstrap', err); + process.exit(1); +}); diff --git a/src/metrics/metrics.service.ts b/src/metrics/metrics.service.ts index e69de29..768b0cd 100644 --- a/src/metrics/metrics.service.ts +++ b/src/metrics/metrics.service.ts @@ -0,0 +1,170 @@ +import { Injectable } from "@nestjs/common"; +import { Counter, Gauge, Histogram, Registry } from "@prometheus-io/client"; +import { NextFunction, Request, RequestHandler, Response } from "express"; +import type { RelayerMetricsSink } from "../stellar/relayer-monitor/relayer-balance-monitor.service"; +import { BalanceAssessment, FeeSpendSummary, tierRank } from "../stellar/relayer-monitor/relayer-balance.policy"; + +type Outcome = "success" | "failure"; +type OracleOutcome = Outcome | "mocked"; + +@Injectable() +export class MetricsService implements RelayerMetricsSink { + private readonly registry = new Registry(); + private readonly httpRequests = new Counter<"method" | "route" | "status_code">({ + name: "refract_http_requests_total", + help: "Total HTTP requests completed by the backend.", + labelNames: ["method", "route", "status_code"], + registers: [this.registry], + }); + private readonly httpRequestDuration = new Histogram<"method" | "route" | "status_code">({ + name: "refract_http_request_duration_seconds", + help: "HTTP request duration in seconds.", + labelNames: ["method", "route", "status_code"], + buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10], + registers: [this.registry], + }); + private readonly schedulerRuns = new Counter<"scheduler" | "outcome">({ + name: "refract_scheduler_runs_total", + help: "Completed scheduler runs, labeled by scheduler and outcome.", + labelNames: ["scheduler", "outcome"], + registers: [this.registry], + }); + private readonly schedulerDuration = new Histogram<"scheduler">({ + name: "refract_scheduler_run_duration_seconds", + help: "Scheduler run duration in seconds.", + labelNames: ["scheduler"], + buckets: [0.1, 0.5, 1, 2.5, 5, 10, 30, 60, 120, 300], + registers: [this.registry], + }); + private readonly sorobanRpcDuration = new Histogram<"operation" | "outcome">({ + name: "refract_soroban_rpc_duration_seconds", + help: "Soroban RPC call duration in seconds, labeled by method and outcome.", + labelNames: ["operation", "outcome"], + buckets: [0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30], + registers: [this.registry], + }); + private readonly oracleChecks = new Counter<"coverage_type" | "outcome">({ + name: "refract_oracle_checks_total", + help: "Oracle checks completed, labeled by coverage type and outcome.", + labelNames: ["coverage_type", "outcome"], + registers: [this.registry], + }); + private readonly oracleCheckDuration = new Histogram<"coverage_type">({ + name: "refract_oracle_check_duration_seconds", + help: "Oracle check duration in seconds.", + labelNames: ["coverage_type"], + buckets: [0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30], + registers: [this.registry], + }); + + private readonly relayerBalance = new Gauge({ + name: "refract_relayer_balance_stroops", + help: "Relayer native balance in stroops.", + registers: [this.registry], + }); + private readonly relayerSpendable = new Gauge({ + name: "refract_relayer_spendable_stroops", + help: "Relayer balance available for fees after reserve and liabilities, in stroops.", + registers: [this.registry], + }); + private readonly relayerRunwayHours = new Gauge({ + name: "refract_relayer_runway_hours", + help: "Hours of fees the relayer can still pay at the current spend rate (-1 when unknown).", + registers: [this.registry], + }); + private readonly relayerRunwaySubmissions = new Gauge({ + name: "refract_relayer_runway_submissions", + help: "Submissions the relayer can still pay for at the average fee (-1 when unknown).", + registers: [this.registry], + }); + private readonly relayerFeeRate = new Gauge({ + name: "refract_relayer_fee_spend_per_hour_stroops", + help: "Relayer fee spend per hour over the monitoring window, in stroops.", + registers: [this.registry], + }); + private readonly relayerAlertTier = new Gauge({ + name: "refract_relayer_alert_tier", + help: "Relayer balance alert tier: 0=ok, 1=warning, 2=critical, 3=exhausted.", + registers: [this.registry], + }); + private readonly relayerFeesTotal = new Counter<"outcome">({ + name: "refract_relayer_fees_paid_stroops_total", + help: "Fees charged to the relayer, in stroops, by transaction outcome.", + labelNames: ["outcome"], + registers: [this.registry], + }); + private readonly relayerTxFee = new Histogram({ + name: "refract_relayer_tx_fee_stroops", + help: "Fee charged per relayer-submitted transaction, in stroops.", + buckets: [100, 1_000, 10_000, 50_000, 100_000, 500_000, 1_000_000, 5_000_000, 10_000_000], + registers: [this.registry], + }); + + readonly httpMiddleware: RequestHandler = (req: Request, res: Response, next: NextFunction): void => { + if (req.path === "/metrics") { + next(); + return; + } + + const startedAt = process.hrtime.bigint(); + res.once("finish", () => { + const route = typeof req.route?.path === "string" ? req.route.path : "unmatched"; + const labels = { + method: req.method, + route, + status_code: String(res.statusCode), + }; + const durationSeconds = Number(process.hrtime.bigint() - startedAt) / 1_000_000_000; + this.httpRequests.labels(labels).inc(); + this.httpRequestDuration.labels(labels).observe(durationSeconds); + }); + next(); + }; + + recordSchedulerRun(scheduler: string, outcome: Outcome, durationSeconds: number): void { + this.schedulerRuns.labels({ scheduler, outcome }).inc(); + this.schedulerDuration.labels({ scheduler }).observe(durationSeconds); + } + + recordOracleCheck(coverageType: string, outcome: OracleOutcome, durationSeconds: number): void { + this.oracleChecks.labels({ coverage_type: coverageType, outcome }).inc(); + this.oracleCheckDuration.labels({ coverage_type: coverageType }).observe(durationSeconds); + } + + recordRelayerBalance(a: BalanceAssessment, spend: FeeSpendSummary): void { + this.relayerBalance.set(Number(a.balanceStroops)); + this.relayerSpendable.set(Number(a.spendableStroops)); + this.relayerRunwayHours.set(a.runwayHours ?? -1); + this.relayerRunwaySubmissions.set(a.runwaySubmissions ?? -1); + this.relayerFeeRate.set(spend.ratePerHourStroops); + this.relayerAlertTier.set(tierRank(a.tier)); + } + + recordRelayerFee(feeStroops: bigint, successful: boolean): void { + const fee = Number(feeStroops); + this.relayerFeesTotal.labels({ outcome: successful ? "success" : "failure" }).inc(fee); + this.relayerTxFee.observe(fee); + } + + async observeSorobanRpc(operation: string, call: () => Promise): Promise { + const startedAt = process.hrtime.bigint(); + let outcome: Outcome = "success"; + try { + return await call(); + } catch (error) { + outcome = "failure"; + throw error; + } finally { + const durationSeconds = Number(process.hrtime.bigint() - startedAt) / 1_000_000_000; + this.sorobanRpcDuration.labels({ operation, outcome }).observe(durationSeconds); + } + } + + metrics(): Promise { + return this.registry.metrics(); + } + + get contentType(): string { + return this.registry.contentType; + } +} diff --git a/src/oracle/consensus/consensus.property.spec.ts b/src/oracle/consensus/consensus.property.spec.ts new file mode 100644 index 0000000..66ce35c --- /dev/null +++ b/src/oracle/consensus/consensus.property.spec.ts @@ -0,0 +1,488 @@ +import fc from "fast-check"; +import { + ConsensusDecision, + ConsensusPolicy, + SourceObservation, + TrustTier, + deviationTolerance, + evaluateConsensus, + isAnomalous, + median, + meetsQuorum, +} from "./consensus"; + +/** + * Property-based and fuzz tests for the consensus math. + * + * Each property states an invariant that must hold for every input, and + * fast-check searches for a counterexample (including adversarial ones: + * NaN, Infinity, huge magnitudes, duplicated ids, colluding outliers). + * On failure fast-check prints the seed and a shrunk counterexample; rerun + * with FC_SEED= to reproduce. + * + * FC_NUM_RUNS raises the run count for deeper soak runs, e.g. + * FC_NUM_RUNS=20000 npm run test:property + */ +const numRuns = Number(process.env.FC_NUM_RUNS ?? 300); +const seed = process.env.FC_SEED ? Number(process.env.FC_SEED) : undefined; +fc.configureGlobal({ numRuns, ...(seed !== undefined ? { seed } : {}) }); + +const NOW = 1_800_000_000_000; + +// ---------------------------------------------------------------- arbitraries + +/** Finite doubles across a wide but non-overflowing range. */ +const finite = fc.double({ min: -1e12, max: 1e12, noNaN: true, noDefaultInfinity: true }); + +/** Integers small enough that sums and products stay exact in a double. */ +const exactInt = fc.integer({ min: -1_000_000, max: 1_000_000 }); + +/** Any number at all, including the values that break naive math. */ +const hostileNumber = fc.oneof( + finite, + fc.constantFrom(NaN, Infinity, -Infinity, -0, Number.MAX_VALUE, -Number.MAX_VALUE, Number.MIN_VALUE), + fc.double() +); + +const validPolicy: fc.Arbitrary = fc.record({ + quorumRatio: fc.double({ min: 0.01, max: 1, noNaN: true }), + minAgreeingSources: fc.integer({ min: 1, max: 5 }), + maxDeviationRatio: fc.double({ min: 0, max: 0.5, noNaN: true }), + absoluteDeviationFloor: fc.double({ min: 0, max: 10, noNaN: true }), + maxStalenessMs: fc.integer({ min: 0, max: 600_000 }), + maxFutureSkewMs: fc.integer({ min: 0, max: 60_000 }), +}); + +/** Well-formed observation with a unique-ish id drawn from a small pool. */ +function wellFormedObs(ids: fc.Arbitrary): fc.Arbitrary { + return fc.record({ + sourceId: ids, + value: finite, + weight: fc.integer({ min: 0, max: 1_000 }), + observedAt: fc.integer({ min: NOW - 900_000, max: NOW + 90_000 }), + }); +} + +/** Observation where every field may be garbage. */ +const hostileObs: fc.Arbitrary = fc.record({ + sourceId: fc.oneof(fc.constantFrom("a", "b", "c", "d", ""), fc.string({ maxLength: 4 })), + value: hostileNumber, + weight: hostileNumber, + observedAt: fc.oneof(fc.integer({ min: NOW - 900_000, max: NOW + 90_000 }), hostileNumber), +}); + +/** Uniquely-identified, fresh, well-formed observations. */ +function freshUniqueObs( + values: fc.Arbitrary, + weights: fc.Arbitrary = fc.integer({ min: 1, max: 100 }), + { minLength = 1, maxLength = 12 } = {} +): fc.Arbitrary { + return fc + .array(fc.record({ value: values, weight: weights, age: fc.integer({ min: 0, max: 1_000 }) }), { + minLength, + maxLength, + }) + .map((rows) => rows.map((r, i) => ({ sourceId: `s${i}`, value: r.value, weight: r.weight, observedAt: NOW - r.age }))); +} + +/** A permutation of `xs`, chosen by fast-check so it shrinks well. */ +function permuted(xs: T[]): fc.Arbitrary { + return fc + .array(fc.nat(), { minLength: xs.length, maxLength: xs.length }) + .map((keys) => xs.map((x, i) => ({ x, k: keys[i], i })).sort((p, q) => p.k - q.k || p.i - q.i).map((p) => p.x)); +} + +function sortedLists(d: ConsensusDecision) { + return [d.agreeingSources, d.outlierSources, d.staleSources, d.invalidSources]; +} + +// ---------------------------------------------------------------- median + +describe("median — properties", () => { + it("lies between the minimum and maximum", () => { + fc.assert( + fc.property(fc.array(finite, { minLength: 1 }), (xs) => { + const m = median(xs); + return m >= Math.min(...xs) && m <= Math.max(...xs); + }) + ); + }); + + it("does not depend on input order", () => { + fc.assert( + fc.property( + fc.array(finite, { minLength: 1 }).chain((xs) => fc.tuple(fc.constant(xs), permuted(xs))), + ([xs, ys]) => Object.is(median(xs), median(ys)) + ) + ); + }); + + it("is an element of the input for odd lengths", () => { + fc.assert( + fc.property( + fc.array(finite, { minLength: 1 }).filter((xs) => xs.length % 2 === 1), + (xs) => xs.includes(median(xs)) + ) + ); + }); + + it("splits the input: at most half the values lie strictly on either side", () => { + fc.assert( + fc.property(fc.array(finite, { minLength: 1 }), (xs) => { + const m = median(xs); + const below = xs.filter((x) => x < m).length; + const above = xs.filter((x) => x > m).length; + return below <= xs.length / 2 && above <= xs.length / 2; + }) + ); + }); + + it("never decreases when one input increases", () => { + fc.assert( + fc.property( + fc.array(exactInt, { minLength: 1 }), + fc.nat(), + fc.integer({ min: 0, max: 1_000_000 }), + (xs, idx, bump) => { + const ys = [...xs]; + ys[idx % ys.length] += bump; + return median(ys) >= median(xs); + } + ) + ); + }); + + it("shifts exactly with a constant offset (exact integer arithmetic)", () => { + fc.assert( + fc.property(fc.array(exactInt, { minLength: 1 }), exactInt, (xs, c) => { + return median(xs.map((x) => x + c)) === median(xs) + c; + }) + ); + }); + + it("stays inside the honest range while honest values outnumber adversarial ones", () => { + fc.assert( + fc.property( + fc.array(finite, { minLength: 1, maxLength: 20 }), + fc.array(fc.oneof(finite, fc.constantFrom(Number.MAX_VALUE, -Number.MAX_VALUE)), { maxLength: 19 }), + (honest, adversarial) => { + fc.pre(adversarial.length < honest.length); + const m = median([...honest, ...adversarial]); + return m >= Math.min(...honest) && m <= Math.max(...honest); + } + ) + ); + }); + + it("rejects any non-finite input instead of returning garbage", () => { + fc.assert( + fc.property( + fc.array(finite), + fc.constantFrom(NaN, Infinity, -Infinity), + fc.nat(), + (xs, bad, at) => { + const ys = [...xs]; + ys.splice(at % (ys.length + 1), 0, bad); + expect(() => median(ys)).toThrow(RangeError); + } + ) + ); + }); +}); + +// ---------------------------------------------------------------- weighted quorum + +describe("weighted quorum — properties", () => { + const weightPair = fc + .tuple(fc.integer({ min: 0, max: 1_000_000 }), fc.integer({ min: 1, max: 1_000_000 })) + .map(([a, extra]) => ({ agreeing: a, total: a + extra - 1 })) + .filter(({ total }) => total > 0); + const ratio = fc.double({ min: 0.001, max: 1, noNaN: true }); + + it("does not change when every weight is scaled by the same factor", () => { + fc.assert( + fc.property(weightPair, ratio, fc.integer({ min: 1, max: 1_000_000 }), ({ agreeing, total }, r, k) => { + return meetsQuorum(agreeing, total, r) === meetsQuorum(agreeing * k, total * k, r); + }) + ); + }); + + it("is monotone: more agreeing weight never loses quorum", () => { + fc.assert( + fc.property(weightPair, ratio, fc.nat(), ({ agreeing, total }, r, extra) => { + const more = Math.min(agreeing + extra, total); + return !meetsQuorum(agreeing, total, r) || meetsQuorum(more, total, r); + }) + ); + }); + + it("is monotone: a lower bar never loses quorum", () => { + fc.assert( + fc.property(weightPair, ratio, ratio, ({ agreeing, total }, r1, r2) => { + const [lo, hi] = r1 <= r2 ? [r1, r2] : [r2, r1]; + return !meetsQuorum(agreeing, total, hi) || meetsQuorum(agreeing, total, lo); + }) + ); + }); + + it("always passes on full agreement and never on zero agreement", () => { + fc.assert( + fc.property(fc.integer({ min: 1, max: 1_000_000 }), ratio, (total, r) => { + return meetsQuorum(total, total, r) && !meetsQuorum(0, total, r); + }) + ); + }); + + it("never passes when total weight is zero, negative or non-finite", () => { + fc.assert( + fc.property(hostileNumber, fc.oneof(fc.constantFrom(0, -0, NaN, Infinity, -Infinity), fc.double({ max: 0 })), ratio, (a, t, r) => { + return !meetsQuorum(a, t, r); + }) + ); + }); +}); + +// ---------------------------------------------------------------- anomaly threshold + +describe("anomaly threshold — properties", () => { + it("never flags the center itself", () => { + fc.assert(fc.property(finite, validPolicy, (c, p) => !isAnomalous(c, c, p))); + }); + + it("is symmetric around the center", () => { + fc.assert( + fc.property(exactInt, fc.integer({ min: 0, max: 1_000_000 }), validPolicy, (c, d, p) => { + return isAnomalous(c + d, c, p) === isAnomalous(c - d, c, p); + }) + ); + }); + + it("is monotone in distance: further from the center never becomes less anomalous", () => { + fc.assert( + fc.property(finite, fc.double({ min: 0, max: 1e6, noNaN: true }), fc.double({ min: 0, max: 1e6, noNaN: true }), validPolicy, (c, d1, d2, p) => { + const [near, far] = d1 <= d2 ? [d1, d2] : [d2, d1]; + return !isAnomalous(c + near, c, p) || isAnomalous(c + far, c, p) || c + near === c + far; + }) + ); + }); + + it("widening the tolerance never turns an agreeing value into an outlier", () => { + fc.assert( + fc.property(finite, finite, validPolicy, fc.double({ min: 0, max: 1, noNaN: true }), (v, c, p, widen) => { + const wider = { ...p, maxDeviationRatio: p.maxDeviationRatio + widen, absoluteDeviationFloor: p.absoluteDeviationFloor + widen }; + return isAnomalous(v, c, p) || !isAnomalous(v, c, wider); + }) + ); + }); +}); + +// ---------------------------------------------------------------- evaluateConsensus + +describe("evaluateConsensus — fuzz", () => { + it("never throws on hostile observations and always returns a coherent decision", () => { + fc.assert( + fc.property(fc.array(hostileObs, { maxLength: 15 }), validPolicy, (xs, p) => { + const d = evaluateConsensus(xs, p, NOW); + + // Shape + expect(d.decisionId).toMatch(/^[0-9a-f]{16}$/); + expect(Number.isFinite(d.totalWeight) || d.totalWeight === Infinity).toBe(true); + + // Every distinct id lands in exactly one bucket. + const buckets = sortedLists(d).flat(); + expect(new Set(buckets).size).toBe(buckets.length); + expect(new Set(buckets)).toEqual(new Set(xs.map((o) => String(o.sourceId)))); + + // Agreed ⇔ a finite value is published. + if (d.status === "agreed") { + expect(Number.isFinite(d.value)).toBe(true); + expect(d.trustTier).not.toBe(TrustTier.Unverified); + expect(Number.isFinite(d.sourceTimestamp)).toBe(true); + } else { + expect(d.value).toBeNull(); + expect(d.trustTier).toBe(TrustTier.Unverified); + expect(d.sourceTimestamp).toBeNull(); + } + }) + ); + }); + + it("only agrees when the source count and weighted quorum are both met", () => { + fc.assert( + fc.property(fc.array(hostileObs, { maxLength: 15 }), validPolicy, (xs, p) => { + const d = evaluateConsensus(xs, p, NOW); + if (d.status !== "agreed") return true; + return d.agreeingSources.length >= p.minAgreeingSources && d.agreeingWeight / d.totalWeight >= p.quorumRatio; + }) + ); + }); + + it("never lets a source vote twice", () => { + fc.assert( + fc.property(fc.array(wellFormedObs(fc.constantFrom("a", "b", "c")), { maxLength: 10 }), validPolicy, (xs, p) => { + const d = evaluateConsensus(xs, p, NOW); + const counts = new Map(); + for (const o of xs) counts.set(o.sourceId, (counts.get(o.sourceId) ?? 0) + 1); + for (const [id, n] of counts) { + if (n > 1 && !d.invalidSources.includes(id)) return false; + } + return true; + }) + ); + }); +}); + +describe("evaluateConsensus — properties", () => { + it("does not depend on observation order", () => { + fc.assert( + fc.property( + fc.array(hostileObs, { maxLength: 12 }).chain((xs) => fc.tuple(fc.constant(xs), permuted(xs))), + validPolicy, + ([xs, ys], p) => { + const a = evaluateConsensus(xs, p, NOW); + const b = evaluateConsensus(ys, p, NOW); + expect(b).toEqual(a); + } + ) + ); + }); + + it("is deterministic", () => { + fc.assert( + fc.property(fc.array(hostileObs, { maxLength: 12 }), validPolicy, (xs, p) => { + expect(evaluateConsensus(xs, p, NOW)).toEqual(evaluateConsensus(xs, p, NOW)); + }) + ); + }); + + it("classifies agreeing and outlier sources by the tolerance around the center", () => { + fc.assert( + fc.property(freshUniqueObs(finite), validPolicy, (xs, p) => { + const d = evaluateConsensus(xs, { ...p, maxStalenessMs: 10_000 }, NOW); + const byId = new Map(xs.map((o) => [o.sourceId, o.value])); + const tol = deviationTolerance(d.center!, p); + for (const id of d.agreeingSources) expect(Math.abs(byId.get(id)! - d.center!)).toBeLessThanOrEqual(tol); + for (const id of d.outlierSources) expect(Math.abs(byId.get(id)! - d.center!)).toBeGreaterThan(tol); + }) + ); + }); + + it("publishes a value inside the range of the agreeing sources", () => { + fc.assert( + fc.property(freshUniqueObs(finite), validPolicy, (xs, p) => { + const d = evaluateConsensus(xs, { ...p, maxStalenessMs: 10_000 }, NOW); + if (d.status !== "agreed") return true; + const vals = xs.filter((o) => d.agreeingSources.includes(o.sourceId)).map((o) => o.value); + return d.value! >= Math.min(...vals) && d.value! <= Math.max(...vals); + }) + ); + }); + + it("never treats a stale observation as agreeing", () => { + fc.assert( + fc.property(fc.array(wellFormedObs(fc.string({ minLength: 1, maxLength: 3 })), { maxLength: 12 }), validPolicy, (xs, p) => { + const d = evaluateConsensus(xs, p, NOW); + return xs.every((o) => !(NOW - o.observedAt > p.maxStalenessMs) || !d.agreeingSources.includes(o.sourceId)); + }) + ); + }); + + it("bounds adversarial influence: a minority cannot move the value outside honest range ± tolerance", () => { + fc.assert( + fc.property( + freshUniqueObs(finite, fc.integer({ min: 1, max: 100 }), { minLength: 1, maxLength: 10 }), + fc.array(hostileNumber, { maxLength: 9 }), + validPolicy, + (honest, adversarialValues, p) => { + fc.pre(adversarialValues.length < honest.length); + const adversaries = adversarialValues.map((value, i) => ({ + sourceId: `adv${i}`, + value, + weight: 100, + observedAt: NOW, + })); + const d = evaluateConsensus([...honest, ...adversaries], { ...p, maxStalenessMs: 10_000 }, NOW); + if (d.status !== "agreed") return true; + const hv = honest.map((o) => o.value); + const lo = Math.min(...hv); + const hi = Math.max(...hv); + // Center sits inside the honest range; every agreeing value is + // within tolerance of the center, so is the median of them. + const tol = deviationTolerance(d.center!, p); + return d.center! >= lo && d.center! <= hi && d.value! >= lo - tol && d.value! <= hi + tol; + } + ) + ); + }); + + it("an honest, tightly clustered majority with quorum weight always agrees and rejects distant adversaries", () => { + fc.assert( + fc.property( + exactInt, + fc.integer({ min: 2, max: 10 }), + fc.integer({ min: 0, max: 9 }), + fc.double({ min: 1, max: 100, noNaN: true }), + fc.array(fc.double({ min: -0.25, max: 0.25, noNaN: true }), { minLength: 10, maxLength: 10 }), + fc.array(fc.oneof(fc.double({ min: 3, max: 1e9, noNaN: true }), fc.double({ min: -1e9, max: -3, noNaN: true })), { + minLength: 9, + maxLength: 9, + }), + (center, honestCount, advCount, floor, jitter, advOffsets) => { + fc.pre(advCount < honestCount); + // Purely absolute tolerance; honest spread is ±floor/4, adversaries + // sit at least 3·floor away from the cluster. + const p: ConsensusPolicy = { + quorumRatio: honestCount / (honestCount + advCount), + minAgreeingSources: 2, + maxDeviationRatio: 0, + absoluteDeviationFloor: floor, + maxStalenessMs: 60_000, + maxFutureSkewMs: 0, + }; + const honest = Array.from({ length: honestCount }, (_, i) => ({ + sourceId: `h${i}`, + value: center + jitter[i] * floor, + weight: 1, + observedAt: NOW, + })); + const adv = Array.from({ length: advCount }, (_, i) => ({ + sourceId: `x${i}`, + value: center + advOffsets[i] * floor, + weight: 1, + observedAt: NOW, + })); + const d = evaluateConsensus([...honest, ...adv], p, NOW); + expect(d.status).toBe("agreed"); + expect(d.agreeingSources).toEqual(honest.map((o) => o.sourceId).sort()); + expect(d.outlierSources).toEqual(adv.map((o) => o.sourceId).sort()); + const hv = honest.map((o) => o.value); + expect(d.value!).toBeGreaterThanOrEqual(Math.min(...hv)); + expect(d.value!).toBeLessThanOrEqual(Math.max(...hv)); + } + ) + ); + }); + + it("assigns trust tiers consistently with the agreeing set", () => { + fc.assert( + fc.property(fc.array(hostileObs, { maxLength: 12 }), validPolicy, (xs, p) => { + const d = evaluateConsensus(xs, p, NOW); + const n = d.agreeingSources.length; + switch (d.trustTier) { + case TrustTier.Unverified: + return d.status !== "agreed"; + case TrustTier.SingleSource: + return d.status === "agreed" && n === 1; + case TrustTier.Corroborated: + return d.status === "agreed" && n >= 2; + case TrustTier.Unanimous: + return ( + d.status === "agreed" && + n >= 3 && + d.outlierSources.length + d.staleSources.length + d.invalidSources.length === 0 + ); + } + }) + ); + }); +}); diff --git a/src/oracle/consensus/consensus.spec.ts b/src/oracle/consensus/consensus.spec.ts new file mode 100644 index 0000000..2f9d4b0 --- /dev/null +++ b/src/oracle/consensus/consensus.spec.ts @@ -0,0 +1,178 @@ +import { + ConsensusPolicy, + DEFAULT_CONSENSUS_POLICY, + SourceObservation, + TrustTier, + deviationTolerance, + evaluateConsensus, + isAnomalous, + median, + meetsQuorum, + validateConsensusPolicy, +} from "./consensus"; + +const NOW = 1_800_000_000_000; + +function obs(sourceId: string, value: number, overrides: Partial = {}): SourceObservation { + return { sourceId, value, weight: 1, observedAt: NOW - 1_000, ...overrides }; +} + +const policy: ConsensusPolicy = { ...DEFAULT_CONSENSUS_POLICY }; + +describe("median", () => { + it("returns the middle element for odd lengths", () => { + expect(median([3, 1, 2])).toBe(2); + }); + + it("averages the middle pair for even lengths", () => { + expect(median([4, 1, 3, 2])).toBe(2.5); + }); + + it("does not overflow on large values", () => { + expect(median([Number.MAX_VALUE, Number.MAX_VALUE])).toBe(Number.MAX_VALUE); + }); + + it("stays inside the input range for subnormal values (found by property test)", () => { + expect(median([5e-324, 5e-324, 5e-324, 5e-324])).toBe(5e-324); + }); + + it("rejects empty and non-finite input", () => { + expect(() => median([])).toThrow(RangeError); + expect(() => median([1, NaN])).toThrow(RangeError); + expect(() => median([Infinity])).toThrow(RangeError); + }); +}); + +describe("meetsQuorum", () => { + it("is inclusive at the boundary", () => { + expect(meetsQuorum(2, 3, 2 / 3)).toBe(true); + expect(meetsQuorum(1.99, 3, 2 / 3)).toBe(false); + }); + + it("never passes with zero or invalid total weight", () => { + expect(meetsQuorum(0, 0, 0.5)).toBe(false); + expect(meetsQuorum(1, NaN, 0.5)).toBe(false); + expect(meetsQuorum(1, Infinity, 0.5)).toBe(false); + }); +}); + +describe("anomaly threshold", () => { + it("accepts values exactly at the tolerance and rejects values past it", () => { + const p = { ...policy, maxDeviationRatio: 0.5, absoluteDeviationFloor: 0 }; + expect(deviationTolerance(10, p)).toBe(5); + expect(isAnomalous(15, 10, p)).toBe(false); + expect(isAnomalous(15.000001, 10, p)).toBe(true); + }); + + it("falls back to the absolute floor around zero", () => { + const p = { ...policy, maxDeviationRatio: 0.02, absoluteDeviationFloor: 0.5 }; + expect(isAnomalous(0.4, 0, p)).toBe(false); + expect(isAnomalous(0.6, 0, p)).toBe(true); + }); +}); + +describe("validateConsensusPolicy", () => { + it.each<[string, Partial]>([ + ["zero quorum", { quorumRatio: 0 }], + ["quorum above one", { quorumRatio: 1.01 }], + ["fractional min sources", { minAgreeingSources: 1.5 }], + ["negative deviation", { maxDeviationRatio: -1 }], + ["NaN staleness", { maxStalenessMs: NaN }], + ])("rejects %s", (_label, override) => { + expect(() => validateConsensusPolicy({ ...policy, ...override })).toThrow(RangeError); + }); +}); + +describe("evaluateConsensus", () => { + it("agrees on the median of agreeing sources and drops the outlier", () => { + const d = evaluateConsensus( + [obs("a", 0.9990), obs("b", 1.0000), obs("c", 1.0010), obs("evil", 0.5)], + { ...policy, quorumRatio: 0.5 }, + NOW + ); + expect(d.status).toBe("agreed"); + expect(d.value).toBeCloseTo(1.0, 10); + expect(d.agreeingSources).toEqual(["a", "b", "c"]); + expect(d.outlierSources).toEqual(["evil"]); + expect(d.trustTier).toBe(TrustTier.Corroborated); + expect(d.decisionId).toMatch(/^[0-9a-f]{16}$/); + }); + + it("reports Unanimous only when every source is fresh, valid and agreeing", () => { + const d = evaluateConsensus([obs("a", 1), obs("b", 1), obs("c", 1)], policy, NOW); + expect(d.trustTier).toBe(TrustTier.Unanimous); + }); + + it("reports SingleSource when policy allows one source", () => { + const d = evaluateConsensus([obs("a", 1)], { ...policy, minAgreeingSources: 1 }, NOW); + expect(d.status).toBe("agreed"); + expect(d.trustTier).toBe(TrustTier.SingleSource); + }); + + it("refuses a single source under the default policy", () => { + const d = evaluateConsensus([obs("a", 1)], policy, NOW); + expect(d.status).toBe("insufficient_sources"); + expect(d.value).toBeNull(); + expect(d.trustTier).toBe(TrustTier.Unverified); + }); + + it("fails quorum when agreeing weight is too small, even with more agreeing sources", () => { + const d = evaluateConsensus( + [obs("a", 1, { weight: 1 }), obs("b", 1, { weight: 1 }), obs("whale", 2, { weight: 10 })], + { ...policy, quorumRatio: 0.5 }, + NOW + ); + // Median of [1, 1, 2] is 1, so the whale is the outlier — but it still + // holds most of the weight, so the honest pair cannot reach quorum. + expect(d.outlierSources).toEqual(["whale"]); + expect(d.status).toBe("no_quorum"); + }); + + it("marks stale and future-dated observations", () => { + const d = evaluateConsensus( + [ + obs("a", 1), + obs("b", 1), + obs("old", 1, { observedAt: NOW - policy.maxStalenessMs - 1 }), + obs("future", 1, { observedAt: NOW + policy.maxFutureSkewMs + 1 }), + ], + { ...policy, quorumRatio: 0.5 }, + NOW + ); + expect(d.staleSources).toEqual(["old"]); + expect(d.invalidSources).toEqual(["future"]); + expect(d.status).toBe("agreed"); + expect(d.trustTier).toBe(TrustTier.Corroborated); + }); + + it("invalidates every copy of a duplicated source id", () => { + const d = evaluateConsensus([obs("a", 1), obs("a", 1), obs("b", 1)], policy, NOW); + expect(d.invalidSources).toEqual(["a"]); + expect(d.agreeingSources).toEqual(["b"]); + expect(d.status).toBe("insufficient_sources"); + }); + + it("counts invalid sources in the quorum denominator", () => { + const d = evaluateConsensus( + [obs("a", 1), obs("b", 1), obs("bad1", NaN), obs("bad2", NaN)], + { ...policy, quorumRatio: 0.75 }, + NOW + ); + expect(d.totalWeight).toBe(4); + expect(d.status).toBe("no_quorum"); + }); + + it("uses the earliest agreeing timestamp as the source timestamp", () => { + const d = evaluateConsensus([obs("a", 1, { observedAt: NOW - 5_000 }), obs("b", 1)], policy, NOW); + expect(d.sourceTimestamp).toBe(NOW - 5_000); + }); + + it("produces the same decision id for the same inputs in any order", () => { + const xs = [obs("a", 1), obs("b", 1.001), obs("c", 0.999)]; + const d1 = evaluateConsensus(xs, policy, NOW); + const d2 = evaluateConsensus([...xs].reverse(), policy, NOW); + expect(d2.decisionId).toBe(d1.decisionId); + const d3 = evaluateConsensus(xs, policy, NOW + 1); + expect(d3.decisionId).not.toBe(d1.decisionId); + }); +}); diff --git a/src/oracle/consensus/consensus.ts b/src/oracle/consensus/consensus.ts new file mode 100644 index 0000000..71f147a --- /dev/null +++ b/src/oracle/consensus/consensus.ts @@ -0,0 +1,329 @@ +import { createHash } from "crypto"; + +/** + * Multi-source oracle consensus. + * + * Every payout decision rests on the numbers produced here, so this module is + * deliberately pure: no I/O, no clock reads (callers pass `now`), no Nest + * dependencies. That keeps it easy to exercise with property-based tests + * (see consensus.property.spec.ts). + * + * Pipeline for one evaluation: + * 1. Screen each observation: malformed or duplicated sources are + * `invalid`, old ones are `stale`. + * 2. Take the median of the fresh values as the screening center. + * 3. Sources further than the deviation tolerance from the center are + * `outlier`s; the rest are `agreeing`. + * 4. Agreement needs enough agreeing sources and enough agreeing weight + * (weighted quorum over every source that reported). + * 5. The published value is the median of the agreeing values. + */ + +/** One source's report for a single coverage type. */ +export interface SourceObservation { + /** Stable identifier of the data source (e.g. "coingecko"). */ + sourceId: string; + /** The reported value, in the coverage type's natural unit. */ + value: number; + /** Voting weight. Must be finite and >= 0. */ + weight: number; + /** When the source observed the value (epoch milliseconds). */ + observedAt: number; +} + +export interface ConsensusPolicy { + /** Fraction (0, 1] of total reporting weight that must agree. */ + quorumRatio: number; + /** Minimum number of agreeing sources (>= 1). */ + minAgreeingSources: number; + /** Allowed deviation from the center, relative to |center| (>= 0). */ + maxDeviationRatio: number; + /** + * Absolute deviation that is always tolerated (>= 0). Needed when the + * center is near zero (e.g. a 0% market move), where a purely relative + * tolerance would reject every source. + */ + absoluteDeviationFloor: number; + /** Observations older than this are stale (ms, >= 0). */ + maxStalenessMs: number; + /** Observations further in the future than this are invalid (ms, >= 0). */ + maxFutureSkewMs: number; +} + +export const DEFAULT_CONSENSUS_POLICY: Readonly = Object.freeze({ + quorumRatio: 2 / 3, + minAgreeingSources: 2, + maxDeviationRatio: 0.02, + absoluteDeviationFloor: 0, + maxStalenessMs: 5 * 60_000, + maxFutureSkewMs: 30_000, +}); + +/** + * How much the published value can be trusted. The numeric values are part + * of the on-chain provenance encoding (see oracle-encoding.ts) — never + * renumber them. + */ +export enum TrustTier { + /** No consensus; must never be published as authoritative. */ + Unverified = 0, + /** Exactly one source agreed (only possible when minAgreeingSources = 1). */ + SingleSource = 1, + /** Two or more sources agreed and quorum was met. */ + Corroborated = 2, + /** Three or more sources, all fresh, valid and in agreement. */ + Unanimous = 3, +} + +export type ConsensusStatus = "agreed" | "no_valid_sources" | "insufficient_sources" | "no_quorum"; + +export interface ConsensusDecision { + status: ConsensusStatus; + /** Median of agreeing values; null unless status is "agreed". */ + value: number | null; + trustTier: TrustTier; + /** Deterministic 64-bit identifier of this decision, as 16 hex chars. */ + decisionId: string; + /** Median of fresh values used to screen outliers; null if none were fresh. */ + center: number | null; + /** Source ids by disposition. Each list is sorted and duplicate-free. */ + agreeingSources: string[]; + outlierSources: string[]; + staleSources: string[]; + invalidSources: string[]; + agreeingWeight: number; + /** Weight of every distinct source that reported, valid or not. */ + totalWeight: number; + /** Earliest observedAt among agreeing sources (ms); null unless agreed. */ + sourceTimestamp: number | null; + evaluatedAt: number; +} + +/** Median of a non-empty list of finite numbers. Mean of the middle pair for even lengths. */ +export function median(values: readonly number[]): number { + if (values.length === 0) { + throw new RangeError("median() of an empty list"); + } + for (const v of values) { + if (!Number.isFinite(v)) { + throw new RangeError(`median() requires finite numbers, got ${v}`); + } + } + const sorted = [...values].sort((a, b) => a - b); + const mid = sorted.length >> 1; + if (sorted.length % 2 === 1) { + return sorted[mid]; + } + const lo = sorted[mid - 1]; + const hi = sorted[mid]; + // (lo + hi) / 2 is correctly rounded, so it stays inside [lo, hi] even for + // subnormals. Only when the sum overflows do we halve first — at that + // magnitude halving is exact. + const sum = lo + hi; + return Number.isFinite(sum) ? sum / 2 : lo / 2 + hi / 2; +} + +/** Largest distance from `center` that still counts as agreement. */ +export function deviationTolerance(center: number, policy: ConsensusPolicy): number { + return Math.max(policy.maxDeviationRatio * Math.abs(center), policy.absoluteDeviationFloor); +} + +export function isAnomalous(value: number, center: number, policy: ConsensusPolicy): boolean { + return Math.abs(value - center) > deviationTolerance(center, policy); +} + +/** + * Weighted quorum check. Uses a division rather than `ratio * total` so the + * result does not change when every weight is scaled by the same factor. + */ +export function meetsQuorum(agreeingWeight: number, totalWeight: number, quorumRatio: number): boolean { + if (!(totalWeight > 0) || !Number.isFinite(totalWeight) || !(agreeingWeight >= 0)) { + return false; + } + return agreeingWeight / totalWeight >= quorumRatio; +} + +/** Throws if the policy is unusable. Policies are operator config, so failing loudly is correct. */ +export function validateConsensusPolicy(policy: ConsensusPolicy): void { + const problems: string[] = []; + if (!(policy.quorumRatio > 0 && policy.quorumRatio <= 1)) problems.push("quorumRatio must be in (0, 1]"); + if (!Number.isInteger(policy.minAgreeingSources) || policy.minAgreeingSources < 1) { + problems.push("minAgreeingSources must be an integer >= 1"); + } + for (const key of ["maxDeviationRatio", "absoluteDeviationFloor", "maxStalenessMs", "maxFutureSkewMs"] as const) { + if (!(Number.isFinite(policy[key]) && policy[key] >= 0)) problems.push(`${key} must be finite and >= 0`); + } + if (problems.length > 0) { + throw new RangeError(`Invalid consensus policy: ${problems.join("; ")}`); + } +} + +function isUsableWeight(w: unknown): w is number { + return typeof w === "number" && Number.isFinite(w) && w >= 0; +} + +/** + * Evaluate consensus over one round of observations. + * + * Never throws for bad observations — they are classified as invalid. It + * only throws for an invalid policy or a non-finite `now`. + * + * The result does not depend on the order of `observations`. + */ +export function evaluateConsensus( + observations: readonly SourceObservation[], + policy: ConsensusPolicy, + now: number +): ConsensusDecision { + validateConsensusPolicy(policy); + if (!Number.isFinite(now)) { + throw new RangeError(`evaluateConsensus() requires a finite 'now', got ${now}`); + } + + // Group by source id. A source id that appears more than once is treated + // as invalid in every copy, so a source cannot vote twice and the outcome + // does not depend on which copy came first. + const bySource = new Map(); + for (const obs of observations) { + const id = typeof obs?.sourceId === "string" ? obs.sourceId : String(obs?.sourceId); + const list = bySource.get(id); + if (list) list.push(obs); + else bySource.set(id, [obs]); + } + + const invalid: string[] = []; + const stale: string[] = []; + const fresh: SourceObservation[] = []; + let totalWeight = 0; + + for (const [id, copies] of bySource) { + // Every source that reported counts toward the quorum denominator, so a + // misbehaving source cannot shrink the bar by sending garbage. + const usableWeights = copies.map((c) => c.weight).filter(isUsableWeight); + if (usableWeights.length > 0) totalWeight += Math.max(...usableWeights); + + if (copies.length !== 1) { + invalid.push(id); + continue; + } + const obs = copies[0]; + const wellFormed = + typeof obs.sourceId === "string" && + obs.sourceId.length > 0 && + typeof obs.value === "number" && + Number.isFinite(obs.value) && + isUsableWeight(obs.weight) && + typeof obs.observedAt === "number" && + Number.isFinite(obs.observedAt) && + obs.observedAt <= now + policy.maxFutureSkewMs; + if (!wellFormed) { + invalid.push(id); + } else if (now - obs.observedAt > policy.maxStalenessMs) { + stale.push(id); + } else { + fresh.push(obs); + } + } + + const base = { + invalidSources: invalid.sort(), + staleSources: stale.sort(), + totalWeight, + evaluatedAt: now, + }; + + if (fresh.length === 0) { + return finalize(observations, policy, { + ...base, + status: "no_valid_sources", + value: null, + trustTier: TrustTier.Unverified, + center: null, + agreeingSources: [], + outlierSources: [], + agreeingWeight: 0, + sourceTimestamp: null, + }); + } + + const center = median(fresh.map((o) => o.value)); + const agreeing = fresh.filter((o) => !isAnomalous(o.value, center, policy)); + const outliers = fresh.filter((o) => isAnomalous(o.value, center, policy)); + const agreeingWeight = agreeing.reduce((sum, o) => sum + o.weight, 0); + + const common = { + ...base, + center, + agreeingSources: agreeing.map((o) => o.sourceId).sort(), + outlierSources: outliers.map((o) => o.sourceId).sort(), + agreeingWeight, + }; + + let status: ConsensusStatus = "agreed"; + if (agreeing.length < policy.minAgreeingSources) status = "insufficient_sources"; + else if (!meetsQuorum(agreeingWeight, totalWeight, policy.quorumRatio)) status = "no_quorum"; + + if (status !== "agreed") { + return finalize(observations, policy, { + ...common, + status, + value: null, + trustTier: TrustTier.Unverified, + sourceTimestamp: null, + }); + } + + let trustTier: TrustTier; + if (agreeing.length === 1) trustTier = TrustTier.SingleSource; + else if (agreeing.length >= 3 && outliers.length === 0 && stale.length === 0 && invalid.length === 0) { + trustTier = TrustTier.Unanimous; + } else trustTier = TrustTier.Corroborated; + + return finalize(observations, policy, { + ...common, + status, + value: median(agreeing.map((o) => o.value)), + trustTier, + sourceTimestamp: Math.min(...agreeing.map((o) => o.observedAt)), + }); +} + +function finalize( + observations: readonly SourceObservation[], + policy: ConsensusPolicy, + decision: Omit +): ConsensusDecision { + return { ...decision, decisionId: computeDecisionId(observations, policy, decision) }; +} + +/** + * Deterministic 64-bit decision id: a truncated SHA-256 over the canonical + * (sorted) inputs and the outcome. Re-running the same round reproduces the + * same id, which lets an auditor tie an on-chain value back to its inputs. + */ +export function computeDecisionId( + observations: readonly SourceObservation[], + policy: ConsensusPolicy, + decision: Pick +): string { + // String() keeps NaN / Infinity / -0 distinguishable, unlike JSON numbers. + const rows = observations + .map((o) => [String(o?.sourceId), String(o?.value), String(o?.weight), String(o?.observedAt)].join("|")) + .sort(); + const canonical = JSON.stringify({ + v: 1, + rows, + policy: [ + policy.quorumRatio, + policy.minAgreeingSources, + policy.maxDeviationRatio, + policy.absoluteDeviationFloor, + policy.maxStalenessMs, + policy.maxFutureSkewMs, + ].map(String), + status: decision.status, + value: String(decision.value), + at: String(decision.evaluatedAt), + }); + return createHash("sha256").update(canonical).digest("hex").slice(0, 16); +} diff --git a/src/oracle/oracle-encoding.spec.ts b/src/oracle/oracle-encoding.spec.ts new file mode 100644 index 0000000..e467141 --- /dev/null +++ b/src/oracle/oracle-encoding.spec.ts @@ -0,0 +1,165 @@ +import * as fs from "fs"; +import * as path from "path"; +import fc from "fast-check"; +import { scValToNative } from "@stellar/stellar-sdk"; +import { DEFAULT_CONSENSUS_POLICY, evaluateConsensus } from "./consensus/consensus"; +import { + PROVENANCE_VERSION, + ProvenanceEncodingError, + ReadingProvenance, + TrustTier, + buildUpdateReadingArgs, + coverageTypeToU32, + encodeReadingForChain, + packProvenance, + provenanceFromDecision, + scaleOracleValue, + unpackProvenance, +} from "./oracle-encoding"; +import { OracleReading } from "./oracle-reading"; + +const FIXTURES = path.join(__dirname, "../stellar/xdr-fixtures"); + +function readFixture(name: string): string { + const raw = fs.readFileSync(path.join(FIXTURES, name), "utf8"); + return raw + .split("\n") + .filter((l) => l && !l.startsWith("//")) + .join("") + .trim(); +} + +const SAMPLE: ReadingProvenance = { + trustTier: TrustTier.Corroborated, + agreeingSources: 3, + decisionId: "0123456789abcdef", + sourceTimestamp: 1_700_000_000, +}; + +describe("value scaling", () => { + it("scales each coverage type deterministically", () => { + expect(scaleOracleValue("StablecoinDepeg", 0.9998)).toBe(9_998_000n); + expect(scaleOracleValue("MarketCrash", -30.12)).toBe(-3012n); + expect(scaleOracleValue("LiquidationShield", 0.42)).toBe(4200n); + expect(scaleOracleValue("SmartContractRisk", -50)).toBe(-5000n); + expect(scaleOracleValue("FlightDelay", 247.4)).toBe(247n); + }); + + it("maps coverage type names to u32 discriminants", () => { + expect(coverageTypeToU32("FlightDelay")).toBe(4); + expect(() => coverageTypeToU32("Nope")).toThrow(); + }); +}); + +describe("provenance packing", () => { + it("matches the documented bit layout", () => { + // version=1 | tier=2 | count=0x03 | reserved=0x00 | ts=0x006553f100 | id + expect(packProvenance(SAMPLE)).toBe(0x1203_00_006553f100_0123456789abcdefn); + expect(PROVENANCE_VERSION).toBe(1); + }); + + it("round-trips every valid provenance", () => { + const tierAndCount = fc.oneof( + fc.constant({ trustTier: TrustTier.SingleSource, agreeingSources: 1 }), + fc.integer({ min: 2, max: 255 }).map((n) => ({ trustTier: TrustTier.Corroborated, agreeingSources: n })), + fc.integer({ min: 3, max: 255 }).map((n) => ({ trustTier: TrustTier.Unanimous, agreeingSources: n })) + ); + fc.assert( + fc.property( + tierAndCount, + fc.bigInt({ min: 0n, max: (1n << 64n) - 1n }), + fc.integer({ min: 0, max: 2 ** 40 - 1 }), + (tc, id, ts) => { + const p: ReadingProvenance = { ...tc, decisionId: id.toString(16).padStart(16, "0"), sourceTimestamp: ts }; + const packed = packProvenance(p); + expect(packed).toBeGreaterThanOrEqual(0n); + expect(packed).toBeLessThan(1n << 128n); + expect(unpackProvenance(packed)).toEqual(p); + } + ) + ); + }); + + it.each<[string, Partial]>([ + ["unverified tier", { trustTier: TrustTier.Unverified }], + ["unknown tier", { trustTier: 9 as TrustTier }], + ["single source with two sources", { trustTier: TrustTier.SingleSource, agreeingSources: 2 }], + ["corroborated with one source", { trustTier: TrustTier.Corroborated, agreeingSources: 1 }], + ["unanimous with two sources", { trustTier: TrustTier.Unanimous, agreeingSources: 2 }], + ["source count over 255", { agreeingSources: 256 }], + ["fractional source count", { agreeingSources: 2.5 }], + ["short decision id", { decisionId: "abc" }], + ["upper-case decision id", { decisionId: "0123456789ABCDEF" }], + ["negative timestamp", { sourceTimestamp: -1 }], + ["timestamp in milliseconds", { sourceTimestamp: 1_700_000_000_000 * 1000 }], + ])("rejects %s", (_label, override) => { + expect(() => packProvenance({ ...SAMPLE, ...override })).toThrow(ProvenanceEncodingError); + }); + + it("rejects unknown versions, set reserved bits and out-of-range input when unpacking", () => { + const good = packProvenance(SAMPLE); + expect(() => unpackProvenance(0n)).toThrow(/version 0/); + expect(() => unpackProvenance(good | (1n << 104n))).toThrow(/Reserved/); + expect(() => unpackProvenance((good & ~(0xfn << 124n)) | (2n << 124n))).toThrow(/version 2/); + expect(() => unpackProvenance(-1n)).toThrow(ProvenanceEncodingError); + expect(() => unpackProvenance(1n << 128n)).toThrow(ProvenanceEncodingError); + }); +}); + +describe("provenanceFromDecision", () => { + const NOW = 1_700_000_000_999; + + it("carries tier, agreeing count, decision id and floored timestamp", () => { + const decision = evaluateConsensus( + [ + { sourceId: "a", value: 1, weight: 1, observedAt: NOW - 1_500 }, + { sourceId: "b", value: 1, weight: 1, observedAt: NOW }, + { sourceId: "evil", value: 9, weight: 1, observedAt: NOW }, + ], + { ...DEFAULT_CONSENSUS_POLICY, quorumRatio: 0.5 }, + NOW + ); + const p = provenanceFromDecision(decision); + expect(p).toEqual({ + trustTier: TrustTier.Corroborated, + agreeingSources: 2, + decisionId: decision.decisionId, + sourceTimestamp: 1_699_999_999, + }); + expect(unpackProvenance(packProvenance(p))).toEqual(p); + }); + + it("refuses decisions that did not reach consensus", () => { + const decision = evaluateConsensus([], DEFAULT_CONSENSUS_POLICY, NOW); + expect(() => provenanceFromDecision(decision)).toThrow(/no_valid_sources/); + }); +}); + +describe("on-chain encoding", () => { + it("builds update_reading(u32, i128, u128) arguments", () => { + const encoded = encodeReadingForChain("StablecoinDepeg", 0.9998, SAMPLE); + const args = buildUpdateReadingArgs(encoded); + + expect(args.map((a) => a.switch().name)).toEqual(["scvU32", "scvI128", "scvU128"]); + expect(args.map((a) => scValToNative(a))).toEqual([0, 9_998_000n, packProvenance(SAMPLE)]); + expect(unpackProvenance(scValToNative(args[2]) as bigint)).toEqual(SAMPLE); + }); + + it("matches the golden XDR fixture for the provenance argument", () => { + const [, , provenance] = buildUpdateReadingArgs(encodeReadingForChain("StablecoinDepeg", 0.9998, SAMPLE)); + expect(provenance.toXDR("base64")).toBe(readFixture("oracle-update-reading-provenance.xdr.b64")); + }); + + it("keeps readings with provenance JSON-serialisable", () => { + const reading: OracleReading = { + coverageType: "StablecoinDepeg", + type: "oracle_update", + value: 0.9998, + threshold: 0.95, + severity: "low", + message: "ok", + provenance: SAMPLE, + }; + expect(JSON.parse(JSON.stringify(reading))).toEqual(reading); + }); +}); diff --git a/src/oracle/oracle-encoding.ts b/src/oracle/oracle-encoding.ts index f80f678..2e4a4ed 100644 --- a/src/oracle/oracle-encoding.ts +++ b/src/oracle/oracle-encoding.ts @@ -1,5 +1,9 @@ import { createHash } from "node:crypto"; +import { nativeToScVal, xdr } from "@stellar/stellar-sdk"; import type { CoverageType } from "../common/coverage-catalog"; +import { ConsensusDecision, TrustTier } from "./consensus/consensus"; + +export { TrustTier } from "./consensus/consensus"; /** Whether a source reads real market data or returns simulated values. */ export type OracleSourceKind = "live" | "mocked"; @@ -202,6 +206,182 @@ export function coverageTypeToU32(coverageType: string): number { return id; } +// --------------------------------------------------------------------------- +// Reading provenance +// --------------------------------------------------------------------------- + +/** + * Where a published value came from, carried on-chain next to the value so + * consumers and auditors can tell a corroborated reading from a + * single-source one without leaving the ledger. + */ +export interface ReadingProvenance { + /** Consensus trust tier. Unverified readings are never encoded. */ + trustTier: TrustTier; + /** How many independent sources agreed (0–255). */ + agreeingSources: number; + /** + * Consensus decision id (u64) as 16 lowercase hex chars; reproducible from + * the round's inputs. A string, not a bigint, so readings stay JSON-safe. + */ + decisionId: string; + /** Earliest agreeing source observation time, Unix seconds (40-bit). */ + sourceTimestamp: number; +} + +/** Current provenance layout version. Bump when the layout changes. */ +export const PROVENANCE_VERSION = 1; + +const TIMESTAMP_MAX = 2 ** 40 - 1; // ~year 36812 +const U128_MAX = (1n << 128n) - 1n; + +/** + * Packed provenance layout (u128, most significant bits first): + * + * | bits | width | field | + * |----------|-------|----------------------------------------| + * | 127..124 | 4 | layout version (currently 1) | + * | 123..120 | 4 | trust tier (TrustTier) | + * | 119..112 | 8 | agreeing source count | + * | 111..104 | 8 | reserved, must be 0 | + * | 103..64 | 40 | source timestamp, Unix seconds | + * | 63..0 | 64 | consensus decision id | + * + * Version 0 is never valid, so a zero u128 means "no provenance". + */ +const SHIFT = { + version: 124n, + trustTier: 120n, + agreeingSources: 112n, + reserved: 104n, + sourceTimestamp: 64n, +} as const; + +export class ProvenanceEncodingError extends Error { + constructor(message: string) { + super(message); + this.name = "ProvenanceEncodingError"; + } +} + +/** Rejects provenance that is out of range or internally inconsistent. */ +export function assertValidProvenance(p: ReadingProvenance): void { + if (typeof p.trustTier !== "number" || TrustTier[p.trustTier] === undefined) { + throw new ProvenanceEncodingError(`Unknown trust tier: ${String(p.trustTier)}`); + } + if (p.trustTier === TrustTier.Unverified) { + throw new ProvenanceEncodingError("Unverified readings must not be published on-chain"); + } + if (!Number.isInteger(p.agreeingSources) || p.agreeingSources < 0 || p.agreeingSources > 255) { + throw new ProvenanceEncodingError(`agreeingSources must be an integer in 0..255, got ${p.agreeingSources}`); + } + const minSources: Record = { + [TrustTier.SingleSource]: 1, + [TrustTier.Corroborated]: 2, + [TrustTier.Unanimous]: 3, + }; + if (p.trustTier === TrustTier.SingleSource ? p.agreeingSources !== 1 : p.agreeingSources < minSources[p.trustTier]) { + throw new ProvenanceEncodingError( + `Trust tier ${TrustTier[p.trustTier]} is inconsistent with ${p.agreeingSources} agreeing source(s)` + ); + } + if (typeof p.decisionId !== "string" || !/^[0-9a-f]{16}$/.test(p.decisionId)) { + throw new ProvenanceEncodingError("decisionId must be 16 lowercase hex chars (u64)"); + } + if (!Number.isInteger(p.sourceTimestamp) || p.sourceTimestamp < 0 || p.sourceTimestamp > TIMESTAMP_MAX) { + throw new ProvenanceEncodingError(`sourceTimestamp must be Unix seconds in 0..2^40-1, got ${p.sourceTimestamp}`); + } +} + +/** Pack provenance into the u128 layout above. */ +export function packProvenance(p: ReadingProvenance): bigint { + assertValidProvenance(p); + return ( + (BigInt(PROVENANCE_VERSION) << SHIFT.version) | + (BigInt(p.trustTier) << SHIFT.trustTier) | + (BigInt(p.agreeingSources) << SHIFT.agreeingSources) | + (BigInt(p.sourceTimestamp) << SHIFT.sourceTimestamp) | + BigInt(`0x${p.decisionId}`) + ); +} + +/** Inverse of packProvenance. Rejects unknown versions and non-zero reserved bits. */ +export function unpackProvenance(packed: bigint): ReadingProvenance { + if (typeof packed !== "bigint" || packed < 0n || packed > U128_MAX) { + throw new ProvenanceEncodingError("Packed provenance must be a u128 bigint"); + } + const field = (shift: bigint, bits: bigint) => (packed >> shift) & ((1n << bits) - 1n); + const version = Number(field(SHIFT.version, 4n)); + if (version !== PROVENANCE_VERSION) { + throw new ProvenanceEncodingError(`Unsupported provenance version ${version}`); + } + if (field(SHIFT.reserved, 8n) !== 0n) { + throw new ProvenanceEncodingError("Reserved provenance bits are set"); + } + const provenance: ReadingProvenance = { + trustTier: Number(field(SHIFT.trustTier, 4n)) as TrustTier, + agreeingSources: Number(field(SHIFT.agreeingSources, 8n)), + sourceTimestamp: Number(field(SHIFT.sourceTimestamp, 40n)), + decisionId: field(0n, 64n).toString(16).padStart(16, "0"), + }; + assertValidProvenance(provenance); + return provenance; +} + +/** + * Derive on-chain provenance from a consensus decision. Throws for anything + * other than an agreed decision — those must never be published. + */ +export function provenanceFromDecision(decision: ConsensusDecision): ReadingProvenance { + if (decision.status !== "agreed" || decision.sourceTimestamp === null) { + throw new ProvenanceEncodingError(`Cannot publish a ${decision.status} consensus decision`); + } + const provenance: ReadingProvenance = { + trustTier: decision.trustTier, + agreeingSources: decision.agreeingSources.length, + decisionId: decision.decisionId, + // Floor so the on-chain timestamp never claims fresher data than we had. + sourceTimestamp: Math.floor(decision.sourceTimestamp / 1000), + }; + assertValidProvenance(provenance); + return provenance; +} + +/** Everything `update_reading` needs, already scaled and packed. */ +export interface EncodedOracleReading { + coverageType: number; + value: bigint; + provenance: bigint; +} + +/** Scale the value and pack provenance for one reading. */ +export function encodeReadingForChain( + coverageType: string, + value: number, + provenance: ReadingProvenance +): EncodedOracleReading { + return { + coverageType: coverageTypeToU32(coverageType), + value: scaleOracleValue(coverageType, value), + provenance: packProvenance(provenance), + }; +} + +/** + * ScVal arguments for + * `RefractOracle.update_reading(coverage_type: u32, value: i128, provenance: u128)`. + * + * The provenance argument requires the matching contract change; until that + * ships, publishers keep calling the two-argument form. + */ +export function buildUpdateReadingArgs(encoded: EncodedOracleReading): xdr.ScVal[] { + return [ + nativeToScVal(encoded.coverageType, { type: "u32" }), + nativeToScVal(encoded.value, { type: "i128" }), + nativeToScVal(encoded.provenance, { type: "u128" }), + ]; +} + /** * Canonical bytes every quorum submitter signs for one oracle decision. * Length-prefixed fields so they cannot bleed into each other: diff --git a/src/oracle/oracle-feed.module.ts b/src/oracle/oracle-feed.module.ts new file mode 100644 index 0000000..271136c --- /dev/null +++ b/src/oracle/oracle-feed.module.ts @@ -0,0 +1,14 @@ +import { Module } from "@nestjs/common"; +import { OracleFeedRelay } from "./oracle-feed.relay"; +import { OracleGateway } from "./oracle.gateway"; + +/** + * Read side of the oracle: the WebSocket gateway and the relay that feeds it + * from the ReadingBus. Imported by the API (AppModule) only — the worker + * has no HTTP server to attach a gateway to. + */ +@Module({ + providers: [OracleGateway, OracleFeedRelay], + exports: [OracleGateway, OracleFeedRelay], +}) +export class OracleFeedModule {} diff --git a/src/oracle/oracle-feed.relay.spec.ts b/src/oracle/oracle-feed.relay.spec.ts new file mode 100644 index 0000000..35866d0 --- /dev/null +++ b/src/oracle/oracle-feed.relay.spec.ts @@ -0,0 +1,62 @@ +import { OracleFeedRelay } from "./oracle-feed.relay"; +import { OracleGateway } from "./oracle.gateway"; +import { OracleReading } from "./oracle-reading"; +import { InMemoryReadingBus } from "./reading-bus"; + +function reading(overrides: Partial = {}): OracleReading { + return { + coverageType: "MarketCrash", + type: "oracle_update", + value: -12, + threshold: -20, + severity: "high", + message: "drawdown", + ...overrides, + }; +} + +describe("OracleFeedRelay", () => { + const originalRole = process.env.PROCESS_ROLE; + afterEach(() => { + if (originalRole === undefined) delete process.env.PROCESS_ROLE; + else process.env.PROCESS_ROLE = originalRole; + }); + + function setup() { + const bus = new InMemoryReadingBus(); + const gateway = { broadcastAlert: jest.fn() } as unknown as OracleGateway; + return { bus, gateway, relay: new OracleFeedRelay(bus, gateway) }; + } + + it("relays non-low readings published by the worker to WebSocket clients", async () => { + process.env.PROCESS_ROLE = "api"; + const { bus, gateway, relay } = setup(); + await relay.onModuleInit(); + + await bus.publish(reading()); + await bus.publish(reading({ coverageType: "FlightDelay", severity: "low" })); + + expect(gateway.broadcastAlert).toHaveBeenCalledTimes(1); + expect(gateway.broadcastAlert).toHaveBeenCalledWith(reading()); + expect((await relay.latest()).map((r) => r.coverageType)).toEqual(["MarketCrash", "FlightDelay"]); + }); + + it("stops relaying after shutdown", async () => { + process.env.PROCESS_ROLE = "api"; + const { bus, gateway, relay } = setup(); + await relay.onModuleInit(); + await relay.onModuleDestroy(); + + await bus.publish(reading()); + expect(gateway.broadcastAlert).not.toHaveBeenCalled(); + }); + + it("does not subscribe in a worker-only process", async () => { + process.env.PROCESS_ROLE = "worker"; + const { bus, gateway, relay } = setup(); + await relay.onModuleInit(); + + await bus.publish(reading()); + expect(gateway.broadcastAlert).not.toHaveBeenCalled(); + }); +}); diff --git a/src/oracle/oracle-feed.relay.ts b/src/oracle/oracle-feed.relay.ts new file mode 100644 index 0000000..b081eda --- /dev/null +++ b/src/oracle/oracle-feed.relay.ts @@ -0,0 +1,50 @@ +import { Injectable, Logger, OnModuleDestroy, OnModuleInit } from "@nestjs/common"; +import { OracleGateway } from "./oracle.gateway"; +import { OracleReading } from "./oracle-reading"; +import { ReadingBus } from "./reading-bus"; +import { servesApi } from "../runtime/process-role"; + +/** + * API-side half of the worker split: forwards readings the worker publishes + * on the ReadingBus to WebSocket clients, and answers "latest reading" + * queries from the shared store. It never polls upstream sources. + * + * Only non-"low" readings are pushed over the socket, matching the feed's + * behaviour before the split. + */ +@Injectable() +export class OracleFeedRelay implements OnModuleInit, OnModuleDestroy { + private readonly logger = new Logger(OracleFeedRelay.name); + private unsubscribe: (() => Promise) | null = null; + + constructor( + private readonly bus: ReadingBus, + private readonly gateway: OracleGateway + ) {} + + async onModuleInit(): Promise { + if (!servesApi()) { + return; + } + this.unsubscribe = await this.bus.subscribe((reading) => this.relay(reading)); + this.logger.log("Relaying oracle readings from the worker to WebSocket clients"); + } + + async onModuleDestroy(): Promise { + await this.unsubscribe?.(); + this.unsubscribe = null; + } + + /** Latest reading per coverage type, as last published by the worker. */ + latest(): Promise { + return this.bus.latest(); + } + + private relay(reading: OracleReading): void { + if (reading.severity === "low") { + return; + } + this.logger.warn(`Oracle alert: ${reading.coverageType} — ${reading.message}`); + this.gateway.broadcastAlert(reading); + } +} diff --git a/src/oracle/oracle-reading.ts b/src/oracle/oracle-reading.ts index b61e9ba..cec5cdc 100644 --- a/src/oracle/oracle-reading.ts +++ b/src/oracle/oracle-reading.ts @@ -1,3 +1,4 @@ +import type { ReadingProvenance } from "./oracle-encoding"; import { FreshnessRejectionReason } from "./freshnessPolicy"; export type Severity = "low" | "medium" | "high" | "triggered"; @@ -14,6 +15,12 @@ export interface OracleReading { * OraclePublisherService must never publish degraded readings on-chain. */ degraded?: boolean; + /** + * Consensus provenance (trust tier, agreeing source count, decision id, + * source timestamp). Present only for readings that reached consensus; + * published on-chain next to the value. Never set on degraded readings. + */ + provenance?: ReadingProvenance; /** * Upstream observation time (ISO-8601 with explicit offset, or a Date). * Consensus uses this; readings without it are rejected at ingestion. diff --git a/src/oracle/oracle.gateway.ts b/src/oracle/oracle.gateway.ts index 3c0622d..d3670ad 100644 --- a/src/oracle/oracle.gateway.ts +++ b/src/oracle/oracle.gateway.ts @@ -62,13 +62,15 @@ export class OracleGateway this.clients.set(client, state); this.startTimers(); - client.on('error', (err) => { - this.logger.warn(`Oracle WebSocket client error: ${err.message}`); - }); - client.on('pong', () => { - state.alive = true; - }); - client.on('message', (data) => this.onMessage(client, state, data.toString())); + if (typeof client.on === 'function') { + client.on('error', (err) => { + this.logger.warn(`Oracle WebSocket client error: ${err.message}`); + }); + client.on('pong', () => { + state.alive = true; + }); + client.on('message', (data) => this.onMessage(client, state, data.toString())); + } // Snapshot first, so clients need no racing REST bootstrap. this.safeSend(client, { @@ -79,6 +81,35 @@ export class OracleGateway .filter((e) => this.matches(state, e)) .map((e) => this.view(state, e)), }); + + this.safeSend(client, { type: 'connected', message: 'Refract oracle feed' }); + } + + /** + * Push a reading to every open client. In a split deployment the readings + * come from the worker via the ReadingBus (see OracleFeedRelay); this + * process never polls upstreams itself. + */ + broadcastAlert(alert: OracleReading): void { + if (!this.server?.clients) { + return; + } + const payload = JSON.stringify({ ...alert, timestamp: Date.now() }); + let sent = 0; + for (const client of this.server.clients) { + if (client.readyState !== WebSocket.OPEN) continue; + // send() can throw for a socket that is mid-close; one bad client must + // not stop the broadcast to everyone else. + try { + client.send(payload); + sent++; + } catch (err) { + this.logger.warn( + `Failed to send oracle alert to a client: ${err instanceof Error ? err.message : String(err)}`, + ); + } + } + this.logger.debug(`Broadcast ${alert.coverageType} alert to ${sent} client(s)`); } handleDisconnect(client: WebSocket): void { diff --git a/src/oracle/oracle.scheduler.ts b/src/oracle/oracle.scheduler.ts index e69de29..0259d8a 100644 --- a/src/oracle/oracle.scheduler.ts +++ b/src/oracle/oracle.scheduler.ts @@ -0,0 +1,69 @@ +import { Injectable, Logger, OnModuleInit, OnModuleDestroy, Optional } from '@nestjs/common'; +import { SchedulerRegistry } from '@nestjs/schedule'; +import { CronJob } from 'cron'; +import { OracleService } from './oracle.service'; +import { ProtocolParametersService } from '../parameters/protocol-parameters.service'; +import { ProtocolParameterKey } from '../parameters/protocol-parameter-key.enum'; +import { ReadingBus } from './reading-bus'; +import { OracleReading } from './oracle-reading'; +import { runsPipeline } from '../runtime/process-role'; +import { PipelineHeartbeat } from '../runtime/pipeline-heartbeat'; + +const ORACLE_REFRESH_JOB_NAME = 'oracle-refresh'; +export const ORACLE_PIPELINE_STAGE = 'oracle'; + +@Injectable() +export class OracleScheduler implements OnModuleInit, OnModuleDestroy { + private readonly logger = new Logger(OracleScheduler.name); + + constructor( + private readonly oracleService: OracleService, + private readonly schedulerRegistry: SchedulerRegistry, + private readonly protocolParameters: ProtocolParametersService, + private readonly readingBus: ReadingBus, + @Optional() private readonly heartbeat?: PipelineHeartbeat, + ) {} + + async onModuleInit(): Promise { + // Polling and consensus belong to the worker tier. The API only relays + // what the worker publishes (see OracleFeedRelay). + if (!runsPipeline()) { + this.logger.log('Oracle polling disabled in this process (PROCESS_ROLE=api)'); + return; + } + this.heartbeat?.expect(ORACLE_PIPELINE_STAGE); + + const intervalSeconds = await this.protocolParameters.getNumber( + ProtocolParameterKey.ORACLE_SCHEDULER_INTERVAL_SECONDS, + ); + + const job = new CronJob(`*/${intervalSeconds} * * * * *`, () => { + void this.refresh(); + }); + + this.schedulerRegistry.addCronJob(ORACLE_REFRESH_JOB_NAME, job); + job.start(); + + this.logger.log( + `Oracle refresh scheduler started with ${intervalSeconds}s interval`, + ); + } + + onModuleDestroy(): void { + if (this.schedulerRegistry.doesExist('cron', ORACLE_REFRESH_JOB_NAME)) { + this.schedulerRegistry.deleteCronJob(ORACLE_REFRESH_JOB_NAME); + } + } + + private async refresh(): Promise { + try { + const result: unknown = await this.oracleService.refreshPrices(); + const readings = Array.isArray(result) ? (result as OracleReading[]) : []; + // Hand readings to the API tier for the WebSocket feed and reads. + await this.readingBus.publishAll(readings); + this.heartbeat?.beat(ORACLE_PIPELINE_STAGE); + } catch (error) { + this.logger.error('Oracle refresh failed', error as Error); + } + } +} diff --git a/src/oracle/reading-bus.module.ts b/src/oracle/reading-bus.module.ts new file mode 100644 index 0000000..06a2550 --- /dev/null +++ b/src/oracle/reading-bus.module.ts @@ -0,0 +1,25 @@ +import { Global, Inject, Module, OnApplicationShutdown } from "@nestjs/common"; +import { ReadingBus, createReadingBus } from "./reading-bus"; +import { resolveProcessRole } from "../runtime/process-role"; + +/** + * Provides the ReadingBus to both tiers. Global so the oracle scheduler + * (worker) and the feed relay (API) can inject it without import churn. + */ +@Global() +@Module({ + providers: [ + { + provide: ReadingBus, + useFactory: () => createReadingBus(resolveProcessRole()), + }, + ], + exports: [ReadingBus], +}) +export class ReadingBusModule implements OnApplicationShutdown { + constructor(@Inject(ReadingBus) private readonly bus: ReadingBus) {} + + async onApplicationShutdown(): Promise { + await this.bus.close(); + } +} diff --git a/src/oracle/reading-bus.spec.ts b/src/oracle/reading-bus.spec.ts new file mode 100644 index 0000000..0194e31 --- /dev/null +++ b/src/oracle/reading-bus.spec.ts @@ -0,0 +1,177 @@ +import { OracleReading } from "./oracle-reading"; +import { + InMemoryReadingBus, + ORACLE_LATEST_KEY, + ORACLE_READINGS_CHANNEL, + RedisLike, + RedisReadingBus, + createReadingBus, + decodeReading, + encodeReading, +} from "./reading-bus"; +import { TrustTier } from "./consensus/consensus"; + +function reading(overrides: Partial = {}): OracleReading { + return { + coverageType: "StablecoinDepeg", + type: "oracle_update", + value: 0.97, + threshold: 0.95, + severity: "medium", + message: "USDC slipping", + ...overrides, + }; +} + +/** In-process stand-in for a Redis server shared by several clients. */ +class FakeRedisServer { + hashes = new Map>(); + subscribers = new Set(); + client(): FakeRedis { + return new FakeRedis(this); + } +} + +class FakeRedis implements RedisLike { + private listeners: Array<(channel: string, message: string) => void> = []; + channels = new Set(); + quitCalled = false; + constructor(private readonly server: FakeRedisServer) {} + + async hset(key: string, field: string, value: string) { + if (!this.server.hashes.has(key)) this.server.hashes.set(key, new Map()); + this.server.hashes.get(key)!.set(field, value); + return 1; + } + async hgetall(key: string) { + return Object.fromEntries(this.server.hashes.get(key) ?? []); + } + async publish(channel: string, message: string) { + for (const sub of this.server.subscribers) { + if (sub.channels.has(channel)) sub.listeners.forEach((l) => l(channel, message)); + } + return 1; + } + async subscribe(channel: string) { + this.channels.add(channel); + this.server.subscribers.add(this); + return 1; + } + async unsubscribe(channel: string) { + this.channels.delete(channel); + return 1; + } + on(_event: "message", listener: (channel: string, message: string) => void) { + this.listeners.push(listener); + return this; + } + off(_event: "message", listener: (channel: string, message: string) => void) { + this.listeners = this.listeners.filter((l) => l !== listener); + return this; + } + async quit() { + this.quitCalled = true; + return "OK"; + } +} + +describe("reading wire format", () => { + it("round-trips readings, including provenance", () => { + const r = reading({ + provenance: { trustTier: TrustTier.Corroborated, agreeingSources: 2, decisionId: "00000000000000ff", sourceTimestamp: 1 }, + }); + expect(decodeReading(encodeReading(r))).toEqual(r); + }); + + it.each([ + ["not JSON", "{"], + ["wrong version", JSON.stringify({ v: 2, reading: reading() })], + ["missing reading", JSON.stringify({ v: 1 })], + ["non-finite value", JSON.stringify({ v: 1, reading: { ...reading(), value: null } })], + ["unknown severity", JSON.stringify({ v: 1, reading: { ...reading(), severity: "panic" } })], + ])("rejects %s", (_label, raw) => { + expect(decodeReading(raw)).toBeNull(); + }); +}); + +describe("InMemoryReadingBus", () => { + it("delivers to subscribers and keeps the latest reading per coverage type", async () => { + const bus = new InMemoryReadingBus(); + const seen: OracleReading[] = []; + const unsubscribe = await bus.subscribe((r) => seen.push(r)); + + await bus.publishAll([reading({ value: 0.98 }), reading({ value: 0.97 }), reading({ coverageType: "FlightDelay", value: 200 })]); + await unsubscribe(); + await bus.publish(reading({ value: 0.5 })); + + expect(seen.map((r) => r.value)).toEqual([0.98, 0.97, 200]); + expect((await bus.latest()).map((r) => [r.coverageType, r.value])).toEqual([ + ["StablecoinDepeg", 0.5], + ["FlightDelay", 200], + ]); + }); + + it("refuses to publish malformed readings", async () => { + await expect(new InMemoryReadingBus().publish(reading({ value: NaN }))).rejects.toThrow(/malformed/); + }); +}); + +describe("RedisReadingBus", () => { + it("carries readings from a worker bus to an API bus through Redis", async () => { + const server = new FakeRedisServer(); + const worker = new RedisReadingBus(server.client(), server.client()); + const api = new RedisReadingBus(server.client(), server.client()); + + const seen: OracleReading[] = []; + await api.subscribe((r) => seen.push(r)); + await worker.publish(reading()); + + expect(seen).toEqual([reading()]); + expect(await api.latest()).toEqual([reading()]); + expect(server.hashes.get(ORACLE_LATEST_KEY)?.has("StablecoinDepeg")).toBe(true); + }); + + it("drops malformed messages instead of passing them to handlers", async () => { + const server = new FakeRedisServer(); + const api = new RedisReadingBus(server.client(), server.client()); + const handler = jest.fn(); + await api.subscribe(handler); + + await server.client().publish(ORACLE_READINGS_CHANNEL, "garbage"); + await server.client().hset(ORACLE_LATEST_KEY, "X", "garbage"); + + expect(handler).not.toHaveBeenCalled(); + expect(await api.latest()).toEqual([]); + }); + + it("closes both connections", async () => { + const server = new FakeRedisServer(); + const commands = server.client(); + const subscriber = server.client(); + const bus = new RedisReadingBus(commands, subscriber); + await bus.subscribe(() => undefined); + await bus.close(); + expect(commands.quitCalled && subscriber.quitCalled).toBe(true); + expect(subscriber.channels.size).toBe(0); + }); +}); + +describe("createReadingBus", () => { + const connect = () => new FakeRedisServer().client(); + + it("uses Redis whenever it is configured", () => { + expect(createReadingBus("api", { REDIS_URL: "redis://x" }, connect)).toBeInstanceOf(RedisReadingBus); + expect(createReadingBus("all", { REDIS_HOST: "x" }, connect)).toBeInstanceOf(RedisReadingBus); + }); + + it("falls back to in-memory for single-process and sandbox runs", () => { + expect(createReadingBus("all", {}, connect)).toBeInstanceOf(InMemoryReadingBus); + expect(createReadingBus("all", { SANDBOX_MODE: "true", REDIS_URL: "redis://x" }, connect)).toBeInstanceOf( + InMemoryReadingBus + ); + }); + + it.each(["api", "worker"] as const)("fails fast when PROCESS_ROLE=%s has no Redis", (role) => { + expect(() => createReadingBus(role, {}, connect)).toThrow(/requires REDIS_URL/); + }); +}); diff --git a/src/oracle/reading-bus.ts b/src/oracle/reading-bus.ts new file mode 100644 index 0000000..84d06e7 --- /dev/null +++ b/src/oracle/reading-bus.ts @@ -0,0 +1,213 @@ +import { EventEmitter } from "events"; +import { Logger } from "@nestjs/common"; +import Redis, { RedisOptions } from "ioredis"; +import { OracleReading } from "./oracle-reading"; +import { ProcessRole } from "../runtime/process-role"; + +/** + * Carries oracle readings from the worker (which polls and evaluates + * consensus) to the API (which serves reads and the WebSocket feed). + * + * The API never polls upstreams itself: it relays what the worker publishes + * and answers "latest reading" queries from the shared store. + */ +export abstract class ReadingBus { + abstract publish(reading: OracleReading): Promise; + /** Subscribe to new readings. Resolves to an unsubscribe function. */ + abstract subscribe(handler: (reading: OracleReading) => void): Promise<() => Promise>; + /** Most recent reading per coverage type. */ + abstract latest(): Promise; + abstract close(): Promise; + + async publishAll(readings: readonly OracleReading[]): Promise { + for (const reading of readings) { + await this.publish(reading); + } + } +} + +export const ORACLE_READINGS_CHANNEL = "refract:oracle:readings"; +export const ORACLE_LATEST_KEY = "refract:oracle:latest"; + +interface Envelope { + v: 1; + publishedAt: number; + reading: OracleReading; +} + +const SEVERITIES = new Set(["low", "medium", "high", "triggered"]); + +/** Parse and validate a bus message. Returns null for anything malformed. */ +export function decodeReading(raw: string): OracleReading | null { + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + return null; + } + const env = parsed as Partial | null; + const r = env?.reading as Partial | undefined; + if ( + env?.v !== 1 || + !r || + typeof r.coverageType !== "string" || + r.type !== "oracle_update" || + typeof r.value !== "number" || + !Number.isFinite(r.value) || + typeof r.threshold !== "number" || + typeof r.message !== "string" || + !SEVERITIES.has(r.severity as string) + ) { + return null; + } + return r as OracleReading; +} + +export function encodeReading(reading: OracleReading, publishedAt = Date.now()): string { + const envelope: Envelope = { v: 1, publishedAt, reading }; + return JSON.stringify(envelope); +} + +/** Single-process bus for PROCESS_ROLE=all, sandbox mode and tests. */ +export class InMemoryReadingBus extends ReadingBus { + private readonly emitter = new EventEmitter(); + private readonly latestByType = new Map(); + + async publish(reading: OracleReading): Promise { + // Round-trip through the wire format so in-memory and Redis behave the same. + const decoded = decodeReading(encodeReading(reading)); + if (!decoded) throw new Error(`Refusing to publish malformed reading for ${reading.coverageType}`); + this.latestByType.set(decoded.coverageType, decoded); + this.emitter.emit("reading", decoded); + } + + async subscribe(handler: (reading: OracleReading) => void): Promise<() => Promise> { + this.emitter.on("reading", handler); + return async () => { + this.emitter.off("reading", handler); + }; + } + + async latest(): Promise { + return [...this.latestByType.values()]; + } + + async close(): Promise { + this.emitter.removeAllListeners(); + } +} + +/** Subset of ioredis used here, so tests can pass a fake. */ +export interface RedisLike { + hset(key: string, field: string, value: string): Promise; + hgetall(key: string): Promise>; + publish(channel: string, message: string): Promise; + subscribe(channel: string): Promise; + unsubscribe(channel: string): Promise; + on(event: "message", listener: (channel: string, message: string) => void): unknown; + off(event: "message", listener: (channel: string, message: string) => void): unknown; + quit(): Promise; +} + +/** + * Cross-process bus over Redis: latest readings in a hash, live updates via + * pub/sub. Uses a separate connection for subscribing, as Redis requires. + */ +export class RedisReadingBus extends ReadingBus { + private readonly logger = new Logger(RedisReadingBus.name); + private subscribed = false; + + constructor( + private readonly commands: RedisLike, + private readonly subscriber: RedisLike + ) { + super(); + } + + async publish(reading: OracleReading): Promise { + const payload = encodeReading(reading); + if (!decodeReading(payload)) { + throw new Error(`Refusing to publish malformed reading for ${reading.coverageType}`); + } + // Store first so a subscriber that reacts to the message can read it back. + await this.commands.hset(ORACLE_LATEST_KEY, reading.coverageType, payload); + await this.commands.publish(ORACLE_READINGS_CHANNEL, payload); + } + + async subscribe(handler: (reading: OracleReading) => void): Promise<() => Promise> { + const listener = (channel: string, message: string) => { + if (channel !== ORACLE_READINGS_CHANNEL) return; + const reading = decodeReading(message); + if (!reading) { + this.logger.warn("Dropping malformed oracle reading from Redis"); + return; + } + handler(reading); + }; + this.subscriber.on("message", listener); + if (!this.subscribed) { + await this.subscriber.subscribe(ORACLE_READINGS_CHANNEL); + this.subscribed = true; + } + return async () => { + this.subscriber.off("message", listener); + }; + } + + async latest(): Promise { + const all = await this.commands.hgetall(ORACLE_LATEST_KEY); + return Object.values(all ?? {}) + .map(decodeReading) + .filter((r): r is OracleReading => r !== null); + } + + async close(): Promise { + if (this.subscribed) { + await this.subscriber.unsubscribe(ORACLE_READINGS_CHANNEL).catch(() => undefined); + } + await Promise.allSettled([this.commands.quit(), this.subscriber.quit()]); + } +} + +export function redisOptionsFromEnv(env: NodeJS.ProcessEnv): { url?: string; options?: RedisOptions } | null { + if (env.REDIS_URL) return { url: env.REDIS_URL }; + if (env.REDIS_HOST) { + return { + options: { + host: env.REDIS_HOST, + port: Number(env.REDIS_PORT ?? 6379), + password: env.REDIS_PASSWORD || undefined, + }, + }; + } + return null; +} + +/** + * Pick the bus for this process. A split deployment (api / worker) needs + * Redis, otherwise the API would never see the worker's readings — fail at + * boot rather than serve a silently frozen feed. + */ +export function createReadingBus( + role: ProcessRole, + env: NodeJS.ProcessEnv = process.env, + connect: (cfg: { url?: string; options?: RedisOptions }) => RedisLike = defaultConnect +): ReadingBus { + const sandbox = (env.SANDBOX_MODE ?? "").toLowerCase() === "true"; + const redis = sandbox ? null : redisOptionsFromEnv(env); + if (!redis) { + if (role !== "all") { + throw new Error( + `PROCESS_ROLE=${role} requires REDIS_URL (or REDIS_HOST) so the API and worker can share oracle readings` + ); + } + return new InMemoryReadingBus(); + } + return new RedisReadingBus(connect(redis), connect(redis)); +} + +function defaultConnect(cfg: { url?: string; options?: RedisOptions }): RedisLike { + const client = cfg.url ? new Redis(cfg.url) : new Redis(cfg.options ?? {}); + client.on("error", (err: Error) => new Logger("ReadingBus").error(`Redis error: ${err.message}`)); + return client as unknown as RedisLike; +} diff --git a/src/runtime/graceful-shutdown.ts b/src/runtime/graceful-shutdown.ts new file mode 100644 index 0000000..29521f0 --- /dev/null +++ b/src/runtime/graceful-shutdown.ts @@ -0,0 +1,54 @@ +import { Logger } from "@nestjs/common"; + +/** + * Hard upper bound on how long we wait for in-flight work (HTTP requests, + * scheduler runs, connection-pool drains) to finish before forcing exit. + * Prevents a stuck settlement from hanging the process forever. + */ +export const SHUTDOWN_GRACE_PERIOD_MS = Number(process.env.SHUTDOWN_GRACE_PERIOD_MS ?? 30_000); + +/** + * Install SIGTERM/SIGINT handlers that run `close()` once, bounded by the + * grace period, then exit. Shared by the API (main.ts) and worker + * (worker.ts) entrypoints so both tiers drain the same way. + */ +export function installGracefulShutdown( + close: () => Promise, + logger: Logger, + graceMs: number = SHUTDOWN_GRACE_PERIOD_MS, + exit: (code: number) => void = (code) => process.exit(code) +): (signal: NodeJS.Signals) => Promise { + let shuttingDown = false; + + const shutdown = async (signal: NodeJS.Signals): Promise => { + if (shuttingDown) { + logger.warn(`Received ${signal} while already shutting down; ignoring`); + return; + } + shuttingDown = true; + logger.log(`Received ${signal}; starting graceful shutdown (grace period ${graceMs}ms)`); + + // If draining does not finish in time, log and exit so the orchestrator + // is not left waiting on a stuck in-flight settlement. + const forceExitTimer = setTimeout(() => { + logger.error(`Graceful shutdown exceeded ${graceMs}ms; forcing exit`); + exit(1); + }, graceMs); + forceExitTimer.unref(); + + try { + await close(); + clearTimeout(forceExitTimer); + logger.log("Graceful shutdown complete"); + exit(0); + } catch (err) { + clearTimeout(forceExitTimer); + logger.error(`Error during graceful shutdown: ${(err as Error).message}`, (err as Error).stack); + exit(1); + } + }; + + process.on("SIGTERM", () => void shutdown("SIGTERM")); + process.on("SIGINT", () => void shutdown("SIGINT")); + return shutdown; +} diff --git a/src/runtime/pipeline-heartbeat.ts b/src/runtime/pipeline-heartbeat.ts new file mode 100644 index 0000000..7a6c723 --- /dev/null +++ b/src/runtime/pipeline-heartbeat.ts @@ -0,0 +1,53 @@ +import { Injectable } from "@nestjs/common"; + +export interface HeartbeatStatus { + healthy: boolean; + startedAt: number; + /** Last successful run per pipeline stage (epoch ms). */ + lastBeats: Record; + /** Stages that have not succeeded within `staleAfterMs`. */ + staleStages: string[]; +} + +/** + * Records when each pipeline stage last completed, so the worker's health + * endpoint can report a stuck pipeline and the orchestrator can restart it. + * + * A stage counts as stale only after it has been registered with `expect()` + * and the grace period since startup has passed without a beat. + */ +@Injectable() +export class PipelineHeartbeat { + private readonly startedAt: number; + private readonly beats = new Map(); + private readonly expected = new Set(); + + constructor( + private readonly staleAfterMs: number = Number(process.env.WORKER_STALE_AFTER_MS ?? 10 * 60_000), + private readonly now: () => number = Date.now + ) { + this.startedAt = this.now(); + } + + /** Declare a stage that must keep beating for the worker to be healthy. */ + expect(stage: string): void { + this.expected.add(stage); + } + + beat(stage: string): void { + this.beats.set(stage, this.now()); + } + + status(): HeartbeatStatus { + const now = this.now(); + const staleStages = [...this.expected] + .filter((stage) => now - (this.beats.get(stage) ?? this.startedAt) > this.staleAfterMs) + .sort(); + return { + healthy: staleStages.length === 0, + startedAt: this.startedAt, + lastBeats: Object.fromEntries(this.beats), + staleStages, + }; + } +} diff --git a/src/runtime/process-role.ts b/src/runtime/process-role.ts new file mode 100644 index 0000000..6644f52 --- /dev/null +++ b/src/runtime/process-role.ts @@ -0,0 +1,34 @@ +/** + * Which tier this process runs. + * + * - `api` — HTTP + WebSocket only. Serves reads; never polls oracles, + * evaluates consensus, publishes on-chain or settles claims. + * - `worker` — the payout pipeline (oracle polling, consensus, on-chain + * publishing, claim settlement, relayer monitoring). No HTTP API. + * - `all` — both in one process. Kept as the default so existing + * single-process deployments keep paying out until they are + * split; use it for local development only. + * + * `src/main.ts` starts the API (role from PROCESS_ROLE, default `all`); + * `src/worker.ts` always runs as `worker`. + */ +export type ProcessRole = "api" | "worker" | "all"; + +export const PROCESS_ROLES: readonly ProcessRole[] = ["api", "worker", "all"]; + +export function resolveProcessRole(env: NodeJS.ProcessEnv = process.env): ProcessRole { + const raw = (env.PROCESS_ROLE ?? "").trim().toLowerCase(); + if (raw === "") return "all"; + if ((PROCESS_ROLES as readonly string[]).includes(raw)) return raw as ProcessRole; + throw new Error(`Invalid PROCESS_ROLE "${env.PROCESS_ROLE}". Expected one of: ${PROCESS_ROLES.join(", ")}`); +} + +/** True when this process should run the payout pipeline (polling, consensus, settlement). */ +export function runsPipeline(env: NodeJS.ProcessEnv = process.env): boolean { + return resolveProcessRole(env) !== "api"; +} + +/** True when this process serves the HTTP API / WebSocket feed. */ +export function servesApi(env: NodeJS.ProcessEnv = process.env): boolean { + return resolveProcessRole(env) !== "worker"; +} diff --git a/src/runtime/runtime.module.ts b/src/runtime/runtime.module.ts new file mode 100644 index 0000000..ac379eb --- /dev/null +++ b/src/runtime/runtime.module.ts @@ -0,0 +1,13 @@ +import { Global, Module } from "@nestjs/common"; +import { PipelineHeartbeat } from "./pipeline-heartbeat"; + +/** + * Worker-only runtime services. Global so schedulers inside feature modules + * (oracle, claim, relayer monitor) can inject PipelineHeartbeat. + */ +@Global() +@Module({ + providers: [{ provide: PipelineHeartbeat, useFactory: () => new PipelineHeartbeat() }], + exports: [PipelineHeartbeat], +}) +export class RuntimeModule {} diff --git a/src/runtime/runtime.spec.ts b/src/runtime/runtime.spec.ts new file mode 100644 index 0000000..390fa79 --- /dev/null +++ b/src/runtime/runtime.spec.ts @@ -0,0 +1,120 @@ +import { Injectable, Logger, Module, Optional } from "@nestjs/common"; +import { Test } from "@nestjs/testing"; +import { RuntimeModule } from "./runtime.module"; +import { resolveProcessRole, runsPipeline, servesApi } from "./process-role"; +import { PipelineHeartbeat } from "./pipeline-heartbeat"; +import { installGracefulShutdown } from "./graceful-shutdown"; + +describe("process role", () => { + it("defaults to 'all' so unsplit deployments keep paying out", () => { + expect(resolveProcessRole({})).toBe("all"); + expect(resolveProcessRole({ PROCESS_ROLE: " " })).toBe("all"); + }); + + it("accepts api / worker / all case-insensitively", () => { + expect(resolveProcessRole({ PROCESS_ROLE: "API" })).toBe("api"); + expect(resolveProcessRole({ PROCESS_ROLE: "worker" })).toBe("worker"); + expect(resolveProcessRole({ PROCESS_ROLE: " All " })).toBe("all"); + }); + + it("rejects unknown roles instead of guessing", () => { + expect(() => resolveProcessRole({ PROCESS_ROLE: "both" })).toThrow(/Invalid PROCESS_ROLE/); + }); + + it.each([ + ["api", false, true], + ["worker", true, false], + ["all", true, true], + ])("role %s: runsPipeline=%s servesApi=%s", (role, pipeline, api) => { + const env = { PROCESS_ROLE: role }; + expect(runsPipeline(env)).toBe(pipeline); + expect(servesApi(env)).toBe(api); + }); +}); + +describe("PipelineHeartbeat", () => { + it("is healthy until an expected stage misses its window", () => { + let now = 1_000; + const hb = new PipelineHeartbeat(100, () => now); + hb.expect("oracle"); + + now = 1_100; + expect(hb.status().healthy).toBe(true); + + now = 1_101; + expect(hb.status()).toMatchObject({ healthy: false, staleStages: ["oracle"] }); + + hb.beat("oracle"); + expect(hb.status()).toMatchObject({ healthy: true, lastBeats: { oracle: 1_101 } }); + }); + + it("ignores stages that were never expected", () => { + let now = 0; + const hb = new PipelineHeartbeat(10, () => now); + hb.beat("adhoc"); + now = 1_000; + expect(hb.status().healthy).toBe(true); + }); +}); + +describe("RuntimeModule", () => { + @Injectable() + class FeatureScheduler { + constructor(@Optional() readonly heartbeat?: PipelineHeartbeat) {} + } + @Module({ providers: [FeatureScheduler] }) + class FeatureModule {} + + it("makes one PipelineHeartbeat visible to schedulers in feature modules", async () => { + const moduleRef = await Test.createTestingModule({ imports: [RuntimeModule, FeatureModule] }).compile(); + const scheduler = moduleRef.get(FeatureScheduler); + expect(scheduler.heartbeat).toBeInstanceOf(PipelineHeartbeat); + expect(scheduler.heartbeat).toBe(moduleRef.get(PipelineHeartbeat)); + }); +}); + +describe("installGracefulShutdown", () => { + const logger = { log: jest.fn(), warn: jest.fn(), error: jest.fn() } as unknown as Logger; + let before: { term: Function[]; int: Function[] }; + + beforeEach(() => { + before = { term: process.listeners("SIGTERM"), int: process.listeners("SIGINT") }; + }); + + afterEach(() => { + for (const l of process.listeners("SIGTERM")) if (!before.term.includes(l)) process.off("SIGTERM", l as never); + for (const l of process.listeners("SIGINT")) if (!before.int.includes(l)) process.off("SIGINT", l as never); + }); + + it("closes once and exits 0", async () => { + const close = jest.fn().mockResolvedValue(undefined); + const exit = jest.fn(); + const shutdown = installGracefulShutdown(close, logger, 1_000, exit); + + await shutdown("SIGTERM"); + await shutdown("SIGINT"); + + expect(close).toHaveBeenCalledTimes(1); + expect(exit).toHaveBeenCalledWith(0); + }); + + it("exits 1 when close fails", async () => { + const exit = jest.fn(); + const shutdown = installGracefulShutdown(() => Promise.reject(new Error("boom")), logger, 1_000, exit); + await shutdown("SIGTERM"); + expect(exit).toHaveBeenCalledWith(1); + }); + + it("forces exit 1 when close exceeds the grace period", async () => { + jest.useFakeTimers(); + try { + const exit = jest.fn(); + const shutdown = installGracefulShutdown(() => new Promise(() => undefined), logger, 50, exit); + void shutdown("SIGTERM"); + jest.advanceTimersByTime(51); + expect(exit).toHaveBeenCalledWith(1); + } finally { + jest.useRealTimers(); + } + }); +}); diff --git a/src/stellar/relayer-monitor/fee-spend-tracker.ts b/src/stellar/relayer-monitor/fee-spend-tracker.ts new file mode 100644 index 0000000..e0d721a --- /dev/null +++ b/src/stellar/relayer-monitor/fee-spend-tracker.ts @@ -0,0 +1,65 @@ +import { FeeSpendSummary } from "./relayer-balance.policy"; + +export interface FeeSpendRecord { + txHash: string; + feeChargedStroops: bigint; + /** Ledger close time of the transaction (epoch ms). */ + at: number; + successful: boolean; +} + +/** + * Rolling window of fees the relayer actually paid, one record per + * submitted transaction (oracle publishes and claim settlements share the + * relayer key). Failed transactions are kept: they still cost a fee. + */ +export class FeeSpendTracker { + private readonly records = new Map(); + + /** + * @param windowMs how far back the summary looks + * @param minRateSpanMs lower bound on the span used for the hourly rate, + * so a burst right after startup is not extrapolated into a huge rate + */ + constructor( + private readonly windowMs: number, + private readonly minRateSpanMs = 15 * 60_000 + ) {} + + /** Adds a record; returns false if the transaction was already counted. */ + record(r: FeeSpendRecord): boolean { + if (this.records.has(r.txHash)) return false; + this.records.set(r.txHash, r); + return true; + } + + summary(now: number): FeeSpendSummary { + this.prune(now); + const inWindow = [...this.records.values()]; + if (inWindow.length === 0) { + return { count: 0, totalStroops: 0n, avgStroops: 0, maxStroops: 0n, ratePerHourStroops: 0 }; + } + let total = 0n; + let max = 0n; + let oldest = Infinity; + for (const r of inWindow) { + total += r.feeChargedStroops; + if (r.feeChargedStroops > max) max = r.feeChargedStroops; + if (r.at < oldest) oldest = r.at; + } + const spanMs = Math.min(Math.max(now - oldest, this.minRateSpanMs), this.windowMs); + return { + count: inWindow.length, + totalStroops: total, + avgStroops: Number(total) / inWindow.length, + maxStroops: max, + ratePerHourStroops: (Number(total) / spanMs) * 3_600_000, + }; + } + + private prune(now: number): void { + for (const [hash, r] of this.records) { + if (now - r.at > this.windowMs) this.records.delete(hash); + } + } +} diff --git a/src/stellar/relayer-monitor/horizon-relayer.client.ts b/src/stellar/relayer-monitor/horizon-relayer.client.ts new file mode 100644 index 0000000..9cf70b2 --- /dev/null +++ b/src/stellar/relayer-monitor/horizon-relayer.client.ts @@ -0,0 +1,112 @@ +import { FeeSpendRecord } from "./fee-spend-tracker"; +import { RelayerAccountSnapshot, parseXlmToStroops } from "./relayer-balance.policy"; + +export class RelayerAccountNotFoundError extends Error { + constructor(publicKey: string) { + super(`Relayer account ${publicKey} does not exist on the network (unfunded or wrong network)`); + this.name = "RelayerAccountNotFoundError"; + } +} + +export interface FeeSpendPage { + records: FeeSpendRecord[]; + /** Paging token to resume after; null when the page was empty. */ + cursor: string | null; +} + +type FetchFn = (url: string, init?: { signal?: AbortSignal }) => Promise<{ + ok: boolean; + status: number; + json(): Promise; +}>; + +interface HorizonBalance { + asset_type: string; + balance: string; + selling_liabilities?: string; +} + +interface HorizonAccount { + balances: HorizonBalance[]; + subentry_count: number; + num_sponsoring?: number; + num_sponsored?: number; +} + +interface HorizonTransaction { + hash: string; + paging_token: string; + fee_charged: string | number; + fee_account?: string; + source_account: string; + successful: boolean; + created_at: string; +} + +/** + * Reads the relayer's balance and fee history from Horizon. + * + * Soroban RPC's getAccount returns only the sequence number, so balances + * and per-transaction `fee_charged` come from Horizon. Only the public key + * is needed — the monitor never touches the relayer secret. + */ +export class HorizonRelayerClient { + constructor( + private readonly horizonUrl: string, + private readonly publicKey: string, + private readonly timeoutMs = 10_000, + private readonly fetchFn: FetchFn = (url, init) => fetch(url, init) + ) {} + + async getAccount(): Promise { + const res = await this.get(`/accounts/${this.publicKey}`); + if (res.status === 404) throw new RelayerAccountNotFoundError(this.publicKey); + if (!res.ok) throw new Error(`Horizon /accounts returned HTTP ${res.status}`); + const body = (await res.json()) as HorizonAccount; + const native = body.balances?.find((b) => b.asset_type === "native"); + if (!native) throw new Error("Horizon account has no native balance"); + return { + balanceStroops: parseXlmToStroops(native.balance), + sellingLiabilitiesStroops: parseXlmToStroops(native.selling_liabilities ?? "0"), + subentryCount: Number(body.subentry_count ?? 0), + numSponsoring: Number(body.num_sponsoring ?? 0), + numSponsored: Number(body.num_sponsored ?? 0), + }; + } + + /** + * Transactions whose fee the relayer paid, oldest first, starting after + * `cursor`. Without a cursor, returns the most recent page (to backfill + * the window on startup) and a cursor pointing at the newest one. + */ + async listFeeSpend(cursor: string | null, limit = 200): Promise { + const order = cursor ? "asc" : "desc"; + const params = new URLSearchParams({ order, limit: String(limit), include_failed: "true" }); + if (cursor) params.set("cursor", cursor); + const res = await this.get(`/accounts/${this.publicKey}/transactions?${params}`); + if (res.status === 404) throw new RelayerAccountNotFoundError(this.publicKey); + if (!res.ok) throw new Error(`Horizon /transactions returned HTTP ${res.status}`); + + const body = (await res.json()) as { _embedded?: { records?: HorizonTransaction[] } }; + const txs = [...(body._embedded?.records ?? [])]; + if (order === "desc") txs.reverse(); + + const records = txs + // Fee-bumped transactions are charged to the fee account; only count + // what the relayer itself paid. + .filter((tx) => (tx.fee_account ?? tx.source_account) === this.publicKey) + .map((tx) => ({ + txHash: tx.hash, + feeChargedStroops: BigInt(tx.fee_charged), + at: Date.parse(tx.created_at), + successful: Boolean(tx.successful), + })); + const newest = txs[txs.length - 1]; + return { records, cursor: newest ? newest.paging_token : cursor }; + } + + private get(path: string) { + const base = this.horizonUrl.replace(/\/+$/, ""); + return this.fetchFn(`${base}${path}`, { signal: AbortSignal.timeout(this.timeoutMs) }); + } +} diff --git a/src/stellar/relayer-monitor/relayer-balance-monitor.service.ts b/src/stellar/relayer-monitor/relayer-balance-monitor.service.ts new file mode 100644 index 0000000..b6a8cf8 --- /dev/null +++ b/src/stellar/relayer-monitor/relayer-balance-monitor.service.ts @@ -0,0 +1,211 @@ +import { Logger, OnModuleDestroy, OnModuleInit } from "@nestjs/common"; +import { AlertingService } from "../../alerting/alerting.service"; +import { PipelineHeartbeat } from "../../runtime/pipeline-heartbeat"; +import { runsPipeline } from "../../runtime/process-role"; +import { FeeSpendTracker } from "./fee-spend-tracker"; +import { HorizonRelayerClient, RelayerAccountNotFoundError } from "./horizon-relayer.client"; +import { + BalanceAlertTier, + BalanceAssessment, + FeeSpendSummary, + assessBalance, + decideAlert, + formatXlm, +} from "./relayer-balance.policy"; +import { RelayerMonitorConfig } from "./relayer-monitor.config"; + +export const RELAYER_MONITOR_STAGE = "relayer-monitor"; + +/** Where balance and fee metrics go. MetricsService implements this. */ +export interface RelayerMetricsSink { + recordRelayerBalance(assessment: BalanceAssessment, spend: FeeSpendSummary): void; + recordRelayerFee(feeStroops: bigint, successful: boolean): void; +} + +/** Cap on Horizon pages read per check, so a long outage cannot stall a check. */ +const MAX_PAGES_PER_CHECK = 10; +const PAGE_SIZE = 200; + +export interface RelayerMonitorStatus { + publicKey: string | null; + lastCheckedAt: number | null; + consecutiveFailures: number; + lastError: string | null; + assessment: BalanceAssessment | null; + feeSpend: FeeSpendSummary | null; +} + +/** + * Watches the relayer account that pays every oracle publish and claim + * settlement fee. Each check reads the balance and any new relayer-paid + * transactions from Horizon, works out the spendable balance and runway at + * the current fee spend, and raises tiered alerts (warning → critical → + * exhausted) before the relayer can no longer submit. + * + * Runs in the worker tier only. Built by RelayerMonitorModule's factory. + */ +export class RelayerBalanceMonitor implements OnModuleInit, OnModuleDestroy { + private readonly logger = new Logger(RelayerBalanceMonitor.name); + private readonly tracker: FeeSpendTracker; + private timer: NodeJS.Timeout | null = null; + private running: Promise | null = null; + + private cursor: string | null = null; + private tier: BalanceAlertTier = "ok"; + private lastAlertAt: number | null = null; + private unknownAlerted = false; + private status: RelayerMonitorStatus; + + constructor( + private readonly config: RelayerMonitorConfig, + private readonly client: HorizonRelayerClient | null, + private readonly alerting: AlertingService, + private readonly metrics?: RelayerMetricsSink, + private readonly heartbeat?: PipelineHeartbeat, + private readonly now: () => number = Date.now + ) { + this.tracker = new FeeSpendTracker(config.feeWindowMs); + this.status = { + publicKey: config.publicKey, + lastCheckedAt: null, + consecutiveFailures: 0, + lastError: null, + assessment: null, + feeSpend: null, + }; + } + + onModuleInit(): void { + if (!runsPipeline()) return; + if (!this.config.enabled || !this.client) { + this.logger.warn(`Relayer balance monitoring is off: ${this.config.disabledReason ?? "not configured"}`); + return; + } + this.heartbeat?.expect(RELAYER_MONITOR_STAGE); + this.logger.log( + `Monitoring relayer ${this.config.publicKey} every ${this.config.intervalMs}ms via ${this.config.horizonUrl}` + ); + void this.check(); + this.timer = setInterval(() => void this.check(), this.config.intervalMs); + this.timer.unref(); + } + + async onModuleDestroy(): Promise { + if (this.timer) clearInterval(this.timer); + this.timer = null; + await this.running; + } + + getStatus(): RelayerMonitorStatus { + return { ...this.status }; + } + + /** Run one check. Overlapping calls share the in-flight run. */ + check(): Promise { + if (!this.running) { + this.running = this.runCheck().finally(() => { + this.running = null; + }); + } + return this.running; + } + + private async runCheck(): Promise { + if (!this.client) return; + try { + await this.ingestFeeSpend(); + const snapshot = await this.client.getAccount(); + const now = this.now(); + const spend = this.tracker.summary(now); + const assessment = assessBalance(snapshot, spend, this.config.thresholds); + + this.metrics?.recordRelayerBalance(assessment, spend); + this.handleTier(assessment, now); + if (this.unknownAlerted) { + this.alerting.emit("info", "Relayer balance monitoring restored", "Horizon is reachable again", { + publicKey: this.config.publicKey, + }); + this.unknownAlerted = false; + } + this.status = { + ...this.status, + lastCheckedAt: now, + consecutiveFailures: 0, + lastError: null, + assessment, + feeSpend: spend, + }; + this.heartbeat?.beat(RELAYER_MONITOR_STAGE); + } catch (err) { + this.handleFailure(err); + } + } + + private async ingestFeeSpend(): Promise { + for (let page = 0; page < MAX_PAGES_PER_CHECK; page++) { + const result = await this.client!.listFeeSpend(this.cursor, PAGE_SIZE); + for (const r of result.records) { + if (this.tracker.record(r)) this.metrics?.recordRelayerFee(r.feeChargedStroops, r.successful); + } + const wasBackfill = this.cursor === null; + this.cursor = result.cursor; + // The first (backfill) call reads the newest page only; afterwards keep + // paging forward until a short page means we are caught up. + if (wasBackfill || result.records.length < PAGE_SIZE) return; + } + } + + private handleTier(a: BalanceAssessment, now: number): void { + const decision = decideAlert(this.tier, a.tier, this.lastAlertAt, now, this.config.renotifyMs); + const previous = this.tier; + this.tier = a.tier; + if (!decision.emit) return; + + const title = + decision.kind === "recovered" + ? `Relayer balance recovered to ${a.tier}` + : `Relayer balance ${a.tier}${decision.kind === "reminder" ? " (still)" : ""}`; + const message = + a.reasons.length > 0 + ? a.reasons.join("; ") + : `spendable ${formatXlm(a.spendableStroops)} is above all thresholds`; + this.alerting.emit(decision.severity, title, message, { + publicKey: this.config.publicKey, + previousTier: previous, + tier: a.tier, + balance: formatXlm(a.balanceStroops), + reserve: formatXlm(a.reserveStroops), + spendable: formatXlm(a.spendableStroops), + nextFee: formatXlm(a.nextFeeStroops), + runwayHours: a.runwayHours, + runwaySubmissions: a.runwaySubmissions, + }); + this.lastAlertAt = decision.kind === "recovered" && a.tier === "ok" ? null : now; + } + + private handleFailure(err: unknown): void { + const message = err instanceof Error ? err.message : String(err); + this.status = { ...this.status, consecutiveFailures: this.status.consecutiveFailures + 1, lastError: message }; + this.logger.warn(`Relayer balance check failed: ${message}`); + + // A missing account means the relayer cannot submit at all. + if (err instanceof RelayerAccountNotFoundError) { + if (this.tier !== "exhausted") { + this.alerting.emit("critical", "Relayer balance exhausted", message, { publicKey: this.config.publicKey }); + this.tier = "exhausted"; + this.lastAlertAt = this.now(); + } + return; + } + // Not knowing the balance is itself a risk once it persists. + if (!this.unknownAlerted && this.status.consecutiveFailures >= this.config.failuresBeforeAlert) { + this.alerting.emit( + "warning", + "Relayer balance unknown", + `${this.status.consecutiveFailures} consecutive balance checks failed: ${message}`, + { publicKey: this.config.publicKey, horizonUrl: this.config.horizonUrl } + ); + this.unknownAlerted = true; + } + } +} diff --git a/src/stellar/relayer-monitor/relayer-balance.policy.ts b/src/stellar/relayer-monitor/relayer-balance.policy.ts new file mode 100644 index 0000000..ed1da11 --- /dev/null +++ b/src/stellar/relayer-monitor/relayer-balance.policy.ts @@ -0,0 +1,178 @@ +/** + * Pure rules for relayer balance monitoring: how much XLM the relayer can + * actually spend, how long that lasts at the current fee spend, and which + * alert tier that puts us in. No I/O, so every rule is unit-testable. + */ + +/** Stellar base reserve: 0.5 XLM per entry. */ +export const BASE_RESERVE_STROOPS = 5_000_000n; +export const STROOPS_PER_XLM = 10_000_000n; + +/** Account fields from Horizon that decide how much is spendable. */ +export interface RelayerAccountSnapshot { + balanceStroops: bigint; + /** Native balance locked by open sell offers. */ + sellingLiabilitiesStroops: bigint; + subentryCount: number; + numSponsoring: number; + numSponsored: number; +} + +/** Rolling fee-spend statistics for transactions the relayer paid for. */ +export interface FeeSpendSummary { + count: number; + totalStroops: bigint; + avgStroops: number; + maxStroops: bigint; + /** Fee spend per hour over the observed window, in stroops. */ + ratePerHourStroops: number; +} + +export interface BalanceThresholds { + warningStroops: bigint; + criticalStroops: bigint; + warningRunwayHours: number; + criticalRunwayHours: number; + warningRunwaySubmissions: number; + criticalRunwaySubmissions: number; + /** + * Fee assumed for the next submission when no history exists yet. The + * relayer is `exhausted` once it cannot cover the larger of this and the + * biggest fee seen in the window. + */ + minSubmissionFeeStroops: bigint; +} + +export type BalanceAlertTier = "ok" | "warning" | "critical" | "exhausted"; + +/** Ordered from best to worst; index doubles as the metric value. */ +export const TIER_ORDER: readonly BalanceAlertTier[] = ["ok", "warning", "critical", "exhausted"]; + +export function tierRank(tier: BalanceAlertTier): number { + return TIER_ORDER.indexOf(tier); +} + +export interface BalanceAssessment { + tier: BalanceAlertTier; + balanceStroops: bigint; + reserveStroops: bigint; + spendableStroops: bigint; + /** Fee the next submission is expected to need. */ + nextFeeStroops: bigint; + /** Submissions left at the average fee; null without fee history. */ + runwaySubmissions: number | null; + /** Hours left at the current spend rate; null without fee history. */ + runwayHours: number | null; + /** Human-readable reasons for the tier, worst first. */ + reasons: string[]; +} + +/** Minimum balance Stellar requires the account to keep: (2 + entries) × base reserve. */ +export function minimumReserveStroops(s: RelayerAccountSnapshot): bigint { + const entries = 2 + s.subentryCount + s.numSponsoring - s.numSponsored; + return BigInt(Math.max(entries, 0)) * BASE_RESERVE_STROOPS; +} + +/** Balance available for fees after the reserve and open-offer liabilities. */ +export function spendableStroops(s: RelayerAccountSnapshot): bigint { + const spendable = s.balanceStroops - minimumReserveStroops(s) - s.sellingLiabilitiesStroops; + return spendable > 0n ? spendable : 0n; +} + +export function formatXlm(stroops: bigint): string { + const negative = stroops < 0n; + const abs = negative ? -stroops : stroops; + const whole = abs / STROOPS_PER_XLM; + const frac = (abs % STROOPS_PER_XLM).toString().padStart(7, "0").replace(/0+$/, ""); + return `${negative ? "-" : ""}${whole}${frac ? `.${frac}` : ""} XLM`; +} + +/** Parse a decimal XLM amount (e.g. Horizon's "123.4567890") into stroops without float rounding. */ +export function parseXlmToStroops(amount: string): bigint { + const match = /^(-)?(\d+)(?:\.(\d{1,7}))?$/.exec(amount.trim()); + if (!match) { + throw new Error(`Invalid XLM amount: "${amount}"`); + } + const [, sign, whole, frac = ""] = match; + const stroops = BigInt(whole) * STROOPS_PER_XLM + BigInt(frac.padEnd(7, "0")); + return sign ? -stroops : stroops; +} + +/** Decide the alert tier for the current balance and spend. */ +export function assessBalance( + snapshot: RelayerAccountSnapshot, + spend: FeeSpendSummary, + t: BalanceThresholds +): BalanceAssessment { + const reserve = minimumReserveStroops(snapshot); + const spendable = spendableStroops(snapshot); + const nextFee = spend.maxStroops > t.minSubmissionFeeStroops ? spend.maxStroops : t.minSubmissionFeeStroops; + + const runwaySubmissions = spend.count > 0 && spend.avgStroops > 0 ? Number(spendable) / spend.avgStroops : null; + const runwayHours = spend.ratePerHourStroops > 0 ? Number(spendable) / spend.ratePerHourStroops : null; + + const reasons: Array<{ tier: BalanceAlertTier; text: string }> = []; + const add = (tier: BalanceAlertTier, text: string) => reasons.push({ tier, text }); + + if (spendable < nextFee) { + add("exhausted", `spendable ${formatXlm(spendable)} cannot cover the next fee of ${formatXlm(nextFee)}`); + } + if (spendable < t.criticalStroops) { + add("critical", `spendable ${formatXlm(spendable)} is below ${formatXlm(t.criticalStroops)}`); + } else if (spendable < t.warningStroops) { + add("warning", `spendable ${formatXlm(spendable)} is below ${formatXlm(t.warningStroops)}`); + } + if (runwayHours !== null) { + if (runwayHours < t.criticalRunwayHours) add("critical", `about ${runwayHours.toFixed(1)}h of fees left at the current rate`); + else if (runwayHours < t.warningRunwayHours) add("warning", `about ${runwayHours.toFixed(1)}h of fees left at the current rate`); + } + if (runwaySubmissions !== null) { + if (runwaySubmissions < t.criticalRunwaySubmissions) { + add("critical", `about ${Math.floor(runwaySubmissions)} submissions left at the average fee`); + } else if (runwaySubmissions < t.warningRunwaySubmissions) { + add("warning", `about ${Math.floor(runwaySubmissions)} submissions left at the average fee`); + } + } + + reasons.sort((a, b) => tierRank(b.tier) - tierRank(a.tier)); + return { + tier: reasons[0]?.tier ?? "ok", + balanceStroops: snapshot.balanceStroops, + reserveStroops: reserve, + spendableStroops: spendable, + nextFeeStroops: nextFee, + runwaySubmissions, + runwayHours, + reasons: reasons.map((r) => r.text), + }; +} + +export type AlertDecision = + | { emit: false } + | { emit: true; kind: "escalated" | "recovered" | "reminder"; severity: "info" | "warning" | "critical" }; + +/** + * When to alert: on every escalation, once on recovery, and as a reminder + * every `renotifyMs` while the tier stays above ok. + */ +export function decideAlert( + previous: BalanceAlertTier, + next: BalanceAlertTier, + lastAlertAt: number | null, + now: number, + renotifyMs: number +): AlertDecision { + const severity = next === "warning" ? "warning" : "critical"; + if (tierRank(next) > tierRank(previous)) { + return { emit: true, kind: "escalated", severity }; + } + if (tierRank(next) < tierRank(previous)) { + return next === "ok" + ? { emit: true, kind: "recovered", severity: "info" } + : { emit: true, kind: "recovered", severity }; + } + if (next !== "ok" && (lastAlertAt === null || now - lastAlertAt >= renotifyMs)) { + return { emit: true, kind: "reminder", severity }; + } + return { emit: false }; +} diff --git a/src/stellar/relayer-monitor/relayer-monitor.config.ts b/src/stellar/relayer-monitor/relayer-monitor.config.ts new file mode 100644 index 0000000..b00d04b --- /dev/null +++ b/src/stellar/relayer-monitor/relayer-monitor.config.ts @@ -0,0 +1,100 @@ +import { Keypair, StrKey } from "@stellar/stellar-sdk"; +import { BalanceThresholds, parseXlmToStroops } from "./relayer-balance.policy"; + +export interface RelayerMonitorConfig { + enabled: boolean; + /** Why monitoring is off, when it is. */ + disabledReason?: string; + publicKey: string | null; + horizonUrl: string; + intervalMs: number; + feeWindowMs: number; + renotifyMs: number; + /** Consecutive failed checks before alerting that the balance is unknown. */ + failuresBeforeAlert: number; + thresholds: BalanceThresholds; +} + +const HORIZON_BY_NETWORK: Record = { + testnet: "https://horizon-testnet.stellar.org", + futurenet: "https://horizon-futurenet.stellar.org", + mainnet: "https://horizon.stellar.org", +}; + +function num(env: NodeJS.ProcessEnv, key: string, fallback: number, min = 0): number { + const raw = env[key]; + if (raw === undefined || raw.trim() === "") return fallback; + const value = Number(raw); + if (!Number.isFinite(value) || value < min) { + throw new Error(`${key} must be a number >= ${min} (got "${raw}")`); + } + return value; +} + +function xlm(env: NodeJS.ProcessEnv, key: string, fallback: string): bigint { + const raw = env[key]?.trim() || fallback; + const stroops = parseXlmToStroops(raw); + if (stroops < 0n) throw new Error(`${key} must not be negative`); + return stroops; +} + +/** + * The relayer's public key: RELAYER_PUBLIC_KEY if set (preferred — the + * monitor then never needs the secret), otherwise derived from the + * relayer secret the publisher already uses. + */ +function resolvePublicKey(env: NodeJS.ProcessEnv): string | null { + const explicit = env.RELAYER_PUBLIC_KEY?.trim(); + if (explicit) { + if (!StrKey.isValidEd25519PublicKey(explicit)) { + throw new Error("RELAYER_PUBLIC_KEY must be a valid Stellar public key (G...)"); + } + return explicit; + } + const secret = (env.ORACLE_RELAYER_SECRET ?? env.RELAYER_SECRET_KEY ?? "").trim(); + if (secret && StrKey.isValidEd25519SecretSeed(secret)) { + return Keypair.fromSecret(secret).publicKey(); + } + return null; +} + +export function loadRelayerMonitorConfig(env: NodeJS.ProcessEnv = process.env): RelayerMonitorConfig { + const network = (env.STELLAR_NETWORK ?? "testnet").trim().toLowerCase(); + const horizonUrl = env.STELLAR_HORIZON_URL?.trim() || HORIZON_BY_NETWORK[network] || HORIZON_BY_NETWORK.testnet; + + const thresholds: BalanceThresholds = { + warningStroops: xlm(env, "RELAYER_BALANCE_WARNING_XLM", "100"), + criticalStroops: xlm(env, "RELAYER_BALANCE_CRITICAL_XLM", "25"), + warningRunwayHours: num(env, "RELAYER_RUNWAY_WARNING_HOURS", 72), + criticalRunwayHours: num(env, "RELAYER_RUNWAY_CRITICAL_HOURS", 24), + warningRunwaySubmissions: num(env, "RELAYER_RUNWAY_WARNING_SUBMISSIONS", 1_000), + criticalRunwaySubmissions: num(env, "RELAYER_RUNWAY_CRITICAL_SUBMISSIONS", 200), + minSubmissionFeeStroops: xlm(env, "RELAYER_MIN_SUBMISSION_FEE_XLM", "0.01"), + }; + const pairs: Array<[string, number | bigint, number | bigint]> = [ + ["RELAYER_BALANCE", thresholds.criticalStroops, thresholds.warningStroops], + ["RELAYER_RUNWAY_*_HOURS", thresholds.criticalRunwayHours, thresholds.warningRunwayHours], + ["RELAYER_RUNWAY_*_SUBMISSIONS", thresholds.criticalRunwaySubmissions, thresholds.warningRunwaySubmissions], + ]; + for (const [name, critical, warning] of pairs) { + if (critical > warning) throw new Error(`${name}: the critical threshold must not exceed the warning threshold`); + } + + const publicKey = resolvePublicKey(env); + const optedOut = (env.RELAYER_MONITOR_ENABLED ?? "true").trim().toLowerCase() === "false"; + return { + enabled: !optedOut && publicKey !== null, + disabledReason: optedOut + ? "RELAYER_MONITOR_ENABLED=false" + : publicKey === null + ? "no relayer key configured (RELAYER_PUBLIC_KEY or ORACLE_RELAYER_SECRET)" + : undefined, + publicKey, + horizonUrl, + intervalMs: num(env, "RELAYER_MONITOR_INTERVAL_MS", 60_000, 1_000), + feeWindowMs: num(env, "RELAYER_FEE_WINDOW_HOURS", 24, 1) * 3_600_000, + renotifyMs: num(env, "RELAYER_ALERT_RENOTIFY_MINUTES", 60, 1) * 60_000, + failuresBeforeAlert: num(env, "RELAYER_MONITOR_FAILURES_BEFORE_ALERT", 3, 1), + thresholds, + }; +} diff --git a/src/stellar/relayer-monitor/relayer-monitor.module.ts b/src/stellar/relayer-monitor/relayer-monitor.module.ts new file mode 100644 index 0000000..67dbefd --- /dev/null +++ b/src/stellar/relayer-monitor/relayer-monitor.module.ts @@ -0,0 +1,26 @@ +import { Module } from "@nestjs/common"; +import { AlertingModule } from "../../alerting/alerting.module"; +import { AlertingService } from "../../alerting/alerting.service"; +import { MetricsService } from "../../metrics/metrics.service"; +import { PipelineHeartbeat } from "../../runtime/pipeline-heartbeat"; +import { HorizonRelayerClient } from "./horizon-relayer.client"; +import { RelayerBalanceMonitor } from "./relayer-balance-monitor.service"; +import { loadRelayerMonitorConfig } from "./relayer-monitor.config"; + +/** Relayer balance and fee-spend monitoring. Imported by the worker. */ +@Module({ + imports: [AlertingModule], + providers: [ + { + provide: RelayerBalanceMonitor, + inject: [AlertingService, { token: MetricsService, optional: true }, { token: PipelineHeartbeat, optional: true }], + useFactory: (alerting: AlertingService, metrics?: MetricsService, heartbeat?: PipelineHeartbeat) => { + const config = loadRelayerMonitorConfig(); + const client = config.publicKey ? new HorizonRelayerClient(config.horizonUrl, config.publicKey) : null; + return new RelayerBalanceMonitor(config, client, alerting, metrics, heartbeat); + }, + }, + ], + exports: [RelayerBalanceMonitor], +}) +export class RelayerMonitorModule {} diff --git a/src/stellar/relayer-monitor/relayer-monitor.spec.ts b/src/stellar/relayer-monitor/relayer-monitor.spec.ts new file mode 100644 index 0000000..be60403 --- /dev/null +++ b/src/stellar/relayer-monitor/relayer-monitor.spec.ts @@ -0,0 +1,439 @@ +import fc from "fast-check"; +import { Keypair } from "@stellar/stellar-sdk"; +import { AlertingService } from "../../alerting/alerting.service"; +import { PipelineHeartbeat } from "../../runtime/pipeline-heartbeat"; +import { FeeSpendTracker } from "./fee-spend-tracker"; +import { HorizonRelayerClient, RelayerAccountNotFoundError } from "./horizon-relayer.client"; +import { + BalanceThresholds, + FeeSpendSummary, + RelayerAccountSnapshot, + assessBalance, + decideAlert, + formatXlm, + minimumReserveStroops, + parseXlmToStroops, + spendableStroops, + tierRank, +} from "./relayer-balance.policy"; +import { RelayerBalanceMonitor, RelayerMetricsSink } from "./relayer-balance-monitor.service"; +import { RelayerMonitorConfig, loadRelayerMonitorConfig } from "./relayer-monitor.config"; + +const XLM = 10_000_000n; +const HOUR = 3_600_000; + +const thresholds: BalanceThresholds = { + warningStroops: 100n * XLM, + criticalStroops: 25n * XLM, + warningRunwayHours: 72, + criticalRunwayHours: 24, + warningRunwaySubmissions: 1_000, + criticalRunwaySubmissions: 200, + minSubmissionFeeStroops: XLM / 100n, +}; + +function account(balanceXlm: string, overrides: Partial = {}): RelayerAccountSnapshot { + return { + balanceStroops: parseXlmToStroops(balanceXlm), + sellingLiabilitiesStroops: 0n, + subentryCount: 0, + numSponsoring: 0, + numSponsored: 0, + ...overrides, + }; +} + +const NO_SPEND: FeeSpendSummary = { count: 0, totalStroops: 0n, avgStroops: 0, maxStroops: 0n, ratePerHourStroops: 0 }; + +describe("XLM amounts", () => { + it("parses Horizon balances exactly, without float rounding", () => { + expect(parseXlmToStroops("123.4567890")).toBe(1_234_567_890n); + expect(parseXlmToStroops("0.0000001")).toBe(1n); + expect(parseXlmToStroops("100")).toBe(1_000_000_000n); + expect(() => parseXlmToStroops("1.12345678")).toThrow(); + expect(() => parseXlmToStroops("1e3")).toThrow(); + }); + + it("round-trips through formatXlm", () => { + fc.assert( + fc.property(fc.bigInt({ min: 0n, max: 10n ** 17n }), (stroops) => { + return parseXlmToStroops(formatXlm(stroops).replace(" XLM", "")) === stroops; + }) + ); + }); +}); + +describe("reserve and spendable balance", () => { + it("applies the (2 + entries) × 0.5 XLM minimum reserve", () => { + expect(minimumReserveStroops(account("10"))).toBe(1n * XLM); + expect(minimumReserveStroops(account("10", { subentryCount: 3, numSponsoring: 1, numSponsored: 2 }))).toBe(2n * XLM); + }); + + it("subtracts reserve and selling liabilities and never goes negative", () => { + expect(spendableStroops(account("10", { sellingLiabilitiesStroops: 2n * XLM }))).toBe(7n * XLM); + expect(spendableStroops(account("0.5"))).toBe(0n); + }); +}); + +describe("assessBalance", () => { + it("is ok with a healthy balance and no history", () => { + const a = assessBalance(account("1000"), NO_SPEND, thresholds); + expect(a.tier).toBe("ok"); + expect(a.runwayHours).toBeNull(); + expect(a.reasons).toEqual([]); + }); + + it.each([ + ["warning", "90"], + ["critical", "20"], + ["exhausted", "1.005"], + ])("goes %s on absolute balance (balance %s XLM)", (tier, balance) => { + expect(assessBalance(account(balance), NO_SPEND, thresholds).tier).toBe(tier); + }); + + it("warns on time runway even when the balance looks large", () => { + // 1000 XLM spendable-ish, burning 20 XLM/h → ~50h left + const spend: FeeSpendSummary = { count: 10, totalStroops: 0n, avgStroops: 100, maxStroops: 100n, ratePerHourStroops: 20 * Number(XLM) }; + const a = assessBalance(account("1001"), spend, thresholds); + expect(a.tier).toBe("warning"); + expect(a.runwayHours).toBeCloseTo(50, 5); + expect(a.reasons[0]).toMatch(/50\.0h of fees left/); + }); + + it("goes critical on submission runway when per-publish fees are high", () => { + // 500 XLM spendable at 5 XLM per publish → 100 submissions left + const spend: FeeSpendSummary = { count: 3, totalStroops: 15n * XLM, avgStroops: 5 * Number(XLM), maxStroops: 5n * XLM, ratePerHourStroops: 1 }; + const a = assessBalance(account("501"), spend, thresholds); + expect(a.tier).toBe("critical"); + expect(a.runwaySubmissions).toBeCloseTo(100, 5); + }); + + it("is exhausted when the next fee, sized from the largest recent fee, cannot be covered", () => { + const spend: FeeSpendSummary = { count: 1, totalStroops: 3n * XLM, avgStroops: 3 * Number(XLM), maxStroops: 3n * XLM, ratePerHourStroops: 0 }; + const a = assessBalance(account("3.5"), spend, thresholds); + expect(a.nextFeeStroops).toBe(3n * XLM); + expect(a.tier).toBe("exhausted"); + }); + + it("never reports a better tier for a lower balance (property)", () => { + fc.assert( + fc.property( + fc.bigInt({ min: 0n, max: 10_000n * XLM }), + fc.bigInt({ min: 0n, max: 10_000n * XLM }), + fc.record({ + count: fc.integer({ min: 0, max: 1_000 }), + avg: fc.integer({ min: 0, max: 10_000_000 }), + max: fc.bigInt({ min: 0n, max: 100_000_000n }), + rate: fc.integer({ min: 0, max: 1_000_000_000 }), + }), + (b1, b2, s) => { + const spend: FeeSpendSummary = { + count: s.count, + totalStroops: 0n, + avgStroops: s.avg, + maxStroops: s.max, + ratePerHourStroops: s.rate, + }; + const [lo, hi] = b1 <= b2 ? [b1, b2] : [b2, b1]; + const tLo = assessBalance({ ...account("0"), balanceStroops: lo }, spend, thresholds).tier; + const tHi = assessBalance({ ...account("0"), balanceStroops: hi }, spend, thresholds).tier; + return tierRank(tLo) >= tierRank(tHi); + } + ) + ); + }); +}); + +describe("decideAlert", () => { + const RENOTIFY = HOUR; + + it("alerts on every escalation with matching severity", () => { + expect(decideAlert("ok", "warning", null, 0, RENOTIFY)).toEqual({ emit: true, kind: "escalated", severity: "warning" }); + expect(decideAlert("warning", "critical", 0, 1, RENOTIFY)).toEqual({ emit: true, kind: "escalated", severity: "critical" }); + expect(decideAlert("ok", "exhausted", null, 0, RENOTIFY)).toMatchObject({ severity: "critical" }); + }); + + it("announces recovery once", () => { + expect(decideAlert("critical", "ok", 0, 1, RENOTIFY)).toEqual({ emit: true, kind: "recovered", severity: "info" }); + expect(decideAlert("critical", "warning", 0, 1, RENOTIFY)).toEqual({ emit: true, kind: "recovered", severity: "warning" }); + expect(decideAlert("ok", "ok", null, 1, RENOTIFY)).toEqual({ emit: false }); + }); + + it("reminds while a problem persists, but not more often than the renotify interval", () => { + expect(decideAlert("critical", "critical", 0, RENOTIFY - 1, RENOTIFY)).toEqual({ emit: false }); + expect(decideAlert("critical", "critical", 0, RENOTIFY, RENOTIFY)).toEqual({ emit: true, kind: "reminder", severity: "critical" }); + }); +}); + +describe("FeeSpendTracker", () => { + it("dedupes by hash, counts failed transactions, and prunes outside the window", () => { + const t = new FeeSpendTracker(24 * HOUR, 0); + expect(t.record({ txHash: "a", feeChargedStroops: 100n, at: 0, successful: true })).toBe(true); + expect(t.record({ txHash: "a", feeChargedStroops: 100n, at: 0, successful: true })).toBe(false); + t.record({ txHash: "b", feeChargedStroops: 300n, at: HOUR, successful: false }); + + expect(t.summary(2 * HOUR)).toEqual({ count: 2, totalStroops: 400n, avgStroops: 200, maxStroops: 300n, ratePerHourStroops: 200 }); + expect(t.summary(24 * HOUR + 1).count).toBe(1); + }); + + it("does not extrapolate a startup burst into a huge hourly rate", () => { + const t = new FeeSpendTracker(24 * HOUR, 15 * 60_000); + t.record({ txHash: "a", feeChargedStroops: 1_000n, at: 0, successful: true }); + expect(t.summary(1_000).ratePerHourStroops).toBe(4_000); // 1000 per 15 min, not per second + }); +}); + +// ------------------------------------------------------------------ Horizon client + +const RELAYER = Keypair.random().publicKey(); +const BUMPER = Keypair.random().publicKey(); + +function horizonTx(n: number, overrides: Record = {}) { + return { + hash: `h${n}`, + paging_token: `pt${n}`, + fee_charged: "1000", + source_account: RELAYER, + fee_account: RELAYER, + successful: true, + created_at: new Date(1_800_000_000_000 + n * 60_000).toISOString(), + ...overrides, + }; +} + +function fakeHorizon(routes: Record) { + const calls: string[] = []; + const fetchFn = jest.fn(async (url: string) => { + calls.push(url); + const path = url.replace("https://horizon.test", ""); + const key = Object.keys(routes).find((k) => path.startsWith(k)); + const route = key ? routes[key] : { status: 404 }; + return { ok: (route.status ?? 200) < 400, status: route.status ?? 200, json: async () => route.body }; + }); + return { fetchFn, calls }; +} + +describe("HorizonRelayerClient", () => { + it("reads balance, liabilities and reserve inputs", async () => { + const { fetchFn } = fakeHorizon({ + [`/accounts/${RELAYER}`]: { + body: { + balances: [ + { asset_type: "credit_alphanum4", balance: "5.0000000" }, + { asset_type: "native", balance: "42.1234567", selling_liabilities: "1.0000000" }, + ], + subentry_count: 2, + num_sponsoring: 1, + }, + }, + }); + const client = new HorizonRelayerClient("https://horizon.test/", RELAYER, 1_000, fetchFn); + await expect(client.getAccount()).resolves.toEqual({ + balanceStroops: 421_234_567n, + sellingLiabilitiesStroops: 10_000_000n, + subentryCount: 2, + numSponsoring: 1, + numSponsored: 0, + }); + }); + + it("reports an unfunded account distinctly", async () => { + const { fetchFn } = fakeHorizon({}); + await expect(new HorizonRelayerClient("https://horizon.test", RELAYER, 1_000, fetchFn).getAccount()).rejects.toBeInstanceOf( + RelayerAccountNotFoundError + ); + }); + + it("backfills newest-first, returns oldest-first, and only counts fees the relayer paid", async () => { + const { fetchFn, calls } = fakeHorizon({ + [`/accounts/${RELAYER}/transactions`]: { + body: { _embedded: { records: [horizonTx(3), horizonTx(2, { fee_account: BUMPER }), horizonTx(1, { successful: false })] } }, + }, + }); + const page = await new HorizonRelayerClient("https://horizon.test", RELAYER, 1_000, fetchFn).listFeeSpend(null); + + expect(calls[0]).toContain("order=desc"); + expect(calls[0]).toContain("include_failed=true"); + expect(page.records.map((r) => [r.txHash, r.successful])).toEqual([ + ["h1", false], + ["h3", true], + ]); + expect(page.cursor).toBe("pt3"); + }); +}); + +// ------------------------------------------------------------------ config + +describe("loadRelayerMonitorConfig", () => { + it("prefers RELAYER_PUBLIC_KEY and picks Horizon from the network", () => { + const cfg = loadRelayerMonitorConfig({ RELAYER_PUBLIC_KEY: RELAYER, STELLAR_NETWORK: "mainnet" }); + expect(cfg).toMatchObject({ enabled: true, publicKey: RELAYER, horizonUrl: "https://horizon.stellar.org" }); + expect(cfg.thresholds.warningStroops).toBe(100n * XLM); + }); + + it("derives the public key from the relayer secret when needed", () => { + const kp = Keypair.random(); + expect(loadRelayerMonitorConfig({ ORACLE_RELAYER_SECRET: kp.secret() }).publicKey).toBe(kp.publicKey()); + }); + + it("is disabled, with a reason, when no key is configured or it is opted out", () => { + expect(loadRelayerMonitorConfig({})).toMatchObject({ enabled: false, disabledReason: expect.stringMatching(/no relayer key/) }); + expect(loadRelayerMonitorConfig({ RELAYER_PUBLIC_KEY: RELAYER, RELAYER_MONITOR_ENABLED: "false" }).enabled).toBe(false); + }); + + it.each([ + [{ RELAYER_BALANCE_WARNING_XLM: "10", RELAYER_BALANCE_CRITICAL_XLM: "20" }, /critical threshold/], + [{ RELAYER_MONITOR_INTERVAL_MS: "10" }, /RELAYER_MONITOR_INTERVAL_MS/], + [{ RELAYER_BALANCE_WARNING_XLM: "lots" }, /Invalid XLM amount/], + [{ RELAYER_PUBLIC_KEY: "GNOPE" }, /RELAYER_PUBLIC_KEY/], + ])("rejects bad config %#", (env, message) => { + expect(() => loadRelayerMonitorConfig({ RELAYER_PUBLIC_KEY: RELAYER, ...env })).toThrow(message); + }); +}); + +// ------------------------------------------------------------------ monitor + +describe("RelayerBalanceMonitor", () => { + const originalRole = process.env.PROCESS_ROLE; + afterEach(() => { + if (originalRole === undefined) delete process.env.PROCESS_ROLE; + else process.env.PROCESS_ROLE = originalRole; + }); + + const config: RelayerMonitorConfig = { + enabled: true, + publicKey: RELAYER, + horizonUrl: "https://horizon.test", + intervalMs: 60_000, + feeWindowMs: 24 * HOUR, + renotifyMs: HOUR, + failuresBeforeAlert: 2, + thresholds, + }; + + function setup() { + let now = 1_800_000_000_000; + let balance = "1000"; + let txs: Array> = []; + let failWith: Error | null = null; + const client = { + getAccount: jest.fn(async () => { + if (failWith) throw failWith; + return account(balance); + }), + listFeeSpend: jest.fn(async (cursor: string | null) => { + if (failWith) throw failWith; + const after = cursor ? txs.filter((t) => t.paging_token > cursor) : txs; + return { + records: after.map((t) => ({ txHash: t.hash, feeChargedStroops: BigInt(t.fee_charged), at: Date.parse(t.created_at), successful: t.successful })), + cursor: after.length ? after[after.length - 1].paging_token : cursor, + }; + }), + } as unknown as HorizonRelayerClient; + const alerting = new AlertingService(); + const metrics: RelayerMetricsSink = { recordRelayerBalance: jest.fn(), recordRelayerFee: jest.fn() }; + const heartbeat = new PipelineHeartbeat(HOUR, () => now); + const monitor = new RelayerBalanceMonitor(config, client, alerting, metrics, heartbeat, () => now); + return { + monitor, + alerting, + metrics, + heartbeat, + client, + setBalance: (b: string) => (balance = b), + addTx: (tx: ReturnType) => (txs = [...txs, tx]), + fail: (e: Error | null) => (failWith = e), + advance: (ms: number) => (now += ms), + titles: () => alerting.listRecent().map((a) => `${a.severity}:${a.title}`).reverse(), + }; + } + + it("raises tiered alerts as the balance drains, reminds, then reports recovery", async () => { + const t = setup(); + await t.monitor.check(); + expect(t.titles()).toEqual([]); + + t.setBalance("90"); + await t.monitor.check(); + t.setBalance("20"); + await t.monitor.check(); + t.advance(HOUR); + await t.monitor.check(); + t.setBalance("500"); + await t.monitor.check(); + + expect(t.titles()).toEqual([ + "warning:Relayer balance warning", + "critical:Relayer balance critical", + "critical:Relayer balance critical (still)", + "info:Relayer balance recovered to ok", + ]); + expect(t.monitor.getStatus().assessment?.tier).toBe("ok"); + }); + + it("tracks per-transaction fee spend, counting each transaction once", async () => { + const t = setup(); + t.addTx(horizonTx(1, { fee_charged: "50000" })); + await t.monitor.check(); + t.addTx(horizonTx(2, { fee_charged: "70000", successful: false })); + await t.monitor.check(); + await t.monitor.check(); + + expect(t.metrics.recordRelayerFee).toHaveBeenCalledTimes(2); + expect(t.metrics.recordRelayerFee).toHaveBeenLastCalledWith(70_000n, false); + expect(t.monitor.getStatus().feeSpend).toMatchObject({ count: 2, totalStroops: 120_000n, maxStroops: 70_000n }); + expect(t.metrics.recordRelayerBalance).toHaveBeenCalledTimes(3); + }); + + it("alerts when the balance has been unknown for several checks, and when it is back", async () => { + const t = setup(); + t.fail(new Error("Horizon down")); + await t.monitor.check(); + expect(t.titles()).toEqual([]); + await t.monitor.check(); + await t.monitor.check(); + t.fail(null); + await t.monitor.check(); + + expect(t.titles()).toEqual(["warning:Relayer balance unknown", "info:Relayer balance monitoring restored"]); + expect(t.monitor.getStatus().consecutiveFailures).toBe(0); + }); + + it("treats a missing account as exhausted immediately", async () => { + const t = setup(); + t.fail(new RelayerAccountNotFoundError(RELAYER)); + await t.monitor.check(); + await t.monitor.check(); + expect(t.titles()).toEqual(["critical:Relayer balance exhausted"]); + }); + + it("beats the pipeline heartbeat only on successful checks", async () => { + const t = setup(); + t.fail(new Error("down")); + await t.monitor.check(); + expect(t.heartbeat.status().lastBeats).toEqual({}); + t.fail(null); + await t.monitor.check(); + expect(Object.keys(t.heartbeat.status().lastBeats)).toEqual(["relayer-monitor"]); + }); + + it("does not start in an API-only process or without a key", async () => { + process.env.PROCESS_ROLE = "api"; + const t = setup(); + t.monitor.onModuleInit(); + expect(t.client.getAccount).not.toHaveBeenCalled(); + + process.env.PROCESS_ROLE = "worker"; + const off = new RelayerBalanceMonitor({ ...config, enabled: false, disabledReason: "test" }, null, new AlertingService()); + off.onModuleInit(); + await off.onModuleDestroy(); + }); + + it("starts polling in the worker and stops cleanly", async () => { + process.env.PROCESS_ROLE = "worker"; + const t = setup(); + t.monitor.onModuleInit(); + await t.monitor.onModuleDestroy(); + expect(t.client.getAccount).toHaveBeenCalledTimes(1); + }); +}); diff --git a/src/stellar/xdr-fixtures/oracle-update-reading-provenance.xdr.b64 b/src/stellar/xdr-fixtures/oracle-update-reading-provenance.xdr.b64 new file mode 100644 index 0000000..80bdd4d --- /dev/null +++ b/src/stellar/xdr-fixtures/oracle-update-reading-provenance.xdr.b64 @@ -0,0 +1,6 @@ +// Golden XDR fixtures for hand-built ScVal encodings. +// Regenerated by: npx ts-node scripts/regenerate-xdr-fixtures.ts +// A mismatch means either the contract ABI changed or the encoding regressed — +// check both before regenerating. +// u128 provenance: Corroborated, 3 sources, ts=1700000000 +AAAACRIDAABlU/EAASNFZ4mrze8= diff --git a/src/worker.ts b/src/worker.ts new file mode 100644 index 0000000..7db4fcd --- /dev/null +++ b/src/worker.ts @@ -0,0 +1,62 @@ +import { Logger } from "@nestjs/common"; +import { NestFactory } from "@nestjs/core"; +import { resolveProcessRole } from "./runtime/process-role"; +import { installGracefulShutdown } from "./runtime/graceful-shutdown"; +import { PipelineHeartbeat } from "./runtime/pipeline-heartbeat"; +import { startWorkerHealthServer } from "./worker/worker-health.server"; + +/** + * Worker entrypoint: runs the payout pipeline (oracle polling, consensus, + * on-chain publishing, claim settlement) with no HTTP API, so it can be + * deployed, scaled and restarted independently of the API tier. + * + * Run exactly one worker per relayer key — the relayer's sequence numbers + * are managed in-process (see RelayerAccountService). + */ +async function bootstrap(): Promise { + const logger = new Logger("WorkerBootstrap"); + + const configured = process.env.PROCESS_ROLE; + if (configured && resolveProcessRole() !== "worker") { + throw new Error(`worker.ts must run with PROCESS_ROLE=worker (got "${configured}")`); + } + process.env.PROCESS_ROLE = "worker"; + + // Imported after PROCESS_ROLE is fixed so module factories see the role. + const { WorkerModule } = await import("./worker/worker.module"); + const { MetricsService } = await import("./metrics/metrics.service"); + + const app = await NestFactory.createApplicationContext(WorkerModule); + app.enableShutdownHooks(); + + const heartbeat = app.get(PipelineHeartbeat); + let metrics: InstanceType | undefined; + try { + metrics = app.get(MetricsService, { strict: false }); + } catch { + metrics = undefined; + } + + const port = Number(process.env.WORKER_HEALTH_PORT ?? 4002); + const health = await startWorkerHealthServer( + { + heartbeat, + metrics: metrics ? async () => ({ body: await metrics!.metrics(), contentType: metrics!.contentType }) : undefined, + }, + port + ); + logger.log(`Worker started; health and metrics on port ${port}`); + + installGracefulShutdown(async () => { + await new Promise((resolve) => health.close(() => resolve())); + // Stops schedulers (ClaimScheduler waits for an in-flight settlement), + // closes the reading bus and connection pools. + await app.close(); + }, logger); +} + +bootstrap().catch((err) => { + // eslint-disable-next-line no-console + console.error("Fatal error during worker bootstrap", err); + process.exit(1); +}); diff --git a/src/worker/worker-health.server.spec.ts b/src/worker/worker-health.server.spec.ts new file mode 100644 index 0000000..f9ca0e3 --- /dev/null +++ b/src/worker/worker-health.server.spec.ts @@ -0,0 +1,58 @@ +import { Server } from "http"; +import { PipelineHeartbeat } from "../runtime/pipeline-heartbeat"; +import { boundPort, startWorkerHealthServer } from "./worker-health.server"; + +describe("worker health server", () => { + let server: Server | undefined; + let now = 0; + + afterEach(async () => { + await new Promise((resolve) => (server ? server.close(() => resolve()) : resolve())); + server = undefined; + }); + + async function start(withMetrics = true) { + now = 0; + const heartbeat = new PipelineHeartbeat(100, () => now); + heartbeat.expect("oracle"); + server = await startWorkerHealthServer( + { + heartbeat, + metrics: withMetrics ? async () => ({ body: "refract_up 1\n", contentType: "text/plain" }) : undefined, + }, + 0, + "127.0.0.1" + ); + return { heartbeat, base: `http://127.0.0.1:${boundPort(server)}` }; + } + + it("reports ok while the pipeline is beating and 503 once it goes stale", async () => { + const { heartbeat, base } = await start(); + + let res = await fetch(`${base}/health`); + expect(res.status).toBe(200); + expect(await res.json()).toMatchObject({ status: "ok", role: "worker" }); + + now = 500; + res = await fetch(`${base}/health`); + expect(res.status).toBe(503); + expect(await res.json()).toMatchObject({ status: "stale", staleStages: ["oracle"] }); + + heartbeat.beat("oracle"); + expect((await fetch(`${base}/health`)).status).toBe(200); + }); + + it("serves metrics when available and 404 otherwise", async () => { + const { base } = await start(); + const res = await fetch(`${base}/metrics`); + expect(res.status).toBe(200); + expect(await res.text()).toBe("refract_up 1\n"); + expect((await fetch(`${base}/nope`)).status).toBe(404); + }); + + it("accepts only GET", async () => { + const { base } = await start(false); + expect((await fetch(`${base}/health`, { method: "POST" })).status).toBe(405); + expect((await fetch(`${base}/metrics`)).status).toBe(404); + }); +}); diff --git a/src/worker/worker-health.server.ts b/src/worker/worker-health.server.ts new file mode 100644 index 0000000..7710666 --- /dev/null +++ b/src/worker/worker-health.server.ts @@ -0,0 +1,63 @@ +import * as http from "http"; +import { AddressInfo } from "net"; +import { PipelineHeartbeat } from "../runtime/pipeline-heartbeat"; + +export interface WorkerHealthSources { + heartbeat: PipelineHeartbeat; + /** Prometheus exposition, if metrics are available in this process. */ + metrics?: () => Promise<{ body: string; contentType: string }>; +} + +/** + * Minimal HTTP listener for the worker: `/health` for liveness/readiness + * probes and `/metrics` for Prometheus. The worker has no API, so this is + * deliberately not an Express/Nest app — nothing here accepts input. + * + * `/health` returns 503 when a pipeline stage has stopped succeeding, so + * the orchestrator restarts the worker without touching the API tier. + */ +export function startWorkerHealthServer( + sources: WorkerHealthSources, + port: number, + host = "0.0.0.0" +): Promise { + const server = http.createServer((req, res) => { + void handle(req, res, sources); + }); + return new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(port, host, () => { + server.off("error", reject); + resolve(server); + }); + }); +} + +async function handle(req: http.IncomingMessage, res: http.ServerResponse, sources: WorkerHealthSources) { + const path = (req.url ?? "/").split("?")[0]; + if (req.method !== "GET") { + res.writeHead(405, { Allow: "GET" }).end(); + return; + } + try { + if (path === "/health") { + const status = sources.heartbeat.status(); + res + .writeHead(status.healthy ? 200 : 503, { "Content-Type": "application/json" }) + .end(JSON.stringify({ status: status.healthy ? "ok" : "stale", role: "worker", ...status })); + return; + } + if (path === "/metrics" && sources.metrics) { + const { body, contentType } = await sources.metrics(); + res.writeHead(200, { "Content-Type": contentType }).end(body); + return; + } + res.writeHead(404).end(); + } catch (err) { + res.writeHead(500, { "Content-Type": "application/json" }).end(JSON.stringify({ error: (err as Error).message })); + } +} + +export function boundPort(server: http.Server): number { + return (server.address() as AddressInfo).port; +} diff --git a/src/worker/worker.module.ts b/src/worker/worker.module.ts new file mode 100644 index 0000000..bfd4032 --- /dev/null +++ b/src/worker/worker.module.ts @@ -0,0 +1,48 @@ +import { Module } from "@nestjs/common"; +import { ConfigModule } from "@nestjs/config"; +import { ScheduleModule } from "@nestjs/schedule"; +import configuration from "../config/configuration"; +import { AlertingModule } from "../alerting/alerting.module"; +import { CacheModule } from "../cache/cache.module"; +import { ClaimModule } from "../claim/claim.module"; +import { DatabaseModule } from "../database/database.module"; +import { DbModule } from "../db/db.module"; +import { IdempotencyModule } from "../common/idempotency.module"; +import { MetricsModule } from "../metrics/metrics.module"; +import { OracleModule } from "../oracle/oracle.module"; +import { ReadingBusModule } from "../oracle/reading-bus.module"; +import { PolicyModule } from "../policy/policy.module"; +import { StellarModule } from "../stellar/stellar.module"; +import { RelayerMonitorModule } from "../stellar/relayer-monitor/relayer-monitor.module"; +import { RuntimeModule } from "../runtime/runtime.module"; + +/** + * Root module for the worker tier: oracle polling, consensus evaluation, + * on-chain publishing, claim settlement and relayer balance monitoring. + * + * It shares the data layer with the API but leaves out everything that + * serves requests — controllers are never bound (the worker is started as + * an application context, not an HTTP app), and the throttler, API-key + * guard, quote/pool/tx/security-report modules are not imported at all. + * Readings reach the API through the ReadingBus. + */ +@Module({ + imports: [ + ConfigModule.forRoot({ isGlobal: true, load: [configuration] }), + ScheduleModule.forRoot(), + RuntimeModule, + DatabaseModule, + DbModule, + IdempotencyModule, + CacheModule, + StellarModule, + MetricsModule, + AlertingModule, + ReadingBusModule, + PolicyModule, + OracleModule, + ClaimModule, + RelayerMonitorModule, + ], +}) +export class WorkerModule {}