feat(vindex): add wire codecs and metrics leaf packages

.gitignore

@@ -6,8 +6,12 @@
 /logandmap
 /sumdbindex
 /sumdbverify
+/vindexd
 
 # Go test binaries and coverage/profile outputs
 *.test
 *.out
 *.prof
+
+# WebAssembly binaries
+*.wasm

go.mod

@@ -5,78 +5,24 @@ go 1.27.0
 toolchain go1.27.3
 
 require (
-	filippo.io/sunlight v0.10.2
-	filippo.io/torchwood v0.10.1-0.20261024135344-1c05157b1fd2
-	github.com/cockroachdb/pebble v1.1.5
-	github.com/go-git/go-git/v5 v5.19.3
-	github.com/google/go-cmp v0.7.0
-	github.com/gorilla/mux v1.8.1
 	github.com/prometheus/client_golang v1.25.0
-	github.com/transparency-dev/formats v0.1.1
+	github.com/transparency-dev/formats v0.1.2-0.20261027211732-55fe9bf0fa53
 	github.com/transparency-dev/merkle v0.0.2
-	github.com/transparency-dev/tessera v1.0.4
-	go.opentelemetry.io/otel v1.47.0
-	go.opentelemetry.io/otel/exporters/prometheus v0.69.0
-	go.opentelemetry.io/otel/metric v1.47.0
-	go.opentelemetry.io/otel/sdk/metric v1.47.0
-	golang.org/x/crypto v0.58.0
 	golang.org/x/mod v0.42.0
-	golang.org/x/net v0.62.0
-	golang.org/x/sync v0.24.0
 	k8s.io/klog/v2 v2.140.0
 )
 
 require (
-	dario.cat/mergo v1.0.0 // indirect
-	filippo.io/mldsa v0.0.0-20260711112038-ff3f469cee29 // indirect
-	github.com/DataDog/zstd v1.4.5 // indirect
-	github.com/Microsoft/go-winio v0.6.2 // indirect
-	github.com/ProtonMail/go-crypto v1.1.6 // indirect
+	filippo.io/mldsa v1.0.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/cenkalti/backoff/v5 v5.0.3 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
-	github.com/cloudflare/circl v1.6.3 // indirect
-	github.com/cockroachdb/errors v1.11.3 // indirect
-	github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce // indirect
-	github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
-	github.com/cockroachdb/redact v1.1.5 // indirect
-	github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect
-	github.com/cyphar/filepath-securejoin v0.6.1 // indirect
-	github.com/emirpasic/gods v1.18.1 // indirect
-	github.com/getsentry/sentry-go v0.27.0 // indirect
-	github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
-	github.com/go-git/go-billy/v5 v5.9.2 // indirect
 	github.com/go-logr/logr v1.4.4 // indirect
-	github.com/go-logr/stdr v1.2.2 // indirect
-	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
-	github.com/golang/snappy v0.0.4 // indirect
-	github.com/google/certificate-transparency-go v1.3.3 // indirect
-	github.com/google/uuid v1.6.0 // indirect
-	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
-	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
-	github.com/kevinburke/ssh_config v1.2.0 // indirect
-	github.com/klauspost/compress v1.20.1 // indirect
-	github.com/kr/pretty v0.3.1 // indirect
-	github.com/kr/text v0.2.0 // indirect
+	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
-	github.com/pjbgf/sha1cd v0.7.0 // indirect
-	github.com/pkg/errors v0.9.1 // indirect
 	github.com/prometheus/client_model v0.6.3 // indirect
 	github.com/prometheus/common v0.72.0 // indirect
-	github.com/prometheus/otlptranslator v1.0.0 // indirect
 	github.com/prometheus/procfs v0.22.0 // indirect
-	github.com/rogpeppe/go-internal v1.15.0 // indirect
-	github.com/sergi/go-diff v1.4.0 // indirect
-	github.com/skeema/knownhosts v1.3.1 // indirect
-	github.com/xanzy/ssh-agent v0.3.3 // indirect
-	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
-	go.opentelemetry.io/otel/log v1.47.0 // indirect
-	go.opentelemetry.io/otel/sdk v1.47.0 // indirect
-	go.opentelemetry.io/otel/trace v1.47.0 // indirect
-	golang.org/x/exp v0.0.0-20260603202125-055de637280b // indirect
+	golang.org/x/crypto v0.58.0 // indirect
 	golang.org/x/sys v0.49.0 // indirect
-	golang.org/x/text v0.43.0 // indirect
 	google.golang.org/protobuf v1.36.12 // indirect
-	gopkg.in/warnings.v0 v0.1.2 // indirect
 )

vindex/internal/metrics/metrics.go

@@ -0,0 +1,192 @@
+// Copyright 2026 The Transparency Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package metrics defines the Prometheus observability metrics for VIndex.
+package metrics
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+var (
+	// Checkpoint Progression Gauges (Monotonic Sizes)
+	InputTreeSize = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "vindex_input_tree_size",
+		Help: "Latest discovered size of the upstream Input Log checkpoint.",
+	})
+
+	KVCommittedSize = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "vindex_kv_committed_size",
+		Help: "Highest leaf index + 1 durably committed to the Pebble KV store (m_kv_size).",
+	})
+
+	OutputTreeSize = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "vindex_output_tree_size",
+		Help: "Total committed size of the Output Log.",
+	})
+
+	ServingTreeSize = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "vindex_serving_tree_size",
+		Help: "Input Log size covered by the active serving state exposed to readers.",
+	})
+
+	// Stage Throughput Monotonic Counters
+	LeavesDownloadedTotal = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "vindex_leaves_downloaded_total",
+		Help: "Cumulative number of raw input log leaves downloaded from upstream.",
+	})
+
+	LeavesMappedTotal = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "vindex_leaves_mapped_total",
+		Help: "Cumulative number of input log leaves executed through the MapFn.",
+	})
+
+	LeavesIndexedTotal = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "vindex_leaves_indexed_total",
+		Help: "Cumulative number of input log leaves committed into Pebble inverted chunks.",
+	})
+
+	KeysMappedTotal = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "vindex_keys_mapped_total",
+		Help: "Cumulative number of search key entries produced by the MapFn.",
+	})
+
+	// Pipeline Health & Subsystem Metrics
+	InputFetchErrorsTotal = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "vindex_input_fetch_errors_total",
+		Help: "Total number of network or HTTP errors encountered while polling/fetching from the upstream log.",
+	})
+
+	WitnessSignaturesCount = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "vindex_witness_signatures_count",
+		Help: "Number of valid witness signatures attached to the latest active Output Log checkpoint.",
+	})
+
+	WitnessErrorsTotal = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "vindex_witness_errors_total",
+		Help: "Total number of failed witness signing attempts.",
+	})
+
+	TileCacheBytes = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "vindex_tile_cache_bytes",
+		Help: "Estimated total bytes stored in the local managed tile cache directory.",
+	})
+
+	// Ingestion Metrics
+	IngestionLag = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "vindex_ingestion_lag",
+		Help: "Difference between Input Log size and last indexed leaf index (m_kv_size).",
+	})
+
+	MapDurationSeconds = promauto.NewHistogram(prometheus.HistogramOpts{
+		Name:    "vindex_map_duration_seconds",
+		Help:    "Time spent executing the WASM MapFn per leaf bundle in seconds.",
+		Buckets: prometheus.DefBuckets,
+	})
+
+	MapErrorsTotal = promauto.NewCounterVec(prometheus.CounterOpts{
+		Name: "vindex_map_errors_total",
+		Help: "Rate of mapping failures labeled by policy.",
+	}, []string{"policy"})
+
+	// Indexing & Commit Metrics
+	IndexingLag = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "vindex_indexing_lag",
+		Help: "Difference between m_kv_size and Output_Size.",
+	})
+
+	WitnessWaitSeconds = promauto.NewHistogram(prometheus.HistogramOpts{
+		Name:    "vindex_witness_wait_seconds",
+		Help:    "Latency waiting for remote Output Log witness signatures in seconds.",
+		Buckets: prometheus.DefBuckets,
+	})
+
+	MPTLockWaitSeconds = promauto.NewHistogram(prometheus.HistogramOpts{
+		Name:    "vindex_mpt_lock_wait_seconds",
+		Help:    "Wait duration to acquire mpt_lock.Lock() in seconds.",
+		Buckets: prometheus.DefBuckets,
+	})
+
+	MPTPredictDurationSeconds = promauto.NewHistogram(prometheus.HistogramOpts{
+		Name:    "vindex_mpt_predict_duration_seconds",
+		Help:    "Time spent predicting MPT root hash without lock in seconds.",
+		Buckets: prometheus.DefBuckets,
+	})
+
+	MPTWriteDurationSeconds = promauto.NewHistogram(prometheus.HistogramOpts{
+		Name:    "vindex_mpt_write_duration_seconds",
+		Help:    "Critical section duration applying MPT mutations in seconds.",
+		Buckets: prometheus.DefBuckets,
+	})
+
+	OutputLogAppendDurationSeconds = promauto.NewHistogram(prometheus.HistogramOpts{
+		Name:    "vindex_output_log_append_duration_seconds",
+		Help:    "Time spent appending leaf and fetching proof from Output Log in seconds.",
+		Buckets: prometheus.DefBuckets,
+	})
+
+	PebbleApplyDurationSeconds = promauto.NewHistogram(prometheus.HistogramOpts{
+		Name:    "vindex_pebble_apply_duration_seconds",
+		Help:    "Time spent committing atomic batch to Pebble in seconds.",
+		Buckets: prometheus.DefBuckets,
+	})
+
+	// Lookup Metrics
+	LookupLatencySeconds = promauto.NewHistogram(prometheus.HistogramOpts{
+		Name:    "vindex_lookup_latency_seconds",
+		Help:    "Latency of the Lookup API endpoint in seconds.",
+		Buckets: prometheus.DefBuckets,
+	})
+
+	LookupResultsReturned = promauto.NewHistogram(prometheus.HistogramOpts{
+		Name:    "vindex_lookup_results_returned",
+		Help:    "Count of index pointers returned per lookup.",
+		Buckets: prometheus.ExponentialBuckets(1, 2, 16),
+	})
+
+	// Storage Metrics
+	MPTMMapBytes = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "vindex_mpt_mmap_bytes",
+		Help: "Virtual memory size of the MPT mmap allocation.",
+	})
+
+	// Independent Auditor Metrics
+	AuditorRootMismatchesTotal = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "vindex_auditor_root_mismatches_total",
+		Help: "Cumulative number of detected root hash mismatches between local MPT calculation and OutputLog leaf commitments.",
+	})
+
+	AuditorRootMismatch = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "vindex_auditor_root_mismatch",
+		Help: "Current root mismatch status (0 = no mismatch / healthy, 1 = root mismatch detected).",
+	})
+)
+
+// RecordAuditorRootMismatch records a root hash mismatch event by incrementing
+// AuditorRootMismatchesTotal and setting AuditorRootMismatch to 1.
+func RecordAuditorRootMismatch() {
+	AuditorRootMismatchesTotal.Inc()
+	AuditorRootMismatch.Set(1)
+}
+
+// ResetAuditorRootMismatch resets the AuditorRootMismatch gauge to 0 (healthy).
+func ResetAuditorRootMismatch() {
+	AuditorRootMismatch.Set(0)
+}
+
+// SetAuditorRootMismatch sets the AuditorRootMismatch gauge to val.
+func SetAuditorRootMismatch(val float64) {
+	AuditorRootMismatch.Set(val)
+}