From ce116ed53ca9a81e08536543f1d7c4817d334e19 Mon Sep 17 00:00:00 2001 From: Vaibhav Patel Date: Tue, 4 Aug 2026 22:45:01 -0400 Subject: [PATCH 01/60] feat(gantry): capture benchmark performance telemetry --- cmd/gantry/agent_byte_metrics_test.go | 18 +- cmd/gantry/agent_metrics.go | 139 ++++--- cmd/gantry/main.go | 15 +- cmd/gantry/stream_commit_tracker.go | 76 +++- cmd/gantry/stream_commit_tracker_test.go | 37 +- hack/cmd/gantry-benchmark/enable.go | 14 + hack/cmd/gantry-benchmark/enable_test.go | 32 +- hack/cmd/gantry-benchmark/gantry_only.go | 36 +- hack/cmd/gantry-benchmark/job.go | 34 +- hack/cmd/gantry-benchmark/job_test.go | 9 +- hack/cmd/gantry-benchmark/peer_telemetry.go | 234 ++++++++++++ .../gantry-benchmark/peer_telemetry_test.go | 143 ++++++- .../gantry-benchmark/performance_telemetry.go | 356 ++++++++++++++++++ .../performance_telemetry_test.go | 122 ++++++ hack/cmd/gantry-benchmark/preflight.go | 89 +++++ hack/cmd/gantry-benchmark/results.go | 36 +- hack/cmd/gantry-benchmark/run.go | 87 ++++- hack/gantry-benchmark/README.md | 42 +++ .../manifests/monitoring.yaml.tmpl | 166 +++++++- internal/gantry/mirror/byte_metrics_test.go | 24 +- internal/gantry/mirror/mirror.go | 22 ++ .../phases/nodestart/assets/containerd.toml | 4 + pkg/agent/phases/nodestart/cri_test.go | 20 + 23 files changed, 1639 insertions(+), 116 deletions(-) create mode 100644 hack/cmd/gantry-benchmark/performance_telemetry.go create mode 100644 hack/cmd/gantry-benchmark/performance_telemetry_test.go diff --git a/cmd/gantry/agent_byte_metrics_test.go b/cmd/gantry/agent_byte_metrics_test.go index e700efdb5..565c3a51c 100644 --- a/cmd/gantry/agent_byte_metrics_test.go +++ b/cmd/gantry/agent_byte_metrics_test.go @@ -13,6 +13,7 @@ func TestByteMetricFamiliesMaterializeBoundedLabelsAtStartup(t *testing.T) { reg := metrics.New() _ = newPhase1Metrics(reg) _ = newPhase2Metrics(reg) + _ = newPhase9Metrics(reg) families, err := reg.PrometheusRegistry().Gather() if err != nil { @@ -25,10 +26,19 @@ func TestByteMetricFamiliesMaterializeBoundedLabelsAtStartup(t *testing.T) { } want := map[string]int{ - "gantry_origin_bytes_total": 3, - "gantry_peer_fetch_bytes_total": 3, - "gantry_peer_serve_bytes_total": 3, - "gantry_mirror_bytes_served_total": 9, + "gantry_origin_bytes_total": 3, + "gantry_peer_fetch_bytes_total": 3, + "gantry_peer_serve_bytes_total": 3, + "gantry_mirror_bytes_served_total": 9, + "gantry_mirror_response_completed_timestamp_seconds": 9, + "p2p_peer_fetch_total": 10, + "gantry_peer_fetch_last_timestamp_seconds": 2, + "p2p_peer_fetch_duration_seconds": 10, + "p2p_dht_lookup_total": 4, + "p2p_dht_lookup_duration_seconds": 4, + "gantry_containerd_commit_observed_timestamp_seconds": 1, + "gantry_containerd_commit_observation_duration_seconds": 1, + "gantry_containerd_commit_latest_observation_duration_seconds": 1, } for name, wantSeries := range want { diff --git a/cmd/gantry/agent_metrics.go b/cmd/gantry/agent_metrics.go index 5c1412dcb..0d33cb22b 100644 --- a/cmd/gantry/agent_metrics.go +++ b/cmd/gantry/agent_metrics.go @@ -76,22 +76,24 @@ func newPhase1Metrics(reg *metrics.Registry) *phase1Metrics { // phase2Metrics groups metrics for peer fallback, DHT advertise, // and transfer endpoint (the design doc). type phase2Metrics struct { - peerServe prometheus.Counter - peerServeBytes *prometheus.CounterVec - peerMiss prometheus.Counter - peerFetch *prometheus.CounterVec - peerFetchBytes *prometheus.CounterVec - mirrorServeBytes *prometheus.CounterVec - peerFetchDur *prometheus.HistogramVec - peerDialSuccess prometheus.Counter - peerDialFailure prometheus.Counter - dhtProvide prometheus.Counter - dhtProvideErr *prometheus.CounterVec - dhtReconcile prometheus.Counter - dhtLookup *prometheus.CounterVec - dhtLookupDur *prometheus.HistogramVec - dhtAdvertise prometheus.Counter - cdsubReconnect prometheus.Counter + peerServe prometheus.Counter + peerServeBytes *prometheus.CounterVec + peerMiss prometheus.Counter + peerFetch *prometheus.CounterVec + peerFetchLastAt *prometheus.GaugeVec + peerFetchBytes *prometheus.CounterVec + mirrorServeBytes *prometheus.CounterVec + mirrorCompletedAt *prometheus.GaugeVec + peerFetchDur *prometheus.HistogramVec + peerDialSuccess prometheus.Counter + peerDialFailure prometheus.Counter + dhtProvide prometheus.Counter + dhtProvideErr *prometheus.CounterVec + dhtReconcile prometheus.Counter + dhtLookup *prometheus.CounterVec + dhtLookupDur *prometheus.HistogramVec + dhtAdvertise prometheus.Counter + cdsubReconnect prometheus.Counter } func newPhase2Metrics(reg *metrics.Registry) *phase2Metrics { @@ -112,6 +114,10 @@ func newPhase2Metrics(reg *metrics.Registry) *phase2Metrics { Name: "p2p_peer_fetch_total", Help: "Peer fetches initiated by the mirror miss path.", }, []string{"outcome"}), + peerFetchLastAt: reg.NewGaugeVec("mirror", prometheus.GaugeOpts{ + Name: "gantry_peer_fetch_last_timestamp_seconds", + Help: "Unix timestamp of the most recent peer fetch event, retained for busy and stall outcomes.", + }, []string{"outcome"}), peerFetchBytes: reg.NewCounterVec("mirror", prometheus.CounterOpts{ Name: "gantry_peer_fetch_bytes_total", Help: "Bytes received from peer Gantry agents, including partial failed transfers and retries, labeled by OCI content kind.", @@ -120,6 +126,10 @@ func newPhase2Metrics(reg *metrics.Registry) *phase2Metrics { Name: "gantry_mirror_bytes_served_total", Help: "Bytes written by the containerd-facing mirror, labeled by OCI content kind and source path (cache, peer, or origin).", }, []string{"kind", "source"}), + mirrorCompletedAt: reg.NewGaugeVec("mirror", prometheus.GaugeOpts{ + Name: "gantry_mirror_response_completed_timestamp_seconds", + Help: "Unix timestamp when a complete response body was most recently written to the local containerd client, labeled by content kind and source path.", + }, []string{"kind", "source"}), peerFetchDur: reg.NewHistogramVec("mirror", prometheus.HistogramOpts{ Name: "p2p_peer_fetch_duration_seconds", Help: "End-to-end peer-fetch latency from FetchFromPeer dial to terminal outcome (hit = cache commit, error/stall/notfound = first failing branch). Together with p2p_peer_fetch_total{outcome} this isolates dial vs. body vs. commit-time-digest-verification slowness.", @@ -170,8 +180,31 @@ func newPhase2Metrics(reg *metrics.Registry) *phase2Metrics { for _, source := range []string{"cache", "peer", "origin"} { p.mirrorServeBytes.WithLabelValues(kind, source).Add(0) + p.mirrorCompletedAt.WithLabelValues(kind, source).Set(0) } } + for _, outcome := range []string{ + "hit", + "notfound", + "unavailable", + "digest_mismatch", + "auth_or_config", + "server_error", + "protocol_error", + "stall", + "local_error", + "busy", + } { + p.peerFetch.WithLabelValues(outcome).Add(0) + p.peerFetchDur.WithLabelValues(outcome) + } + for _, outcome := range []string{"busy", "stall"} { + p.peerFetchLastAt.WithLabelValues(outcome).Set(0) + } + for _, outcome := range []string{"hit", "miss", "error", "timeout"} { + p.dhtLookup.WithLabelValues(outcome).Add(0) + p.dhtLookupDur.WithLabelValues(outcome) + } return p } @@ -383,35 +416,38 @@ func newPhase5Metrics(reg *metrics.Registry, healthScore func() float64) *phase5 // in its content inventory" so the agent does not pretend a local // commit happened just because the HTTP stream completed. type phase9Metrics struct { - storageMode *prometheus.GaugeVec - advReconcileTotal prometheus.Counter - advReconcileError prometheus.Counter - advReconcileUnavailable prometheus.Counter - advReconcileDur prometheus.Histogram - advReconcileDigestCount prometheus.Gauge - advReconcileAdded prometheus.Counter - advReconcileRemoved prometheus.Counter - withdrawTotal prometheus.Counter - withdrawError prometheus.Counter - containerdLeaseCreated prometheus.Counter - containerdLeaseReleased prometheus.Counter - containerdLeaseActive prometheus.Gauge - containerdLeaseCleanupErr prometheus.Counter - containerdIngestTotal prometheus.Counter - containerdIngestFailure prometheus.Counter - containerdHit prometheus.Counter - containerdMiss prometheus.Counter - containerdUnavailable prometheus.Counter - containerdOpenError prometheus.Counter - originStreamStarted *prometheus.CounterVec - originStreamCompleted *prometheus.CounterVec - originStreamFailed *prometheus.CounterVec - containerdCommitObserved prometheus.Counter - dhtStaleOnly prometheus.Counter - staleProviderFiltered prometheus.Counter - commitMissingAfterStream prometheus.Counter - advertiseTotal prometheus.Counter - advertiseError prometheus.Counter + storageMode *prometheus.GaugeVec + advReconcileTotal prometheus.Counter + advReconcileError prometheus.Counter + advReconcileUnavailable prometheus.Counter + advReconcileDur prometheus.Histogram + advReconcileDigestCount prometheus.Gauge + advReconcileAdded prometheus.Counter + advReconcileRemoved prometheus.Counter + withdrawTotal prometheus.Counter + withdrawError prometheus.Counter + containerdLeaseCreated prometheus.Counter + containerdLeaseReleased prometheus.Counter + containerdLeaseActive prometheus.Gauge + containerdLeaseCleanupErr prometheus.Counter + containerdIngestTotal prometheus.Counter + containerdIngestFailure prometheus.Counter + containerdHit prometheus.Counter + containerdMiss prometheus.Counter + containerdUnavailable prometheus.Counter + containerdOpenError prometheus.Counter + originStreamStarted *prometheus.CounterVec + originStreamCompleted *prometheus.CounterVec + originStreamFailed *prometheus.CounterVec + containerdCommitObserved prometheus.Counter + containerdCommitObservedAt prometheus.Gauge + containerdCommitObserveDur prometheus.Histogram + containerdCommitLatestDur prometheus.Gauge + dhtStaleOnly prometheus.Counter + staleProviderFiltered prometheus.Counter + commitMissingAfterStream prometheus.Counter + advertiseTotal prometheus.Counter + advertiseError prometheus.Counter } func newPhase9Metrics(reg *metrics.Registry) *phase9Metrics { @@ -513,6 +549,19 @@ func newPhase9Metrics(reg *metrics.Registry) *phase9Metrics { Name: "gantry_containerd_commit_observed_total", Help: "Completed live stream-through responses whose digest later appeared in the local containerd inventory within the verification window. This is the truthful post-stream commit signal for live mirror traffic.", }), + containerdCommitObservedAt: reg.NewGauge("storage", prometheus.GaugeOpts{ + Name: "gantry_containerd_commit_observed_timestamp_seconds", + Help: "Unix timestamp when containerd inventory most recently showed a digest from a completed live stream-through response.", + }), + containerdCommitObserveDur: reg.NewHistogram("storage", prometheus.HistogramOpts{ + Name: "gantry_containerd_commit_observation_duration_seconds", + Help: "Time from a digest-verified live stream-through response completing to the digest appearing in containerd inventory. Resolution is bounded by the inventory probe interval.", + Buckets: prometheus.ExponentialBuckets(0.25, 2, 9), + }), + containerdCommitLatestDur: reg.NewGauge("storage", prometheus.GaugeOpts{ + Name: "gantry_containerd_commit_latest_observation_duration_seconds", + Help: "Most recent measured time from a digest-verified live stream-through response completing to the digest appearing in containerd inventory. Resolution is bounded by the inventory probe interval.", + }), dhtStaleOnly: reg.NewCounter("discovery", prometheus.CounterOpts{ Name: "gantry_dht_stale_only_total", Help: "Mirror cache-miss requests where the DHT returned candidate providers but every candidate was filtered (stale, suspicious, self, unavailable) before any peer fetch was attempted. Treated as if DHT returned empty - falls through to cold-start. Distinct from gantry_dht_lookup_total{outcome=\"miss\"} which counts true empty results.", diff --git a/cmd/gantry/main.go b/cmd/gantry/main.go index 71f3ab5b6..2769f94ae 100644 --- a/cmd/gantry/main.go +++ b/cmd/gantry/main.go @@ -563,6 +563,11 @@ func runAgent(args []string) error { streamCommitTracker := newStreamCommitTracker(containerdInv, logger, func(n int) { p9.containerdCommitObserved.Add(float64(n)) }, + func(duration time.Duration) { + p9.containerdCommitObserveDur.Observe(duration.Seconds()) + p9.containerdCommitLatestDur.Set(duration.Seconds()) + p9.containerdCommitObservedAt.SetToCurrentTime() + }, func(n int) { p9.commitMissingAfterStream.Add(float64(n)) }, ) @@ -588,6 +593,9 @@ func runAgent(args []string) error { p2.mirrorServeBytes.WithLabelValues(kind, source).Add(float64(bytes)) }, ), + mirror.WithMirrorResponseCompletedHook(func(kind, source string) { + p2.mirrorCompletedAt.WithLabelValues(kind, source).SetToCurrentTime() + }), mirror.WithOriginStreamMetrics( func(kind string) { p9.originStreamStarted.WithLabelValues(kind).Inc() }, func(kind string) { p9.originStreamCompleted.WithLabelValues(kind).Inc() }, @@ -609,7 +617,12 @@ func runAgent(args []string) error { mirror.WithSelfNodeID(memberView.Self()), mirror.WithSelfPeerID(ifaces.NodeID(disco.PeerID().String())), mirror.WithPeerMetrics( - func(outcome string) { p2.peerFetch.WithLabelValues(outcome).Inc() }, + func(outcome string) { + p2.peerFetch.WithLabelValues(outcome).Inc() + if outcome == "busy" || outcome == "stall" { + p2.peerFetchLastAt.WithLabelValues(outcome).SetToCurrentTime() + } + }, func(success bool) { if success { p2.peerDialSuccess.Inc() diff --git a/cmd/gantry/stream_commit_tracker.go b/cmd/gantry/stream_commit_tracker.go index 3e0abdb99..2e22c2160 100644 --- a/cmd/gantry/stream_commit_tracker.go +++ b/cmd/gantry/stream_commit_tracker.go @@ -7,6 +7,7 @@ import ( "context" "errors" "log/slog" + "sort" "sync" "time" @@ -37,27 +38,45 @@ type streamCommitTracker struct { verifyWindow time.Duration inventoryBudget time.Duration - onObserved func(n int) - onMissing func(n int) + onObserved func(n int) + onObservedDuration func(time.Duration) + onMissing func(n int) mu sync.Mutex - pending map[string][]time.Time + pending map[string][]pendingStreamCommit } -func newStreamCommitTracker(inv inventorySource, logger *slog.Logger, onObserved, onMissing func(n int)) *streamCommitTracker { +type pendingStreamCommit struct { + completedAt time.Time + deadline time.Time +} + +type observedStreamCommit struct { + completedAt time.Time + duration time.Duration +} + +func newStreamCommitTracker( + inv inventorySource, + logger *slog.Logger, + onObserved func(n int), + onObservedDuration func(time.Duration), + onMissing func(n int), +) *streamCommitTracker { if logger == nil { logger = slog.Default() } return &streamCommitTracker{ - inv: inv, - logger: logger.With(slog.String("subsystem", "stream_commit_tracker")), - probeInterval: defaultStreamCommitProbeInterval, - verifyWindow: defaultStreamCommitVerifyWindow, - inventoryBudget: defaultStreamCommitInventoryBudget, - onObserved: onObserved, - onMissing: onMissing, - pending: map[string][]time.Time{}, + inv: inv, + logger: logger.With(slog.String("subsystem", "stream_commit_tracker")), + probeInterval: defaultStreamCommitProbeInterval, + verifyWindow: defaultStreamCommitVerifyWindow, + inventoryBudget: defaultStreamCommitInventoryBudget, + onObserved: onObserved, + onObservedDuration: onObservedDuration, + onMissing: onMissing, + pending: map[string][]pendingStreamCommit{}, } } @@ -67,7 +86,11 @@ func (t *streamCommitTracker) RecordCompleted(d digest.Digest) { t.mu.Lock() defer t.mu.Unlock() - t.pending[d.String()] = append(t.pending[d.String()], time.Now().Add(t.verifyWindow)) + completedAt := time.Now() + t.pending[d.String()] = append(t.pending[d.String()], pendingStreamCommit{ + completedAt: completedAt, + deadline: completedAt.Add(t.verifyWindow), + }) } func (t *streamCommitTracker) Run(ctx context.Context) error { @@ -123,25 +146,32 @@ func (t *streamCommitTracker) probe(parent context.Context) { now := time.Now() observed := 0 missing := 0 + observedCommits := make([]observedStreamCommit, 0) t.mu.Lock() - for ds, deadlines := range t.pending { + for ds, commits := range t.pending { if _, ok := present[ds]; ok { - observed += len(deadlines) + observed += len(commits) + for _, commit := range commits { + observedCommits = append(observedCommits, observedStreamCommit{ + completedAt: commit.completedAt, + duration: now.Sub(commit.completedAt), + }) + } delete(t.pending, ds) continue } - kept := make([]time.Time, 0, len(deadlines)) - for _, deadline := range deadlines { - if now.After(deadline) || now.Equal(deadline) { + kept := make([]pendingStreamCommit, 0, len(commits)) + for _, commit := range commits { + if now.After(commit.deadline) || now.Equal(commit.deadline) { missing++ continue } - kept = append(kept, deadline) + kept = append(kept, commit) } if len(kept) == 0 { @@ -156,6 +186,14 @@ func (t *streamCommitTracker) probe(parent context.Context) { if observed > 0 && t.onObserved != nil { t.onObserved(observed) } + sort.Slice(observedCommits, func(i, j int) bool { + return observedCommits[i].completedAt.Before(observedCommits[j].completedAt) + }) + if t.onObservedDuration != nil { + for _, commit := range observedCommits { + t.onObservedDuration(commit.duration) + } + } if missing > 0 && t.onMissing != nil { t.onMissing(missing) diff --git a/cmd/gantry/stream_commit_tracker_test.go b/cmd/gantry/stream_commit_tracker_test.go index fc058c70a..98ee74860 100644 --- a/cmd/gantry/stream_commit_tracker_test.go +++ b/cmd/gantry/stream_commit_tracker_test.go @@ -91,10 +91,16 @@ func TestStreamCommitTracker_ObservedAfterInventoryAppears(t *testing.T) { d := trackerDigestOf([]byte("observed-after-stream")) inv := &fakeInventorySource{} - var observed, missing int32 + var observed, missing, durations int32 tracker := newStreamCommitTracker(inv, nil, func(n int) { atomic.AddInt32(&observed, int32(n)) }, + func(duration time.Duration) { + if duration <= 0 { + t.Errorf("observed duration = %s, want positive", duration) + } + atomic.AddInt32(&durations, 1) + }, func(n int) { atomic.AddInt32(&missing, int32(n)) }, ) tracker.probeInterval = 5 * time.Millisecond @@ -111,6 +117,7 @@ func TestStreamCommitTracker_ObservedAfterInventoryAppears(t *testing.T) { inv.SetCurrent(d) waitForAtomic(t, &observed, 1) + waitForAtomic(t, &durations, 1) if got := atomic.LoadInt32(&missing); got != 0 { t.Fatalf("missing = %d, want 0", got) @@ -125,6 +132,7 @@ func TestStreamCommitTracker_MissingAfterDeadline(t *testing.T) { tracker := newStreamCommitTracker(inv, nil, func(n int) { atomic.AddInt32(&observed, int32(n)) }, + nil, func(n int) { atomic.AddInt32(&missing, int32(n)) }, ) tracker.probeInterval = 5 * time.Millisecond @@ -153,6 +161,7 @@ func TestStreamCommitTracker_RetriesAfterUnavailableInventory(t *testing.T) { tracker := newStreamCommitTracker(inv, nil, func(n int) { atomic.AddInt32(&observed, int32(n)) }, + nil, func(n int) { atomic.AddInt32(&missing, int32(n)) }, ) tracker.probeInterval = 5 * time.Millisecond @@ -174,3 +183,29 @@ func TestStreamCommitTracker_RetriesAfterUnavailableInventory(t *testing.T) { t.Fatalf("missing = %d, want 0", got) } } + +func TestStreamCommitTracker_ReportsLatestCompletedStreamLast(t *testing.T) { + earlier := trackerDigestOf([]byte("earlier")) + later := trackerDigestOf([]byte("later")) + inv := &fakeInventorySource{current: []digest.Digest{earlier, later}} + + var durations []time.Duration + tracker := newStreamCommitTracker(inv, nil, nil, func(duration time.Duration) { + durations = append(durations, duration) + }, nil) + now := time.Now() + tracker.pending[earlier.String()] = []pendingStreamCommit{{ + completedAt: now.Add(-2 * time.Second), + deadline: now.Add(time.Minute), + }} + tracker.pending[later.String()] = []pendingStreamCommit{{ + completedAt: now.Add(-time.Second), + deadline: now.Add(time.Minute), + }} + + tracker.probe(context.Background()) + + if len(durations) != 2 || durations[0] <= durations[1] { + t.Fatalf("durations = %v, want earlier completion before latest completion", durations) + } +} diff --git a/hack/cmd/gantry-benchmark/enable.go b/hack/cmd/gantry-benchmark/enable.go index 88aa6a3bf..57792e627 100644 --- a/hack/cmd/gantry-benchmark/enable.go +++ b/hack/cmd/gantry-benchmark/enable.go @@ -146,6 +146,8 @@ func (b *benchmark) enable(ctx context.Context) (returnErr error) { Namespace: b.config.Namespace, GantryNamespace: b.config.GantryNamespace, MonitoringLabel: b.config.KPSRelease, + NodeOS: strings.SplitN(b.config.ImagePlatform, "/", 2)[0], + NodeArch: strings.SplitN(b.config.ImagePlatform, "/", 2)[1], ProxyImage: b.config.ProxyImage, ACRLoginServer: b.config.ACRLoginServer, RunID: runID, @@ -162,6 +164,16 @@ func (b *benchmark) enable(ctx context.Context) (returnErr error) { return err } + if _, err := b.commands.Run( + ctx, + nil, + "kubectl", "-n", b.config.Namespace, + "rollout", "status", "daemonset/gantry-benchmark-node-observer", + "--timeout", b.config.RolloutTimeout.String(), + ); err != nil { + return fmt.Errorf("wait for benchmark node observer: %w", err) + } + if b.config.usesProxy() { secret := map[string]any{ "apiVersion": "v1", @@ -323,6 +335,8 @@ type proxyManifestData struct { Namespace string GantryNamespace string MonitoringLabel string + NodeOS string + NodeArch string ProxyImage string ACRLoginServer string RunID string diff --git a/hack/cmd/gantry-benchmark/enable_test.go b/hack/cmd/gantry-benchmark/enable_test.go index 1813ec14a..b7954e289 100644 --- a/hack/cmd/gantry-benchmark/enable_test.go +++ b/hack/cmd/gantry-benchmark/enable_test.go @@ -6,6 +6,7 @@ package main import ( "bytes" "io" + "slices" "strings" "testing" @@ -68,6 +69,8 @@ func TestRenderMonitoringManifest(t *testing.T) { Namespace: "gantry-benchmark", GantryNamespace: "gantry-system", MonitoringLabel: "kps", + NodeOS: "linux", + NodeArch: "amd64", RunID: "run-1", }) if err != nil { @@ -83,10 +86,30 @@ func TestRenderMonitoringManifest(t *testing.T) { t.Fatalf("rendered manifest is missing benchmark scrape or Gantry revision labels") } - if !strings.Contains(string(rendered), `action: keep`) || - !strings.Contains(string(rendered), `gantry_storage_mode_info|p2p_dht_health_score|gantry_peer_serve_bytes_total`) { + if !strings.Contains(string(rendered), `action: keep`) { t.Fatalf("rendered manifest does not limit Gantry metric cardinality") } + if !strings.Contains(string(rendered), `systemctl show --property MainPID --value containerd`) { + t.Fatalf("rendered manifest does not validate the running containerd debug configuration") + } + for _, metric := range []string{ + "p2p_peer_fetch_duration_seconds_(bucket|sum|count)", + "p2p_dht_lookup_duration_seconds_(bucket|sum|count)", + "gantry_peer_fetch_last_timestamp_seconds", + "gantry_mirror_response_completed_timestamp_seconds", + "gantry_containerd_commit_observation_duration_seconds_(bucket|sum|count)", + "gantry_containerd_commit_latest_observation_duration_seconds", + "node_uname_info", + "node_disk_(read|written)_bytes_total", + "node_network_speed_bytes", + "node_network_(receive|transmit)_(bytes|drop|errs)_total", + "containerd_.*|grpc_server_.*", + "process_(cpu_seconds_total|resident_memory_bytes|virtual_memory_bytes)", + } { + if !strings.Contains(string(rendered), metric) { + t.Fatalf("rendered manifest does not retain required metric %q", metric) + } + } if strings.Contains(string(rendered), "acr-origin-proxy") { t.Fatalf("monitoring manifest must not reference the proxy") @@ -94,8 +117,9 @@ func TestRenderMonitoringManifest(t *testing.T) { kinds := decodeManifestKinds(t, rendered) - if len(kinds) != 1 || kinds[0] != "PodMonitor" { - t.Fatalf("rendered kinds = %v, want [PodMonitor]", kinds) + wantKinds := []string{"PodMonitor", "DaemonSet", "PodMonitor"} + if !slices.Equal(kinds, wantKinds) { + t.Fatalf("rendered kinds = %v, want %v", kinds, wantKinds) } } diff --git a/hack/cmd/gantry-benchmark/gantry_only.go b/hack/cmd/gantry-benchmark/gantry_only.go index 223c5d7bc..33a42cf31 100644 --- a/hack/cmd/gantry-benchmark/gantry_only.go +++ b/hack/cmd/gantry-benchmark/gantry_only.go @@ -642,6 +642,11 @@ func (b *benchmark) runGantryOnly(ctx context.Context) (returnErr error) { return err } + diagnosticsBefore, err := b.fetchGantryDiagnosticSnapshot(ctx, revision) + if err != nil { + return err + } + writeAll(b.stdout, fmt.Sprintf("running Gantry-only cold pull on %d nodes\n", b.config.NodeCount)) job, err := b.runPullJob(ctx, state, proxyPhaseGantryCold, gantryImage) @@ -673,15 +678,42 @@ func (b *benchmark) runGantryOnly(ctx context.Context) (returnErr error) { return err } + diagnosticsAfter, err := b.fetchGantryDiagnosticSnapshot(ctx, revision) + if err != nil { + return err + } + diagnosticTimestamps, err := b.fetchGantryDiagnosticTimestamps(ctx, revision, telemetryWindow{ + StartedAt: job.PhaseStartedAt, + FinishedAt: job.PhaseFinishedAt, + }) + if err != nil { + return err + } + if err := requireFinalLayerResponseTimestamps(diagnosticTimestamps, diagnosticsAfter.PodNodes); err != nil { + return err + } + diagnostics, err := subtractGantryDiagnosticSnapshots(diagnosticsBefore, diagnosticsAfter, diagnosticTimestamps) + if err != nil { + return err + } + bytes, bytesSource := deriveOriginBytes(b.config, proxyPhaseGantryCold, proxyPhaseTotals{}, metrics, job) + performance, err := b.capturePhasePerformanceTelemetry(ctx, proxyPhaseGantryCold, job) + if err != nil { + return err + } + if err := b.writePerformanceTelemetryArtifact(state.RunID, proxyPhaseGantryCold, performance); err != nil { + return err + } gantryResult := phaseResult{ RunID: state.RunID, Phase: proxyPhaseGantryCold, Image: gantryImage, ImageSizeMiB: b.config.ImageSizeMiB, ImageLayers: b.config.ImageLayers, PayloadSHA: state.WorkloadPayloadSHA256, WorkloadComparisonMode: state.WorkloadComparisonMode, - Gantry: metrics, GantryPeer: peer, + Gantry: metrics, GantryPeer: peer, GantryDiagnostics: diagnostics, Azure: azurePhaseMeasurement{Window: telemetryWindow{StartedAt: windowStart, FinishedAt: windowFinish}}, - Job: job, OriginBytes: bytes, OriginBytesSource: bytesSource, RecordedAt: time.Now().UTC(), + Job: job, OriginBytes: bytes, OriginBytesSource: bytesSource, + PerformanceTelemetryArtifact: string(proxyPhaseGantryCold) + "-performance.json", RecordedAt: time.Now().UTC(), } if b.config.AzureTelemetry { if err := b.collectAndPersistAzurePhase(ctx, &gantryResult, "gantry-cold.json"); err != nil { diff --git a/hack/cmd/gantry-benchmark/job.go b/hack/cmd/gantry-benchmark/job.go index 1160c9446..32ec6cdcc 100644 --- a/hack/cmd/gantry-benchmark/job.go +++ b/hack/cmd/gantry-benchmark/job.go @@ -19,14 +19,23 @@ type latencySummary struct { } type jobObservation struct { - JobName string `json:"job_name"` - PhaseStartedAt time.Time `json:"phase_started_at"` - PhaseFinishedAt time.Time `json:"phase_finished_at"` - Nodes []string `json:"nodes"` - Pods []string `json:"pods"` - PodNodes map[string]string `json:"pod_nodes"` - PodStartLatency latencySummary `json:"pod_start_latency"` - PodFinishLatency latencySummary `json:"pod_finish_latency"` + JobName string `json:"job_name"` + PhaseStartedAt time.Time `json:"phase_started_at"` + PhaseFinishedAt time.Time `json:"phase_finished_at"` + Nodes []string `json:"nodes"` + Pods []string `json:"pods"` + PodNodes map[string]string `json:"pod_nodes"` + PodTimings map[string]podTimingObservation `json:"pod_timings"` + PodStartLatency latencySummary `json:"pod_start_latency"` + PodFinishLatency latencySummary `json:"pod_finish_latency"` +} + +type podTimingObservation struct { + NodeName string `json:"node_name"` + ContainerStartedAt time.Time `json:"container_started_at"` + ContainerFinishedAt time.Time `json:"container_finished_at"` + StartLatencySeconds float64 `json:"start_latency_seconds"` + FinishLatencySeconds float64 `json:"finish_latency_seconds"` } type podList struct { @@ -186,6 +195,7 @@ func parseJobObservation(raw []byte, expectedPods int, phaseStartedAt time.Time) nodeSet := make(map[string]struct{}, expectedPods) podNames := make([]string, 0, expectedPods) podNodes := make(map[string]string, expectedPods) + podTimings := make(map[string]podTimingObservation, expectedPods) startLatencies := make([]time.Duration, 0, expectedPods) finishLatencies := make([]time.Duration, 0, expectedPods) @@ -224,6 +234,13 @@ func parseJobObservation(raw []byte, expectedPods int, phaseStartedAt time.Time) startLatencies = append(startLatencies, terminated.StartedAt.Sub(phaseStartedAt)) finishLatencies = append(finishLatencies, terminated.FinishedAt.Sub(phaseStartedAt)) + podTimings[pod.Metadata.Name] = podTimingObservation{ + NodeName: pod.Spec.NodeName, + ContainerStartedAt: terminated.StartedAt, + ContainerFinishedAt: terminated.FinishedAt, + StartLatencySeconds: terminated.StartedAt.Sub(phaseStartedAt).Seconds(), + FinishLatencySeconds: terminated.FinishedAt.Sub(phaseStartedAt).Seconds(), + } terminatedFound = true break @@ -246,6 +263,7 @@ func parseJobObservation(raw []byte, expectedPods int, phaseStartedAt time.Time) Nodes: nodes, Pods: podNames, PodNodes: podNodes, + PodTimings: podTimings, PodStartLatency: summarizeLatencies(startLatencies), PodFinishLatency: summarizeLatencies(finishLatencies), }, nil diff --git a/hack/cmd/gantry-benchmark/job_test.go b/hack/cmd/gantry-benchmark/job_test.go index 99542decd..62dbdff11 100644 --- a/hack/cmd/gantry-benchmark/job_test.go +++ b/hack/cmd/gantry-benchmark/job_test.go @@ -37,8 +37,13 @@ func TestParseJobObservation(t *testing.T) { t.Fatalf("start latency = %+v", observation.PodStartLatency) } - if len(observation.Pods) != 4 || len(observation.PodNodes) != 4 { - t.Fatalf("pod identities = %v nodes=%v, want four", observation.Pods, observation.PodNodes) + if len(observation.Pods) != 4 || len(observation.PodNodes) != 4 || len(observation.PodTimings) != 4 { + t.Fatalf("pod identities = %v nodes=%v timings=%v, want four", observation.Pods, observation.PodNodes, observation.PodTimings) + } + podB := observation.PodTimings["pod-b"] + if podB.NodeName != "node-b" || podB.StartLatencySeconds != 20 || podB.FinishLatencySeconds != 21 || + !podB.ContainerStartedAt.Equal(phaseStartedAt.Add(20*time.Second)) { + t.Fatalf("pod-b timing = %+v, want exact node and timestamps", podB) } } diff --git a/hack/cmd/gantry-benchmark/peer_telemetry.go b/hack/cmd/gantry-benchmark/peer_telemetry.go index 53ac7c51a..0a6c3a3c4 100644 --- a/hack/cmd/gantry-benchmark/peer_telemetry.go +++ b/hack/cmd/gantry-benchmark/peer_telemetry.go @@ -11,6 +11,8 @@ import ( "net/url" "sort" "strconv" + "strings" + "time" ) type gantryPeerPodMeasurement struct { @@ -37,6 +39,25 @@ type peerByteSnapshot struct { Counters map[string]map[string]uint64 } +type gantryPodDiagnosticMeasurement struct { + PodName string `json:"pod_name"` + NodeName string `json:"node_name"` + CounterDeltas map[string]float64 `json:"counter_deltas"` + TimestampSeconds map[string]float64 `json:"timestamp_seconds"` + FinalLayerResponseCompletedTimestampSeconds float64 `json:"final_layer_response_completed_timestamp_seconds,omitempty"` +} + +type gantryDiagnosticPhaseMeasurement struct { + Pods []gantryPodDiagnosticMeasurement `json:"pods"` + Source string `json:"source"` + Complete bool `json:"complete"` +} + +type gantryDiagnosticSnapshot struct { + PodNodes map[string]string + Counters map[string]map[string]float64 +} + func (b *benchmark) queryPrometheusSamples(ctx context.Context, query string) ([]prometheusSample, error) { rawPath := fmt.Sprintf( "/api/v1/namespaces/%s/services/http:%s:9090/proxy/api/v1/query?query=%s", @@ -133,6 +154,219 @@ func (b *benchmark) gantryPodNodes(ctx context.Context, revision string) (map[st return result, nil } +func (b *benchmark) fetchGantryDiagnosticSnapshot(ctx context.Context, revision string) (gantryDiagnosticSnapshot, error) { + podNodes, err := b.gantryPodNodes(ctx, revision) + if err != nil { + return gantryDiagnosticSnapshot{}, err + } + + query := fmt.Sprintf( + `{__name__=~"p2p_peer_fetch_total|p2p_peer_fetch_duration_seconds_(sum|count)|p2p_dht_lookup_total|p2p_dht_lookup_duration_seconds_(sum|count)|gantry_peer_fetch_bytes_total|gantry_containerd_commit_observed_total|gantry_containerd_commit_observation_duration_seconds_(sum|count)|gantry_containerd_commit_missing_after_stream_total",namespace=%q,gantry_benchmark="true",controller_revision_hash=%q}`, + b.config.GantryNamespace, + revision, + ) + + samples, err := b.queryPrometheusSamples(ctx, query) + if err != nil { + return gantryDiagnosticSnapshot{}, err + } + + counters := make(map[string]map[string]float64, len(podNodes)) + for pod := range podNodes { + counters[pod] = map[string]float64{} + } + + for _, sample := range samples { + pod := sample.Metric["pod"] + if _, ok := podNodes[pod]; !ok { + return gantryDiagnosticSnapshot{}, fmt.Errorf("diagnostic sample belongs to unexpected pod %q", pod) + } + if sample.Value < 0 { + return gantryDiagnosticSnapshot{}, fmt.Errorf("diagnostic sample for pod %s is negative: %v", pod, sample.Value) + } + + key, err := diagnosticMetricKey(sample.Metric) + if err != nil { + return gantryDiagnosticSnapshot{}, err + } + counters[pod][key] = sample.Value + } + + return gantryDiagnosticSnapshot{PodNodes: podNodes, Counters: counters}, nil +} + +func diagnosticMetricKey(labels map[string]string) (string, error) { + name := labels["__name__"] + if name == "" { + return "", fmt.Errorf("diagnostic sample has no __name__ label") + } + + parts := make([]string, 0, 3) + for _, label := range []string{"kind", "outcome", "source"} { + if value := labels[label]; value != "" { + parts = append(parts, label+"="+value) + } + } + if len(parts) == 0 { + return name, nil + } + + return name + "{" + strings.Join(parts, ",") + "}", nil +} + +func (b *benchmark) fetchGantryDiagnosticTimestamps( + ctx context.Context, + revision string, + window telemetryWindow, +) (map[string]map[string]float64, error) { + query := fmt.Sprintf( + `gantry_mirror_response_completed_timestamp_seconds{namespace=%q,kind="layer",gantry_benchmark="true",controller_revision_hash=%q} or gantry_containerd_commit_observed_timestamp_seconds{namespace=%q,gantry_benchmark="true",controller_revision_hash=%q} or gantry_peer_fetch_last_timestamp_seconds{namespace=%q,outcome=~"busy|stall",gantry_benchmark="true",controller_revision_hash=%q}`, + b.config.GantryNamespace, + revision, + b.config.GantryNamespace, + revision, + b.config.GantryNamespace, + revision, + ) + + raw, err := b.queryPrometheusRange(ctx, query, window, performanceTelemetryStep) + if err != nil { + return nil, err + } + if err := validatePrometheusRangePodCoverage("gantry diagnostic timestamps", raw, b.config.NodeCount); err != nil { + return nil, err + } + var response struct { + Data struct { + Result []struct { + Metric map[string]string `json:"metric"` + Values [][2]any `json:"values"` + } `json:"result"` + } `json:"data"` + } + if err := json.Unmarshal(raw, &response); err != nil { + return nil, fmt.Errorf("decode diagnostic timestamp range response: %w", err) + } + + result := map[string]map[string]float64{} + for _, series := range response.Data.Result { + key, err := diagnosticMetricKey(series.Metric) + if err != nil { + return nil, err + } + pod := series.Metric["pod"] + for _, pair := range series.Values { + rawValue, ok := pair[1].(string) + if !ok { + return nil, fmt.Errorf("diagnostic timestamp sample has non-string value") + } + value, err := strconv.ParseFloat(rawValue, 64) + if err != nil { + return nil, fmt.Errorf("parse diagnostic timestamp sample %q: %w", rawValue, err) + } + observedAt := time.Unix(0, int64(value*float64(time.Second))) + if observedAt.Before(window.StartedAt) || observedAt.After(window.FinishedAt) { + continue + } + if result[pod] == nil { + result[pod] = map[string]float64{} + } + if value > result[pod][key] { + result[pod][key] = value + } + } + } + + return result, nil +} + +func subtractGantryDiagnosticSnapshots( + before, after gantryDiagnosticSnapshot, + timestamps map[string]map[string]float64, +) (gantryDiagnosticPhaseMeasurement, error) { + if len(before.PodNodes) != len(after.PodNodes) { + return gantryDiagnosticPhaseMeasurement{}, fmt.Errorf( + "gantry diagnostic pod set changed during phase: before=%d after=%d", + len(before.PodNodes), + len(after.PodNodes), + ) + } + + pods := make([]gantryPodDiagnosticMeasurement, 0, len(before.PodNodes)) + for _, pod := range sortedMapKeys(before.PodNodes) { + node := before.PodNodes[pod] + if after.PodNodes[pod] != node { + return gantryDiagnosticPhaseMeasurement{}, fmt.Errorf("gantry pod %s disappeared or moved from node %s", pod, node) + } + + deltas := map[string]float64{} + for key, afterValue := range after.Counters[pod] { + beforeValue := before.Counters[pod][key] + if afterValue < beforeValue { + return gantryDiagnosticPhaseMeasurement{}, fmt.Errorf( + "gantry diagnostic counter decreased for pod %s metric %s: before=%v after=%v", + pod, + key, + beforeValue, + afterValue, + ) + } + if delta := afterValue - beforeValue; delta != 0 { + deltas[key] = delta + } + } + + pods = append(pods, gantryPodDiagnosticMeasurement{ + PodName: pod, + NodeName: node, + CounterDeltas: deltas, + TimestampSeconds: timestamps[pod], + FinalLayerResponseCompletedTimestampSeconds: finalLayerResponseCompletedTimestamp(timestamps[pod]), + }) + } + + return gantryDiagnosticPhaseMeasurement{ + Pods: pods, + Source: "per-pod Prometheus counter deltas and timestamp gauges", + Complete: len(pods) > 0, + }, nil +} + +func finalLayerResponseCompletedTimestamp(timestamps map[string]float64) float64 { + var latest float64 + for key, value := range timestamps { + if strings.HasPrefix(key, "gantry_mirror_response_completed_timestamp_seconds{kind=layer,") && value > latest { + latest = value + } + } + + return latest +} + +func requireFinalLayerResponseTimestamps( + timestamps map[string]map[string]float64, + podNodes map[string]string, +) error { + missing := make([]string, 0) + for pod := range podNodes { + if finalLayerResponseCompletedTimestamp(timestamps[pod]) == 0 { + missing = append(missing, pod) + } + } + if len(missing) == 0 { + return nil + } + + sort.Strings(missing) + + return fmt.Errorf( + "final layer response completion timestamp missing for %d/%d Gantry pods: %s", + len(missing), + len(podNodes), + strings.Join(missing, ","), + ) +} + func (b *benchmark) fetchGantryPeerByteSnapshot(ctx context.Context, revision string) (peerByteSnapshot, error) { podNodes, err := b.gantryPodNodes(ctx, revision) if err != nil { diff --git a/hack/cmd/gantry-benchmark/peer_telemetry_test.go b/hack/cmd/gantry-benchmark/peer_telemetry_test.go index e00f342a6..c361a9a2d 100644 --- a/hack/cmd/gantry-benchmark/peer_telemetry_test.go +++ b/hack/cmd/gantry-benchmark/peer_telemetry_test.go @@ -3,7 +3,85 @@ package main -import "testing" +import ( + "context" + "fmt" + "strings" + "testing" + "time" +) + +type diagnosticTimestampRunner struct { + queryPath string +} + +func (r *diagnosticTimestampRunner) Run(_ context.Context, _ []byte, _ string, args ...string) ([]byte, error) { + r.queryPath = args[2] + + return []byte(fmt.Sprintf(`{"status":"success","data":{"resultType":"matrix","result":[ + {"metric":{"__name__":"gantry_mirror_response_completed_timestamp_seconds","pod":"gantry-a","kind":"layer","source":"peer"},"values":[[0,"%d"],[0,"%d"]]}, + {"metric":{"__name__":"gantry_containerd_commit_observed_timestamp_seconds","pod":"gantry-a"},"values":[[0,"%d"]]}, + {"metric":{"__name__":"gantry_containerd_commit_observed_timestamp_seconds","pod":"gantry-b"},"values":[[0,"%d"]]} + ]}}`, + time.Date(2026, time.August, 4, 1, 2, 30, 0, time.UTC).Unix(), + time.Date(2026, time.August, 4, 1, 3, 0, 0, time.UTC).Unix(), + time.Date(2026, time.August, 4, 1, 4, 0, 0, time.UTC).Unix(), + time.Date(2026, time.August, 4, 1, 5, 1, 0, time.UTC).Unix(), + )), nil +} + +func TestFetchGantryDiagnosticTimestampsUsesExactJobWindow(t *testing.T) { + runner := &diagnosticTimestampRunner{} + benchmark := &benchmark{ + config: benchmarkConfig{ + GantryNamespace: "gantry-system", + MonitoringNamespace: "monitoring", + PrometheusService: "prometheus", + NodeCount: 2, + }, + commands: runner, + } + window := telemetryWindow{ + StartedAt: time.Date(2026, time.August, 4, 1, 2, 0, 0, time.UTC), + FinishedAt: time.Date(2026, time.August, 4, 1, 5, 0, 0, time.UTC), + } + + timestamps, err := benchmark.fetchGantryDiagnosticTimestamps(context.Background(), "revision-a", window) + if err != nil { + t.Fatalf("fetchGantryDiagnosticTimestamps: %v", err) + } + if len(timestamps["gantry-a"]) != 2 || timestamps["gantry-b"] != nil { + t.Fatalf("timestamps = %v, want only two in-window gantry-a values", timestamps) + } + if timestamps["gantry-a"]["gantry_mirror_response_completed_timestamp_seconds{kind=layer,source=peer}"] != + float64(time.Date(2026, time.August, 4, 1, 3, 0, 0, time.UTC).Unix()) { + t.Fatalf("timestamps = %v, want latest in-window layer completion", timestamps) + } + if !strings.Contains(runner.queryPath, `kind%3D%22layer%22`) { + t.Fatalf("query path %q does not restrict completion timestamps to layers", runner.queryPath) + } +} + +func TestRequireFinalLayerResponseTimestamps(t *testing.T) { + timestamps := map[string]map[string]float64{ + "gantry-a": { + "gantry_mirror_response_completed_timestamp_seconds{kind=layer,source=peer}": 1234, + }, + } + podNodes := map[string]string{"gantry-a": "node-a", "gantry-b": "node-b"} + + err := requireFinalLayerResponseTimestamps(timestamps, podNodes) + if err == nil || !strings.Contains(err.Error(), "gantry-b") { + t.Fatalf("error = %v, want missing gantry-b completion", err) + } + + timestamps["gantry-b"] = map[string]float64{ + "gantry_mirror_response_completed_timestamp_seconds{kind=layer,source=origin}": 1235, + } + if err := requireFinalLayerResponseTimestamps(timestamps, podNodes); err != nil { + t.Fatalf("requireFinalLayerResponseTimestamps: %v", err) + } +} func TestSubtractPeerByteSnapshots(t *testing.T) { before := peerByteSnapshot{ @@ -84,3 +162,66 @@ func TestSubtractPeerByteSnapshotsDefaultsMissingKindsToZero(t *testing.T) { t.Fatalf("measurement = %+v, want missing kinds treated as zero", measurement) } } + +func TestDiagnosticMetricKey(t *testing.T) { + key, err := diagnosticMetricKey(map[string]string{ + "__name__": "p2p_peer_fetch_total", + "outcome": "busy", + }) + if err != nil { + t.Fatalf("diagnosticMetricKey: %v", err) + } + if key != "p2p_peer_fetch_total{outcome=busy}" { + t.Fatalf("key = %q, want p2p_peer_fetch_total{outcome=busy}", key) + } +} + +func TestSubtractGantryDiagnosticSnapshots(t *testing.T) { + before := gantryDiagnosticSnapshot{ + PodNodes: map[string]string{"gantry-a": "node-a"}, + Counters: map[string]map[string]float64{ + "gantry-a": {"p2p_peer_fetch_total{outcome=busy}": 10}, + }, + } + after := gantryDiagnosticSnapshot{ + PodNodes: map[string]string{"gantry-a": "node-a"}, + Counters: map[string]map[string]float64{ + "gantry-a": { + "p2p_peer_fetch_total{outcome=busy}": 13, + "p2p_dht_lookup_duration_seconds_sum{outcome=hit}": 1.25, + }, + }, + } + timestamps := map[string]map[string]float64{ + "gantry-a": {"gantry_mirror_response_completed_timestamp_seconds{kind=layer,source=peer}": 1234}, + } + + measurement, err := subtractGantryDiagnosticSnapshots(before, after, timestamps) + if err != nil { + t.Fatalf("subtractGantryDiagnosticSnapshots: %v", err) + } + if !measurement.Complete || len(measurement.Pods) != 1 { + t.Fatalf("measurement = %+v, want one complete pod", measurement) + } + pod := measurement.Pods[0] + if pod.NodeName != "node-a" || pod.CounterDeltas["p2p_peer_fetch_total{outcome=busy}"] != 3 || + pod.TimestampSeconds["gantry_mirror_response_completed_timestamp_seconds{kind=layer,source=peer}"] != 1234 || + pod.FinalLayerResponseCompletedTimestampSeconds != 1234 { + t.Fatalf("pod = %+v, want correlated deltas and timestamp", pod) + } +} + +func TestSubtractGantryDiagnosticSnapshotsRejectsReset(t *testing.T) { + before := gantryDiagnosticSnapshot{ + PodNodes: map[string]string{"gantry-a": "node-a"}, + Counters: map[string]map[string]float64{"gantry-a": {"counter": 2}}, + } + after := gantryDiagnosticSnapshot{ + PodNodes: map[string]string{"gantry-a": "node-a"}, + Counters: map[string]map[string]float64{"gantry-a": {"counter": 1}}, + } + + if _, err := subtractGantryDiagnosticSnapshots(before, after, nil); err == nil { + t.Fatal("expected a counter reset to invalidate diagnostic deltas") + } +} diff --git a/hack/cmd/gantry-benchmark/performance_telemetry.go b/hack/cmd/gantry-benchmark/performance_telemetry.go new file mode 100644 index 000000000..a355527c0 --- /dev/null +++ b/hack/cmd/gantry-benchmark/performance_telemetry.go @@ -0,0 +1,356 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package main + +import ( + "context" + "encoding/json" + "fmt" + "net/url" + "regexp" + "strings" + "time" +) + +const performanceTelemetryStep = 10 * time.Second + +type prometheusRangeCapture struct { + Name string `json:"name"` + Query string `json:"query"` + StepSeconds int `json:"step_seconds"` + Response json.RawMessage `json:"response"` +} + +type containerdJournalEvent struct { + ObserverPod string `json:"observer_pod"` + NodeName string `json:"node_name"` + Timestamp time.Time `json:"timestamp"` + Type string `json:"type"` + DurationSeconds float64 `json:"duration_seconds,omitempty"` + LayerDigest string `json:"layer_digest,omitempty"` + Message string `json:"message"` +} + +type phasePerformanceTelemetry struct { + Window telemetryWindow `json:"window"` + ObserverPodNodes map[string]string `json:"observer_pod_nodes"` + Prometheus []prometheusRangeCapture `json:"prometheus"` + ContainerdJournal string `json:"containerd_journal"` + ContainerdJournalEvents []containerdJournalEvent `json:"containerd_journal_events"` + Complete bool `json:"complete"` +} + +var journalFieldPattern = regexp.MustCompile(`(?:^|[[:space:]])([a-zA-Z_]+)="?([^"[:space:]]+)"?`) + +func (b *benchmark) capturePhasePerformanceTelemetry( + ctx context.Context, + phase proxyPhase, + job jobObservation, +) (phasePerformanceTelemetry, error) { + window := telemetryWindow{ + StartedAt: job.PhaseStartedAt, + FinishedAt: job.PhaseFinishedAt, + } + + observerPods, err := b.observerPodNodes(ctx) + if err != nil { + return phasePerformanceTelemetry{}, err + } + + queries := []struct { + name string + query string + }{ + {name: "node_disk_read_bytes_per_second", query: `rate(node_disk_read_bytes_total{gantry_benchmark="true"}[30s])`}, + {name: "node_disk_written_bytes_per_second", query: `rate(node_disk_written_bytes_total{gantry_benchmark="true"}[30s])`}, + {name: "node_disk_busy_ratio", query: `rate(node_disk_io_time_seconds_total{gantry_benchmark="true"}[30s])`}, + {name: "node_disk_weighted_io_seconds_per_second", query: `rate(node_disk_io_time_weighted_seconds_total{gantry_benchmark="true"}[30s])`}, + {name: "node_network_receive_bytes_per_second", query: `rate(node_network_receive_bytes_total{gantry_benchmark="true",device!="lo"}[30s])`}, + {name: "node_network_transmit_bytes_per_second", query: `rate(node_network_transmit_bytes_total{gantry_benchmark="true",device!="lo"}[30s])`}, + {name: "node_network_receive_utilization_ratio", query: `rate(node_network_receive_bytes_total{gantry_benchmark="true",device!="lo"}[30s]) / node_network_speed_bytes{gantry_benchmark="true",device!="lo"}`}, + {name: "node_network_transmit_utilization_ratio", query: `rate(node_network_transmit_bytes_total{gantry_benchmark="true",device!="lo"}[30s]) / node_network_speed_bytes{gantry_benchmark="true",device!="lo"}`}, + {name: "node_network_receive_drops_per_second", query: `rate(node_network_receive_drop_total{gantry_benchmark="true",device!="lo"}[30s])`}, + {name: "node_network_transmit_drops_per_second", query: `rate(node_network_transmit_drop_total{gantry_benchmark="true",device!="lo"}[30s])`}, + {name: "node_network_receive_errors_per_second", query: `rate(node_network_receive_errs_total{gantry_benchmark="true",device!="lo"}[30s])`}, + {name: "node_network_transmit_errors_per_second", query: `rate(node_network_transmit_errs_total{gantry_benchmark="true",device!="lo"}[30s])`}, + {name: "node_cpu_busy_ratio", query: `1 - avg by(pod) (rate(node_cpu_seconds_total{gantry_benchmark="true",mode="idle"}[30s]))`}, + {name: "node_memory_available_bytes", query: `node_memory_MemAvailable_bytes{gantry_benchmark="true"}`}, + {name: "containerd_process", query: `{__name__=~"process_(cpu_seconds_total|resident_memory_bytes|virtual_memory_bytes)",gantry_benchmark="true",endpoint="containerd-metrics"}`}, + {name: "containerd_metrics", query: `{__name__=~"containerd_.*|grpc_server_.*",gantry_benchmark="true"}`}, + {name: "gantry_peer_outcomes", query: `p2p_peer_fetch_total{gantry_benchmark="true"}`}, + {name: "gantry_peer_busy_stall_timestamps", query: `gantry_peer_fetch_last_timestamp_seconds{outcome=~"busy|stall",gantry_benchmark="true"}`}, + {name: "gantry_peer_duration", query: `{__name__=~"p2p_peer_fetch_duration_seconds_(bucket|sum|count)",outcome=~"busy|stall",gantry_benchmark="true"}`}, + {name: "gantry_dht_outcomes", query: `p2p_dht_lookup_total{gantry_benchmark="true"}`}, + {name: "gantry_dht_duration", query: `{__name__=~"p2p_dht_lookup_duration_seconds_(bucket|sum|count)",gantry_benchmark="true"}`}, + {name: "gantry_mirror_bytes", query: `gantry_mirror_bytes_served_total{gantry_benchmark="true"}`}, + {name: "gantry_response_completed", query: `gantry_mirror_response_completed_timestamp_seconds{kind="layer",gantry_benchmark="true"}`}, + {name: "gantry_commit_observation", query: `{__name__=~"gantry_containerd_commit_(observed_total|observed_timestamp_seconds|observation_duration_seconds_(sum|count)|latest_observation_duration_seconds|missing_after_stream_total)",gantry_benchmark="true"}`}, + } + + captures := make([]prometheusRangeCapture, 0, len(queries)) + for _, item := range queries { + response, err := b.queryPrometheusRange(ctx, item.query, window, performanceTelemetryStep) + if err != nil { + return phasePerformanceTelemetry{}, fmt.Errorf("capture %s: %w", item.name, err) + } + if err := validatePrometheusRangePodCoverage(item.name, response, b.config.NodeCount); err != nil { + return phasePerformanceTelemetry{}, err + } + captures = append(captures, prometheusRangeCapture{ + Name: item.name, + Query: item.query, + StepSeconds: int(performanceTelemetryStep.Seconds()), + Response: response, + }) + } + + journal, err := b.collectContainerdJournal(ctx, window) + if err != nil { + return phasePerformanceTelemetry{}, err + } + journalEvents, err := parseContainerdJournal(journal, observerPods, window) + if err != nil { + return phasePerformanceTelemetry{}, err + } + + return phasePerformanceTelemetry{ + Window: window, + ObserverPodNodes: observerPods, + Prometheus: captures, + ContainerdJournal: journal, + ContainerdJournalEvents: journalEvents, + Complete: true, + }, nil +} + +func validatePrometheusRangePodCoverage(name string, raw json.RawMessage, expectedPods int) error { + var response struct { + Data struct { + Result []struct { + Metric map[string]string `json:"metric"` + Values [][2]any `json:"values"` + } `json:"result"` + } `json:"data"` + } + if err := json.Unmarshal(raw, &response); err != nil { + return fmt.Errorf("decode %s range response for pod coverage: %w", name, err) + } + + pods := map[string]struct{}{} + for _, series := range response.Data.Result { + pod := series.Metric["pod"] + if pod == "" { + return fmt.Errorf("%s range series has no pod label", name) + } + if len(series.Values) > 0 { + pods[pod] = struct{}{} + } + } + if len(pods) != expectedPods { + return fmt.Errorf("%s range capture has samples from %d/%d pods", name, len(pods), expectedPods) + } + + return nil +} + +func parseContainerdJournal( + raw string, + observerPodNodes map[string]string, + window telemetryWindow, +) ([]containerdJournalEvent, error) { + events := []containerdJournalEvent{} + observedPods := map[string]struct{}{} + + for _, line := range strings.Split(raw, "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + + prefixEnd := strings.Index(line, "] ") + if !strings.HasPrefix(line, "[pod/") || prefixEnd < 0 { + return nil, fmt.Errorf("parse containerd journal prefix: %q", line) + } + prefixParts := strings.Split(strings.TrimSuffix(strings.TrimPrefix(line[:prefixEnd], "["), "]"), "/") + if len(prefixParts) != 3 || prefixParts[0] != "pod" || prefixParts[2] != "containerd-journal" { + return nil, fmt.Errorf("parse containerd journal source: %q", line[:prefixEnd+1]) + } + + observerPod := prefixParts[1] + nodeName, ok := observerPodNodes[observerPod] + if !ok { + return nil, fmt.Errorf("containerd journal belongs to unexpected observer pod %q", observerPod) + } + + remainder := line[prefixEnd+2:] + timestampEnd := strings.IndexByte(remainder, ' ') + if timestampEnd < 0 { + return nil, fmt.Errorf("parse containerd journal timestamp: %q", line) + } + timestamp, err := time.Parse(time.RFC3339Nano, remainder[:timestampEnd]) + if err != nil { + return nil, fmt.Errorf("parse containerd journal timestamp %q: %w", remainder[:timestampEnd], err) + } + if timestamp.Before(window.StartedAt) || timestamp.After(window.FinishedAt) { + continue + } + + message := remainder[timestampEnd+1:] + eventType := classifyContainerdJournalEvent(message) + if eventType == "" { + return nil, fmt.Errorf("classify filtered containerd journal message: %q", message) + } + + event := containerdJournalEvent{ + ObserverPod: observerPod, + NodeName: nodeName, + Timestamp: timestamp, + Type: eventType, + Message: message, + } + for _, match := range journalFieldPattern.FindAllStringSubmatch(message, -1) { + switch match[1] { + case "duration": + duration, err := time.ParseDuration(match[2]) + if err != nil { + return nil, fmt.Errorf("parse containerd journal duration %q: %w", match[2], err) + } + event.DurationSeconds = duration.Seconds() + case "layer": + event.LayerDigest = match[2] + } + } + + events = append(events, event) + observedPods[observerPod] = struct{}{} + } + + if len(observedPods) != len(observerPodNodes) { + return nil, fmt.Errorf( + "containerd journal has phase events from %d/%d observer pods", + len(observedPods), + len(observerPodNodes), + ) + } + + return events, nil +} + +func classifyContainerdJournalEvent(message string) string { + switch { + case strings.Contains(message, "layer unpacked"): + return "layer_unpacked" + case strings.Contains(message, "image unpacked"): + return "image_unpacked" + case strings.Contains(message, "cancel pulling image"): + return "pull_cancelled" + case strings.Contains(message, "Pulled image"): + return "pull_completed" + case strings.Contains(message, "PullImage "): + return "pull_started" + default: + return "" + } +} + +func (b *benchmark) queryPrometheusRange( + ctx context.Context, + query string, + window telemetryWindow, + step time.Duration, +) (json.RawMessage, error) { + rawPath := fmt.Sprintf( + "/api/v1/namespaces/%s/services/http:%s:9090/proxy/api/v1/query_range?query=%s&start=%s&end=%s&step=%d", + b.config.MonitoringNamespace, + b.config.PrometheusService, + url.QueryEscape(query), + url.QueryEscape(window.StartedAt.UTC().Format(time.RFC3339Nano)), + url.QueryEscape(window.FinishedAt.UTC().Format(time.RFC3339Nano)), + int(step.Seconds()), + ) + + output, err := b.commands.Run(ctx, nil, "kubectl", "get", "--raw", rawPath) + if err != nil { + return nil, err + } + + var envelope struct { + Status string `json:"status"` + } + if err := json.Unmarshal(output, &envelope); err != nil { + return nil, fmt.Errorf("decode Prometheus range response: %w", err) + } + if envelope.Status != "success" { + return nil, fmt.Errorf("prometheus range query status is %q", envelope.Status) + } + + return json.RawMessage(output), nil +} + +func (b *benchmark) observerPodNodes(ctx context.Context) (map[string]string, error) { + output, err := b.commands.Run( + ctx, + nil, + "kubectl", "-n", b.config.Namespace, + "get", "pods", "-l", "app.kubernetes.io/name=gantry-benchmark-node-observer", + "-o", "json", + ) + if err != nil { + return nil, err + } + + var pods struct { + Items []struct { + Metadata struct { + Name string `json:"name"` + } `json:"metadata"` + Spec struct { + NodeName string `json:"nodeName"` + } `json:"spec"` + } `json:"items"` + } + if err := json.Unmarshal(output, &pods); err != nil { + return nil, fmt.Errorf("decode observer pods: %w", err) + } + + result := make(map[string]string, len(pods.Items)) + for _, pod := range pods.Items { + if pod.Metadata.Name == "" || pod.Spec.NodeName == "" { + return nil, fmt.Errorf("observer pod has empty name or nodeName") + } + result[pod.Metadata.Name] = pod.Spec.NodeName + } + if len(result) != b.config.NodeCount { + return nil, fmt.Errorf("observer pod/node map has %d pods, want %d", len(result), b.config.NodeCount) + } + + return result, nil +} + +func (b *benchmark) collectContainerdJournal(ctx context.Context, window telemetryWindow) (string, error) { + output, err := b.commands.Run( + ctx, + nil, + "kubectl", "-n", b.config.Namespace, + "logs", "-l", "app.kubernetes.io/name=gantry-benchmark-node-observer", + "-c", "containerd-journal", + "--prefix=true", + "--timestamps=true", + "--max-log-requests", fmt.Sprintf("%d", b.config.NodeCount), + "--since-time", window.StartedAt.UTC().Format(time.RFC3339Nano), + ) + if err != nil { + return "", fmt.Errorf("collect containerd journal: %w", err) + } + + return string(output), nil +} + +func (b *benchmark) writePerformanceTelemetryArtifact( + runID string, + phase proxyPhase, + measurement phasePerformanceTelemetry, +) error { + return b.writeJSONArtifact(runID, string(phase)+"-performance.json", measurement) +} diff --git a/hack/cmd/gantry-benchmark/performance_telemetry_test.go b/hack/cmd/gantry-benchmark/performance_telemetry_test.go new file mode 100644 index 000000000..b7ac1e230 --- /dev/null +++ b/hack/cmd/gantry-benchmark/performance_telemetry_test.go @@ -0,0 +1,122 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package main + +import ( + "context" + "encoding/json" + "strings" + "testing" + "time" +) + +type performanceTelemetryRunner struct { + commands [][]string +} + +func (r *performanceTelemetryRunner) Run(_ context.Context, _ []byte, name string, args ...string) ([]byte, error) { + command := append([]string{name}, args...) + r.commands = append(r.commands, command) + + return []byte(`{"status":"success","data":{"resultType":"matrix","result":[]}}`), nil +} + +func TestQueryPrometheusRange(t *testing.T) { + runner := &performanceTelemetryRunner{} + benchmark := &benchmark{ + config: benchmarkConfig{ + MonitoringNamespace: "monitoring", + PrometheusService: "prometheus", + }, + commands: runner, + } + window := telemetryWindow{ + StartedAt: time.Date(2026, time.August, 4, 1, 2, 3, 0, time.UTC), + FinishedAt: time.Date(2026, time.August, 4, 1, 12, 3, 0, time.UTC), + } + + response, err := benchmark.queryPrometheusRange( + context.Background(), + `rate(node_disk_written_bytes_total[30s])`, + window, + 10*time.Second, + ) + if err != nil { + t.Fatalf("queryPrometheusRange: %v", err) + } + if !strings.Contains(string(response), `"status":"success"`) { + t.Fatalf("response = %s, want successful raw envelope", response) + } + if len(runner.commands) != 1 { + t.Fatalf("commands = %v, want one command", runner.commands) + } + + path := runner.commands[0][3] + for _, want := range []string{"/query_range?", "step=10", "node_disk_written_bytes_total", "2026-08-04T01%3A02%3A03Z"} { + if !strings.Contains(path, want) { + t.Fatalf("query path %q is missing %q", path, want) + } + } +} + +func TestParseContainerdJournal(t *testing.T) { + window := telemetryWindow{ + StartedAt: time.Date(2026, time.August, 4, 1, 2, 3, 0, time.UTC), + FinishedAt: time.Date(2026, time.August, 4, 1, 12, 3, 0, time.UTC), + } + raw := strings.Join([]string{ + `[pod/observer-a/containerd-journal] 2026-08-04T01:02:02Z level=debug msg="layer unpacked" duration=1s layer=sha256:before`, + `[pod/observer-a/containerd-journal] 2026-08-04T01:03:04.123456789Z level=debug msg="layer unpacked" duration=2.5s layer=sha256:abc`, + `[pod/observer-b/containerd-journal] 2026-08-04T01:04:05Z level=info msg="Pulled image" image="example/image@sha256:def"`, + `[pod/observer-a/containerd-journal] 2026-08-04T01:12:04Z level=debug msg="image unpacked" duration=3s`, + }, "\n") + + events, err := parseContainerdJournal(raw, map[string]string{ + "observer-a": "node-a", + "observer-b": "node-b", + }, window) + if err != nil { + t.Fatalf("parseContainerdJournal: %v", err) + } + if len(events) != 2 { + t.Fatalf("events = %v, want two phase-bounded events", events) + } + if events[0].NodeName != "node-a" || events[0].Type != "layer_unpacked" || + events[0].LayerDigest != "sha256:abc" || events[0].DurationSeconds != 2.5 { + t.Fatalf("first event = %+v, want parsed layer event", events[0]) + } + if events[1].NodeName != "node-b" || events[1].Type != "pull_completed" { + t.Fatalf("second event = %+v, want correlated pull completion", events[1]) + } +} + +func TestParseContainerdJournalRequiresEveryObserver(t *testing.T) { + window := telemetryWindow{ + StartedAt: time.Date(2026, time.August, 4, 1, 2, 3, 0, time.UTC), + FinishedAt: time.Date(2026, time.August, 4, 1, 12, 3, 0, time.UTC), + } + raw := `[pod/observer-a/containerd-journal] 2026-08-04T01:03:04Z level=debug msg="image unpacked" duration=2s` + + _, err := parseContainerdJournal(raw, map[string]string{ + "observer-a": "node-a", + "observer-b": "node-b", + }, window) + if err == nil || !strings.Contains(err.Error(), "1/2 observer pods") { + t.Fatalf("error = %v, want incomplete observer coverage", err) + } +} + +func TestValidatePrometheusRangePodCoverage(t *testing.T) { + raw := json.RawMessage(`{"data":{"result":[ + {"metric":{"pod":"observer-a"},"values":[[1,"2"]]}, + {"metric":{"pod":"observer-b"},"values":[[1,"3"]]} + ]}}`) + + if err := validatePrometheusRangePodCoverage("disk", raw, 2); err != nil { + t.Fatalf("validatePrometheusRangePodCoverage: %v", err) + } + if err := validatePrometheusRangePodCoverage("disk", raw, 3); err == nil || !strings.Contains(err.Error(), "2/3 pods") { + t.Fatalf("error = %v, want partial pod coverage", err) + } +} diff --git a/hack/cmd/gantry-benchmark/preflight.go b/hack/cmd/gantry-benchmark/preflight.go index e87f7ff16..eb995ae16 100644 --- a/hack/cmd/gantry-benchmark/preflight.go +++ b/hack/cmd/gantry-benchmark/preflight.go @@ -368,6 +368,95 @@ func (b *benchmark) checkMonitoring(ctx context.Context, state benchmarkState) e } } + monitoringChecks := []struct { + description string + query string + }{ + { + description: "node-exporter identity", + query: fmt.Sprintf(`count(node_uname_info{namespace=%q,gantry_benchmark="true"})`, b.config.Namespace), + }, + { + description: "host disk throughput", + query: fmt.Sprintf(`count(count by(pod) (node_disk_written_bytes_total{namespace=%q,gantry_benchmark="true"}))`, b.config.Namespace), + }, + { + description: "host network throughput", + query: fmt.Sprintf(`count(count by(pod) (node_network_receive_bytes_total{namespace=%q,gantry_benchmark="true",device!="lo"}))`, b.config.Namespace), + }, + { + description: "host network link speed", + query: fmt.Sprintf(`count(count by(pod) (node_network_speed_bytes{namespace=%q,gantry_benchmark="true",device!="lo"} > 0))`, b.config.Namespace), + }, + { + description: "containerd build", + query: fmt.Sprintf(`count(containerd_build_info{namespace=%q,gantry_benchmark="true"})`, b.config.Namespace), + }, + { + description: "Gantry response completion", + query: fmt.Sprintf( + `count(count by(pod) (gantry_mirror_response_completed_timestamp_seconds{namespace=%q,kind="layer",gantry_benchmark="true",controller_revision_hash=%q}))`, + b.config.GantryNamespace, + revision, + ), + }, + { + description: "Gantry peer busy and stall outcomes", + query: fmt.Sprintf( + `count(count by(pod) (p2p_peer_fetch_total{namespace=%q,outcome=~"busy|stall",gantry_benchmark="true",controller_revision_hash=%q}))`, + b.config.GantryNamespace, + revision, + ), + }, + { + description: "Gantry peer busy and stall timestamps", + query: fmt.Sprintf( + `count(count by(pod) (gantry_peer_fetch_last_timestamp_seconds{namespace=%q,outcome=~"busy|stall",gantry_benchmark="true",controller_revision_hash=%q}))`, + b.config.GantryNamespace, + revision, + ), + }, + { + description: "Gantry DHT lookup durations", + query: fmt.Sprintf( + `count(count by(pod) (p2p_dht_lookup_duration_seconds_count{namespace=%q,gantry_benchmark="true",controller_revision_hash=%q}))`, + b.config.GantryNamespace, + revision, + ), + }, + { + description: "Gantry containerd commit observation", + query: fmt.Sprintf( + `count(gantry_containerd_commit_observation_duration_seconds_count{namespace=%q,gantry_benchmark="true",controller_revision_hash=%q})`, + b.config.GantryNamespace, + revision, + ), + }, + { + description: "Gantry latest containerd commit observation duration", + query: fmt.Sprintf( + `count(gantry_containerd_commit_latest_observation_duration_seconds{namespace=%q,gantry_benchmark="true",controller_revision_hash=%q})`, + b.config.GantryNamespace, + revision, + ), + }, + } + + for _, check := range monitoringChecks { + count, err := b.queryPrometheus(ctx, check.query) + if err != nil { + return fmt.Errorf("query %s metric count: %w", check.description, err) + } + if int(count) != b.config.NodeCount { + return fmt.Errorf( + "prometheus reports %s metrics for %.0f/%d observer pods", + check.description, + count, + b.config.NodeCount, + ) + } + } + // Direct mode has no proxy, so there are no proxy samples to wait for. The // Gantry scrape checks above already prove the benchmark PodMonitor is // being honoured by Prometheus. diff --git a/hack/cmd/gantry-benchmark/results.go b/hack/cmd/gantry-benchmark/results.go index 466962be3..26771ff4f 100644 --- a/hack/cmd/gantry-benchmark/results.go +++ b/hack/cmd/gantry-benchmark/results.go @@ -72,24 +72,26 @@ const ( ) type phaseResult struct { - RunID string `json:"run_id"` - Phase proxyPhase `json:"phase"` - Image string `json:"image"` - ImageSizeMiB int `json:"image_size_mib"` - ImageLayers int `json:"image_layers,omitempty"` - PayloadSHA string `json:"workload_payload_sha256,omitempty"` - WorkloadComparisonMode workloadComparisonMode `json:"workload_comparison_mode,omitempty"` - Proxy proxyPhaseTotals `json:"proxy"` - Gantry gantryMetrics `json:"gantry"` - GantryPeer gantryPeerPhaseMeasurement `json:"gantry_peer"` - Azure azurePhaseMeasurement `json:"azure"` - Job jobObservation `json:"job"` + RunID string `json:"run_id"` + Phase proxyPhase `json:"phase"` + Image string `json:"image"` + ImageSizeMiB int `json:"image_size_mib"` + ImageLayers int `json:"image_layers,omitempty"` + PayloadSHA string `json:"workload_payload_sha256,omitempty"` + WorkloadComparisonMode workloadComparisonMode `json:"workload_comparison_mode,omitempty"` + Proxy proxyPhaseTotals `json:"proxy"` + Gantry gantryMetrics `json:"gantry"` + GantryPeer gantryPeerPhaseMeasurement `json:"gantry_peer"` + GantryDiagnostics gantryDiagnosticPhaseMeasurement `json:"gantry_diagnostics"` + Azure azurePhaseMeasurement `json:"azure"` + Job jobObservation `json:"job"` // OriginBytes is the phase's ACR traffic as attributed by OriginBytesSource. - OriginBytes uint64 `json:"origin_bytes"` - OriginBytesSource originByteSource `json:"origin_bytes_source"` - PodStartupLatency latencySummary `json:"pod_startup_latency"` - PodStartupLatencySource string `json:"pod_startup_latency_source"` - RecordedAt time.Time `json:"recorded_at"` + OriginBytes uint64 `json:"origin_bytes"` + OriginBytesSource originByteSource `json:"origin_bytes_source"` + PodStartupLatency latencySummary `json:"pod_startup_latency"` + PodStartupLatencySource string `json:"pod_startup_latency_source"` + PerformanceTelemetryArtifact string `json:"performance_telemetry_artifact"` + RecordedAt time.Time `json:"recorded_at"` } type benchmarkComparison struct { diff --git a/hack/cmd/gantry-benchmark/run.go b/hack/cmd/gantry-benchmark/run.go index a6a195fe2..1642085e3 100644 --- a/hack/cmd/gantry-benchmark/run.go +++ b/hack/cmd/gantry-benchmark/run.go @@ -130,6 +130,11 @@ func (b *benchmark) runBenchmark(ctx context.Context) (returnErr error) { return err } + baselineDiagnosticsBefore, err := b.fetchGantryDiagnosticSnapshot(ctx, revision) + if err != nil { + return err + } + writeAll(b.stdout, fmt.Sprintf("running baseline pull on %d nodes\n", b.config.NodeCount)) baselineJob, err := b.runPullJob(ctx, state, proxyPhaseBaseline, baselineImage) @@ -170,7 +175,34 @@ func (b *benchmark) runBenchmark(ctx context.Context) (returnErr error) { return err } + baselineDiagnosticsAfter, err := b.fetchGantryDiagnosticSnapshot(ctx, revision) + if err != nil { + return err + } + baselineDiagnosticTimestamps, err := b.fetchGantryDiagnosticTimestamps(ctx, revision, telemetryWindow{ + StartedAt: baselineJob.PhaseStartedAt, + FinishedAt: baselineJob.PhaseFinishedAt, + }) + if err != nil { + return err + } + baselineDiagnostics, err := subtractGantryDiagnosticSnapshots( + baselineDiagnosticsBefore, + baselineDiagnosticsAfter, + baselineDiagnosticTimestamps, + ) + if err != nil { + return err + } + baselineBytes, baselineBytesSource := deriveOriginBytes(b.config, proxyPhaseBaseline, baselineProxy, baselineGantry, baselineJob) + baselinePerformance, err := b.capturePhasePerformanceTelemetry(ctx, proxyPhaseBaseline, baselineJob) + if err != nil { + return err + } + if err := b.writePerformanceTelemetryArtifact(state.RunID, proxyPhaseBaseline, baselinePerformance); err != nil { + return err + } baselineResult := phaseResult{ RunID: state.RunID, @@ -183,14 +215,16 @@ func (b *benchmark) runBenchmark(ctx context.Context) (returnErr error) { Proxy: baselineProxy, Gantry: baselineGantry, GantryPeer: baselinePeer, + GantryDiagnostics: baselineDiagnostics, Azure: azurePhaseMeasurement{Window: telemetryWindow{ StartedAt: baselineWindowStart, FinishedAt: baselineWindowFinish, }}, - Job: baselineJob, - OriginBytes: baselineBytes, - OriginBytesSource: baselineBytesSource, - RecordedAt: time.Now().UTC(), + Job: baselineJob, + OriginBytes: baselineBytes, + OriginBytesSource: baselineBytesSource, + PerformanceTelemetryArtifact: string(proxyPhaseBaseline) + "-performance.json", + RecordedAt: time.Now().UTC(), } if err := b.writeJSONArtifact(state.RunID, "baseline.json", baselineResult); err != nil { return err @@ -227,6 +261,11 @@ func (b *benchmark) runBenchmark(ctx context.Context) (returnErr error) { return err } + gantryDiagnosticsBefore, err := b.fetchGantryDiagnosticSnapshot(ctx, revision) + if err != nil { + return err + } + writeAll(b.stdout, fmt.Sprintf("running Gantry cold pull on %d nodes\n", b.config.NodeCount)) gantryJob, err := b.runPullJob(ctx, state, proxyPhaseGantryCold, gantryImage) @@ -258,6 +297,29 @@ func (b *benchmark) runBenchmark(ctx context.Context) (returnErr error) { return err } + gantryDiagnosticsAfter, err := b.fetchGantryDiagnosticSnapshot(ctx, revision) + if err != nil { + return err + } + gantryDiagnosticTimestamps, err := b.fetchGantryDiagnosticTimestamps(ctx, revision, telemetryWindow{ + StartedAt: gantryJob.PhaseStartedAt, + FinishedAt: gantryJob.PhaseFinishedAt, + }) + if err != nil { + return err + } + if err := requireFinalLayerResponseTimestamps(gantryDiagnosticTimestamps, gantryDiagnosticsAfter.PodNodes); err != nil { + return err + } + gantryDiagnostics, err := subtractGantryDiagnosticSnapshots( + gantryDiagnosticsBefore, + gantryDiagnosticsAfter, + gantryDiagnosticTimestamps, + ) + if err != nil { + return err + } + var gantryProxy proxyPhaseTotals if state.usesProxy() { @@ -268,6 +330,13 @@ func (b *benchmark) runBenchmark(ctx context.Context) (returnErr error) { } gantryBytes, gantryBytesSource := deriveOriginBytes(b.config, proxyPhaseGantryCold, gantryProxy, phaseMetrics, gantryJob) + gantryPerformance, err := b.capturePhasePerformanceTelemetry(ctx, proxyPhaseGantryCold, gantryJob) + if err != nil { + return err + } + if err := b.writePerformanceTelemetryArtifact(state.RunID, proxyPhaseGantryCold, gantryPerformance); err != nil { + return err + } gantryResult := phaseResult{ RunID: state.RunID, @@ -280,14 +349,16 @@ func (b *benchmark) runBenchmark(ctx context.Context) (returnErr error) { Proxy: gantryProxy, Gantry: phaseMetrics, GantryPeer: gantryPeer, + GantryDiagnostics: gantryDiagnostics, Azure: azurePhaseMeasurement{Window: telemetryWindow{ StartedAt: gantryWindowStart, FinishedAt: gantryWindowFinish, }}, - Job: gantryJob, - OriginBytes: gantryBytes, - OriginBytesSource: gantryBytesSource, - RecordedAt: time.Now().UTC(), + Job: gantryJob, + OriginBytes: gantryBytes, + OriginBytesSource: gantryBytesSource, + PerformanceTelemetryArtifact: string(proxyPhaseGantryCold) + "-performance.json", + RecordedAt: time.Now().UTC(), } if err := b.writeJSONArtifact(state.RunID, "gantry-cold.json", gantryResult); err != nil { return err diff --git a/hack/gantry-benchmark/README.md b/hack/gantry-benchmark/README.md index 0e25dcaf4..23c3c8879 100644 --- a/hack/gantry-benchmark/README.md +++ b/hack/gantry-benchmark/README.md @@ -14,6 +14,10 @@ The workflow does not provision AKS, create ACR, or install Gantry. It expects: a Grafana dashboard sidecar. The workflow installs benchmark-owned PodMonitors for Gantry and, in proxy mode, the proxy. - Containerd configured to read `/etc/containerd/certs.d`. +- Containerd metrics listening on `0.0.0.0:10257` and debug logging enabled. + The managed node template in this repository configures both. Preflight + refuses to run without a containerd scrape from every target node, and the + node-observer DaemonSet fails if effective containerd log level is not debug. - A private operator VM in the AKS VNet. The VM runs every benchmark command, builds and pushes both images, queries Azure telemetry, and stores artifacts. - Cluster permission to create privileged hostPath DaemonSets. Proxy mode also @@ -87,6 +91,44 @@ hostnames alone do not isolate containerd's digest-addressed cache; the phase-specific layer paths provide that isolation while preserving identical payload bytes. +## Performance attribution artifacts + +`enable` installs a benchmark-owned node-observer DaemonSet on every target +node. It exposes node-exporter metrics, provides a Prometheus target for the +host containerd metrics endpoint, and streams a filtered subset of the host +containerd journal. Preflight requires both `node_uname_info` and +`containerd_build_info` from every observer pod. + +Each phase writes `-performance.json` with the unmodified Prometheus +range-query envelopes at 10-second resolution for: + +- host disk bytes, I/O busy time, CPU, memory, and network bytes/errors; +- containerd process and built-in metrics; +- per-Gantry-pod peer outcomes and durations; +- exact latest peer busy/stall event timestamps plus interval counts; +- per-Gantry-pod DHT outcomes and durations; +- mirror bytes and final response-completion timestamps; and +- stream-completion-to-containerd-inventory observation distributions and + latest measured durations. + +The same artifact includes the observer-pod-to-node map, raw filtered +containerd journal, and phase-bounded structured events for `PullImage`, +successful pull completion, no-progress cancellation, `layer unpacked`, and +`image unpacked`. Capture fails unless every observer pod has an event in the +phase window. Containerd's +`layer unpacked` duration spans fetch, apply, and snapshot commit; it is not a +pure filesystem-write duration. Host disk metrics provide the independent +filesystem pressure signal during that span. + +Phase JSON also includes: + +- exact per-workload-pod container start/finish timestamps and node names; and +- per-Gantry-pod counter deltas and phase-local timestamp gauges. + +These two node maps are the supported join key for correlating peer busy/stall +events, DHT latency, final layer-response time, containerd commit observation, +containerd unpack logs, host resource use, and workload startup latency. + ## Lifecycle All lifecycle commands run on the operator VM under its system-assigned managed diff --git a/hack/gantry-benchmark/manifests/monitoring.yaml.tmpl b/hack/gantry-benchmark/manifests/monitoring.yaml.tmpl index b1f121124..bfea61741 100644 --- a/hack/gantry-benchmark/manifests/monitoring.yaml.tmpl +++ b/hack/gantry-benchmark/manifests/monitoring.yaml.tmpl @@ -28,7 +28,171 @@ spec: - action: keep sourceLabels: - __name__ - regex: gantry_storage_mode_info|p2p_dht_health_score|gantry_peer_serve_bytes_total|gantry_origin_bytes_total|p2p_origin_pull_total|p2p_peer_fetch_total|p2p_origin_pull_success_total|p2p_origin_fallback_total|p2p_in_flight_pulls|gantry_peer_fetch_bytes_total|gantry_mirror_bytes_served_total + regex: gantry_storage_mode_info|p2p_dht_health_score|p2p_dht_lookup_total|p2p_dht_lookup_duration_seconds_(bucket|sum|count)|gantry_peer_serve_bytes_total|gantry_origin_bytes_total|p2p_origin_pull_total|p2p_peer_fetch_total|gantry_peer_fetch_last_timestamp_seconds|p2p_peer_fetch_duration_seconds_(bucket|sum|count)|p2p_origin_pull_success_total|p2p_origin_fallback_total|p2p_in_flight_pulls|gantry_peer_fetch_bytes_total|gantry_mirror_bytes_served_total|gantry_mirror_response_completed_timestamp_seconds|gantry_containerd_commit_observed_total|gantry_containerd_commit_observed_timestamp_seconds|gantry_containerd_commit_observation_duration_seconds_(bucket|sum|count)|gantry_containerd_commit_latest_observation_duration_seconds|gantry_containerd_commit_missing_after_stream_total + - action: replace + targetLabel: gantry_benchmark + replacement: "true" +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: gantry-benchmark-node-observer + namespace: {{ .Namespace }} + labels: + app.kubernetes.io/name: gantry-benchmark-node-observer + app.kubernetes.io/part-of: gantry-benchmark +spec: + selector: + matchLabels: + app.kubernetes.io/name: gantry-benchmark-node-observer + template: + metadata: + labels: + app.kubernetes.io/name: gantry-benchmark-node-observer + app.kubernetes.io/part-of: gantry-benchmark + spec: + hostNetwork: true + hostPID: true + dnsPolicy: ClusterFirstWithHostNet + nodeSelector: + kubernetes.io/os: {{ .NodeOS }} + kubernetes.io/arch: {{ .NodeArch }} + tolerations: + - operator: Exists + containers: + - name: node-exporter + image: quay.io/prometheus/node-exporter:v1.9.1 + args: + - --path.procfs=/host/proc + - --path.sysfs=/host/sys + - --path.rootfs=/host/root + - --web.listen-address=:19100 + - --collector.filesystem.mount-points-exclude=^/(dev|proc|sys|var/lib/kubelet/pods/.+)($|/) + ports: + - name: node-metrics + containerPort: 19100 + resources: + requests: + cpu: 10m + memory: 24Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + runAsNonRoot: true + runAsUser: 65534 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumeMounts: + - name: proc + mountPath: /host/proc + readOnly: true + - name: sys + mountPath: /host/sys + readOnly: true + - name: root + mountPath: /host/root + readOnly: true + - name: containerd-metrics-target + image: mcr.microsoft.com/cbl-mariner/busybox:2.0 + command: ["sh", "-c", "exec sleep 2147483647"] + ports: + - name: containerd-metrics + containerPort: 10257 + resources: + requests: + cpu: 1m + memory: 4Mi + limits: + cpu: 20m + memory: 16Mi + securityContext: + runAsNonRoot: true + runAsUser: 65532 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + - name: containerd-journal + image: mcr.microsoft.com/cbl-mariner/busybox:2.0 + command: + - chroot + - /host + - sh + - -c + - | + containerd_pid="$(systemctl show --property MainPID --value containerd)" + containerd_bin="$(readlink -f "/proc/${containerd_pid}/exe")" + if [ -z "${containerd_bin}" ] || ! "${containerd_bin}" config dump 2>/dev/null | grep -Eq "^[[:space:]]*level = ['\"]debug['\"]$"; then + echo "containerd debug logging is required for unpack timing capture" >&2 + exit 1 + fi + exec journalctl -f -n 0 -u containerd -o cat | \ + grep --line-buffered -E 'PullImage |Pulled image |cancel pulling image |layer unpacked|image unpacked' + resources: + requests: + cpu: 2m + memory: 8Mi + limits: + cpu: 50m + memory: 32Mi + securityContext: + privileged: true + runAsUser: 0 + volumeMounts: + - name: host + mountPath: /host + readOnly: true + volumes: + - name: proc + hostPath: + path: /proc + type: Directory + - name: sys + hostPath: + path: /sys + type: Directory + - name: root + hostPath: + path: / + type: Directory + - name: host + hostPath: + path: / + type: Directory +--- +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + name: gantry-benchmark-node-observer + namespace: {{ .Namespace }} + labels: + release: {{ .MonitoringLabel }} + app.kubernetes.io/part-of: gantry-benchmark +spec: + selector: + matchLabels: + app.kubernetes.io/name: gantry-benchmark-node-observer + podMetricsEndpoints: + - port: node-metrics + path: /metrics + interval: 10s + metricRelabelings: + - action: keep + sourceLabels: [__name__] + regex: node_uname_info|node_cpu_seconds_total|node_memory_(MemAvailable|MemTotal)_bytes|node_disk_(read|written)_bytes_total|node_disk_io_time_seconds_total|node_disk_io_time_weighted_seconds_total|node_disk_reads_completed_total|node_disk_writes_completed_total|node_filesystem_(avail|size)_bytes|node_network_speed_bytes|node_network_(receive|transmit)_(bytes|drop|errs)_total + - action: replace + targetLabel: gantry_benchmark + replacement: "true" + - port: containerd-metrics + path: /v1/metrics + interval: 10s + metricRelabelings: + - action: keep + sourceLabels: [__name__] + regex: containerd_.*|grpc_server_.*|process_(cpu_seconds_total|resident_memory_bytes|virtual_memory_bytes) - action: replace targetLabel: gantry_benchmark replacement: "true" diff --git a/internal/gantry/mirror/byte_metrics_test.go b/internal/gantry/mirror/byte_metrics_test.go index 4270da59c..272ae3c68 100644 --- a/internal/gantry/mirror/byte_metrics_test.go +++ b/internal/gantry/mirror/byte_metrics_test.go @@ -59,12 +59,15 @@ func TestMirrorByteMetricsCacheSource(t *testing.T) { cache := fakes.NewCache() cache.Put(d, body) - var served []byteObservation + var served, completed []byteObservation m := mirror.New(cfg, cache, oc, mirror.WithByteMetrics(func(kind, source string, bytes int64) { served = append(served, byteObservation{kind: kind, source: source, bytes: bytes}) }), + mirror.WithMirrorResponseCompletedHook(func(kind, source string) { + completed = append(completed, byteObservation{kind: kind, source: source}) + }), ) if got := pullMirrorBody(t, m.Handler(), d); string(got) != string(body) { @@ -75,6 +78,9 @@ func TestMirrorByteMetricsCacheSource(t *testing.T) { if len(served) != 1 || served[0] != want { t.Fatalf("served observations = %+v, want [%+v]", served, want) } + if len(completed) != 1 || completed[0].kind != want.kind || completed[0].source != want.source { + t.Fatalf("completed observations = %+v, want kind=%s source=%s", completed, want.kind, want.source) + } } func TestMirrorByteMetricsPeerSource(t *testing.T) { @@ -89,7 +95,7 @@ func TestMirrorByteMetricsPeerSource(t *testing.T) { dht := fakes.NewDHT() dht.Inject(d, ifaces.Provider{NodeID: "peer-a", Addr: peerAddr}) - var fetched, served []byteObservation + var fetched, served, completed []byteObservation peerClient := transfer.NewClient(transfer.WithClientByteMetrics(func(kind string, bytes int64) { fetched = append(fetched, byteObservation{kind: kind, bytes: bytes}) @@ -104,6 +110,9 @@ func TestMirrorByteMetricsPeerSource(t *testing.T) { served = append(served, byteObservation{kind: kind, source: source, bytes: bytes}) }, ), + mirror.WithMirrorResponseCompletedHook(func(kind, source string) { + completed = append(completed, byteObservation{kind: kind, source: source}) + }), ) if got := pullMirrorBody(t, m.Handler(), d); string(got) != string(body) { @@ -119,6 +128,9 @@ func TestMirrorByteMetricsPeerSource(t *testing.T) { if len(served) != 1 || served[0] != wantServed { t.Fatalf("served observations = %+v, want [%+v]", served, wantServed) } + if len(completed) != 1 || completed[0].kind != wantServed.kind || completed[0].source != wantServed.source { + t.Fatalf("completed observations = %+v, want kind=%s source=%s", completed, wantServed.kind, wantServed.source) + } } func TestMirrorByteMetricsOriginSource(t *testing.T) { @@ -150,13 +162,16 @@ func TestMirrorByteMetricsOriginSource(t *testing.T) { t.Fatalf("origin.New: %v", err) } - var served []byteObservation + var served, completed []byteObservation m := mirror.New(cfg, fakes.NewCache(), oc, mirror.WithLiveStreamThrough(), mirror.WithByteMetrics(func(kind, source string, bytes int64) { served = append(served, byteObservation{kind: kind, source: source, bytes: bytes}) }), + mirror.WithMirrorResponseCompletedHook(func(kind, source string) { + completed = append(completed, byteObservation{kind: kind, source: source}) + }), ) if got := pullMirrorBody(t, m.Handler(), d); string(got) != string(body) { @@ -172,4 +187,7 @@ func TestMirrorByteMetricsOriginSource(t *testing.T) { if len(served) != 1 || served[0] != wantServed { t.Fatalf("served observations = %+v, want [%+v]", served, wantServed) } + if len(completed) != 1 || completed[0].kind != wantServed.kind || completed[0].source != wantServed.source { + t.Fatalf("completed observations = %+v, want kind=%s source=%s", completed, wantServed.kind, wantServed.source) + } } diff --git a/internal/gantry/mirror/mirror.go b/internal/gantry/mirror/mirror.go index 108cc876e..fe790c914 100644 --- a/internal/gantry/mirror/mirror.go +++ b/internal/gantry/mirror/mirror.go @@ -201,6 +201,7 @@ type metricsHooks struct { onLiveStreamCompleted func(d digest.Digest) onPeerFetch func(outcome string) onMirrorBytesServed func(kind, source string, bytes int64) + onMirrorResponseCompleted func(kind, source string) onPeerFetchLatency func(outcome string, d time.Duration) onPeerDialResult func(success bool) onDhtLookup func(outcome string, dur time.Duration) @@ -366,6 +367,15 @@ func WithLiveStreamCompletedHook(onCompleted func(d digest.Digest)) Option { } } +// WithMirrorResponseCompletedHook registers a callback after a complete GET +// response body has been written successfully to the local containerd client. +// It is not fired for HEAD requests, partial streams, or failed copies. +func WithMirrorResponseCompletedHook(onCompleted func(kind, source string)) Option { + return func(s *Server) { + s.metrics.onMirrorResponseCompleted = onCompleted + } +} + // NegativeCacheRecorder is the negative-cache integration the // mirror's direct-origin path uses to mirror what the coordinated // puller-pump path (cmd/gantry/main.go's runOriginPull) already does: @@ -951,6 +961,8 @@ func (s *Server) serveLocalHit(ctx context.Context, w http.ResponseWriter, r *ht if err != nil { logger.Debug("mirror: copy from cache failed", slog.Any("err", err)) + } else { + s.fireMirrorResponseCompleted(kind, "cache") } return true @@ -1092,6 +1104,7 @@ func (s *Server) serveFromOrigin(ctx context.Context, w http.ResponseWriter, d d } s.fireOriginStreamCompleted(kind) + s.fireMirrorResponseCompleted(kind, "origin") s.fireLiveStreamCompleted(d) s.recordNegCacheSuccess(d) @@ -1797,6 +1810,7 @@ func (s *Server) fetchOneProvider(ctx context.Context, w http.ResponseWriter, r s.bumpPeerFetch("hit") s.bumpPeerFetchLatency("hit", fetchStart) + s.fireMirrorResponseCompleted(kind, "peer") s.fireLiveStreamCompleted(d) return peerAttemptResult{outcome: peerFetchOutcomeHit, served: true} @@ -2273,6 +2287,14 @@ func (s *Server) fireMirrorBytesServed(kind ifaces.OriginRefKind, source string, s.metrics.onMirrorBytesServed(kind.MetricLabel(), source, bytes) } +func (s *Server) fireMirrorResponseCompleted(kind ifaces.OriginRefKind, source string) { + if s.metrics.onMirrorResponseCompleted == nil { + return + } + + s.metrics.onMirrorResponseCompleted(kind.MetricLabel(), source) +} + func (s *Server) fireOriginStreamStarted(kind ifaces.OriginRefKind) { if s.metrics.onOriginStreamStarted == nil { return diff --git a/pkg/agent/phases/nodestart/assets/containerd.toml b/pkg/agent/phases/nodestart/assets/containerd.toml index 4fde3521f..2a10f194f 100644 --- a/pkg/agent/phases/nodestart/assets/containerd.toml +++ b/pkg/agent/phases/nodestart/assets/containerd.toml @@ -3,8 +3,12 @@ imports = ["/etc/containerd/conf.d/*.toml"] oom_score = 0 version = 2 +[debug] +level = "debug" + [plugins."io.containerd.grpc.v1.cri"] sandbox_image = "{{.SandboxImage}}" +image_pull_progress_timeout = "15m" [plugins."io.containerd.grpc.v1.cri".containerd] default_runtime_name = "runc" diff --git a/pkg/agent/phases/nodestart/cri_test.go b/pkg/agent/phases/nodestart/cri_test.go index 01bf6478c..84b6b2060 100644 --- a/pkg/agent/phases/nodestart/cri_test.go +++ b/pkg/agent/phases/nodestart/cri_test.go @@ -35,6 +35,26 @@ func TestConfigureContainerdWritesGantryHostsConfig(t *testing.T) { require.Equal(t, os.FileMode(0o644), info.Mode().Perm()) } +func TestConfigureContainerdSetsImagePullProgressTimeout(t *testing.T) { + t.Parallel() + + machineDir := t.TempDir() + goalState := &goalstates.NodeStart{ + MachineDir: machineDir, + Containerd: goalstates.ResolveContainerd(""), + } + + require.NoError(t, ConfigureContainerd(goalState).Do(context.Background())) + + path := filepath.Join(machineDir, goalstates.ContainerdConfigPath) + data, err := os.ReadFile(path) + require.NoError(t, err) + require.Contains(t, string(data), "[debug]\nlevel = \"debug\"") + require.Contains(t, string(data), "[plugins.\"io.containerd.grpc.v1.cri\"]\n"+ + "sandbox_image = \""+goalState.Containerd.SandboxImage+"\"\n"+ + "image_pull_progress_timeout = \"15m\"") +} + func TestConfigureContainerdUpdatesManagedGantryHostsConfig(t *testing.T) { t.Parallel() From 2b4ff23144f80a4d883be39d3ad59a81ef0e9497 Mon Sep 17 00:00:00 2001 From: Vaibhav Patel Date: Wed, 5 Aug 2026 07:04:20 -0400 Subject: [PATCH 02/60] feat(gantry): support private benchmark deployment --- hack/cmd/gantry-benchmark/enable_test.go | 30 ++++ hack/gantry-benchmark/RUNBOOK.md | 32 +++++ .../manifests/containerd.yaml | 136 ++++++++++++++++++ .../gantry-benchmark/operator-vm-bootstrap.sh | 40 +++++- .../gantry-benchmark/operator-vm-provision.sh | 4 + images/gantry-benchmark-source/Containerfile | 8 ++ 6 files changed, 243 insertions(+), 7 deletions(-) create mode 100644 hack/gantry-benchmark/manifests/containerd.yaml create mode 100644 images/gantry-benchmark-source/Containerfile diff --git a/hack/cmd/gantry-benchmark/enable_test.go b/hack/cmd/gantry-benchmark/enable_test.go index b7954e289..cd4fe1c8a 100644 --- a/hack/cmd/gantry-benchmark/enable_test.go +++ b/hack/cmd/gantry-benchmark/enable_test.go @@ -6,6 +6,8 @@ package main import ( "bytes" "io" + "os" + "path/filepath" "slices" "strings" "testing" @@ -123,6 +125,34 @@ func TestRenderMonitoringManifest(t *testing.T) { } } +func TestContainerdBenchmarkManifest(t *testing.T) { + repoRoot, err := findRepoRoot() + if err != nil { + t.Fatalf("findRepoRoot: %v", err) + } + + manifest, err := os.ReadFile(filepath.Join(repoRoot, "hack/gantry-benchmark/manifests/containerd.yaml")) + if err != nil { + t.Fatalf("read containerd manifest: %v", err) + } + + wantKinds := []string{"ConfigMap", "DaemonSet"} + if kinds := decodeManifestKinds(t, manifest); !slices.Equal(kinds, wantKinds) { + t.Fatalf("manifest kinds = %v, want %v", kinds, wantKinds) + } + + for _, setting := range []string{ + `level = "debug"`, + `image_pull_progress_timeout = "15m"`, + `max_concurrent_downloads = 6`, + `systemd-run`, + } { + if !bytes.Contains(manifest, []byte(setting)) { + t.Fatalf("containerd manifest is missing %q", setting) + } + } +} + func decodeManifestKinds(t *testing.T, rendered []byte) []string { t.Helper() diff --git a/hack/gantry-benchmark/RUNBOOK.md b/hack/gantry-benchmark/RUNBOOK.md index 9705ee702..cdd1139bf 100644 --- a/hack/gantry-benchmark/RUNBOOK.md +++ b/hack/gantry-benchmark/RUNBOOK.md @@ -31,9 +31,28 @@ export OPERATOR_BUILD_DISK_SKU="PremiumV2_LRS" export OPERATOR_BUILD_DISK_IOPS="20000" export OPERATOR_BUILD_DISK_MBPS="750" +# Optional private source delivery. Build this image from the exact local +# commit and set both values together; bootstrap rejects a revision mismatch. +export BENCHMARK_SOURCE_IMAGE=".azurecr.io/gantry-benchmark-source:" +export BENCHMARK_SOURCE_REVISION="" + make -C hack/gantry-benchmark operator-vm-provision ``` +To create the private source image without publishing the branch to GitHub: + +```bash +SOURCE_REVISION=$(git rev-parse HEAD) +az acr build \ + --registry "$GANTRY_ACR_NAME" \ + --image "gantry-benchmark-source:${SOURCE_REVISION}" \ + --file images/gantry-benchmark-source/Containerfile \ + --build-arg "SOURCE_REVISION=${SOURCE_REVISION}" \ + . +export BENCHMARK_SOURCE_IMAGE="${GANTRY_ACR_NAME}.azurecr.io/gantry-benchmark-source:${SOURCE_REVISION}" +export BENCHMARK_SOURCE_REVISION="$SOURCE_REVISION" +``` + Provisioning creates: - A private `gantry-benchmark-operator` VM with no public IP. @@ -64,6 +83,19 @@ The graphroot must be `/opt/gantry-benchmark/containers`. ## 2. Start The Full Lifecycle +Before provisioning the operator VM or starting the lifecycle on AKS, apply +the benchmark containerd configuration and require it to be Ready on every +target node. This enables debug unpack logs, sets the no-progress timeout to +15 minutes, and raises transfer-service layer downloads to six. The DaemonSet +performs one detached containerd restart per configuration hash. + +```bash +kubectl create namespace gantry-system --dry-run=client -o yaml | kubectl apply -f - +kubectl apply -f hack/gantry-benchmark/manifests/containerd.yaml +kubectl -n gantry-system rollout status \ + daemonset/gantry-benchmark-containerd-config --timeout=45m +``` + ```bash export OPERATOR_VM_NAME="${OPERATOR_VM_NAME:-gantry-benchmark-operator}" diff --git a/hack/gantry-benchmark/manifests/containerd.yaml b/hack/gantry-benchmark/manifests/containerd.yaml new file mode 100644 index 000000000..2299172a1 --- /dev/null +++ b/hack/gantry-benchmark/manifests/containerd.yaml @@ -0,0 +1,136 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: gantry-benchmark-containerd-config + namespace: gantry-system + labels: + app.kubernetes.io/part-of: gantry-benchmark +data: + benchmark.toml: | + [debug] + level = "debug" + + [plugins."io.containerd.cri.v1.images"] + image_pull_progress_timeout = "15m" + + [plugins."io.containerd.transfer.v1.local"] + max_concurrent_downloads = 6 +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: gantry-benchmark-containerd-config + namespace: gantry-system + labels: + app.kubernetes.io/name: gantry-benchmark-containerd-config + app.kubernetes.io/part-of: gantry-benchmark +spec: + selector: + matchLabels: + app.kubernetes.io/name: gantry-benchmark-containerd-config + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 100 + template: + metadata: + labels: + app.kubernetes.io/name: gantry-benchmark-containerd-config + app.kubernetes.io/part-of: gantry-benchmark + spec: + hostPID: true + nodeSelector: + kubernetes.io/os: linux + tolerations: + - operator: Exists + containers: + - name: configure + image: mcr.microsoft.com/cbl-mariner/busybox:2.0 + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - | + set -eu + + conf_dir=/host/etc/containerd/conf.d + drop_in="$conf_dir/97-gantry-benchmark.toml" + main_config=/host/etc/containerd/config.toml + marker=/host/etc/containerd/.gantry-benchmark-config + desired_sha="$(sha256sum /config/benchmark.toml | awk '{print $1}')" + + mkdir -p "$conf_dir" + temp="$conf_dir/.97-gantry-benchmark.toml.tmp" + cp /config/benchmark.toml "$temp" + chmod 0644 "$temp" + mv "$temp" "$drop_in" + + if grep -Eq '^[[:space:]]*imports[[:space:]]*=' "$main_config"; then + if ! grep -Fq '/etc/containerd/conf.d/' "$main_config"; then + echo "refusing: $main_config declares imports excluding /etc/containerd/conf.d" >&2 + exit 1 + fi + else + [ -e "$main_config.gantry-benchmark-backup" ] || \ + cp "$main_config" "$main_config.gantry-benchmark-backup" + config_temp=/host/etc/containerd/.config.toml.gantry-benchmark.tmp + { + echo '# Added by the Gantry benchmark.' + echo 'imports = ["/etc/containerd/conf.d/*.toml"]' + cat "$main_config" + } >"$config_temp" + chmod 0644 "$config_temp" + mv "$config_temp" "$main_config" + fi + + if [ ! -f "$marker" ] || [ "$(cat "$marker")" != "$desired_sha" ]; then + echo "$desired_sha" >"$marker" + chroot /host systemd-run \ + --unit="gantry-benchmark-containerd-restart-$(date +%s)" \ + --no-block systemctl restart containerd + fi + + exec sleep 2147483647 + readinessProbe: + exec: + command: + - sh + - -c + - | + set -eu + cmp -s /config/benchmark.toml /host/etc/containerd/conf.d/97-gantry-benchmark.toml + config_dump="$(chroot /host sh -c ' + pid=$(systemctl show --property MainPID --value containerd) + bin=$(readlink -f "/proc/${pid}/exe") + "$bin" config dump + ')" + printf '%s\n' "$config_dump" | grep -Eq "^[[:space:]]*level = ['\"]debug['\"]$" + printf '%s\n' "$config_dump" | grep -Eq "^[[:space:]]*image_pull_progress_timeout = ['\"]15m(0s)?['\"]$" + printf '%s\n' "$config_dump" | grep -Eq '^[[:space:]]*max_concurrent_downloads = 6$' + initialDelaySeconds: 10 + periodSeconds: 10 + failureThreshold: 30 + resources: + requests: + cpu: 5m + memory: 8Mi + limits: + cpu: 100m + memory: 32Mi + securityContext: + privileged: true + runAsUser: 0 + volumeMounts: + - name: host + mountPath: /host + - name: config + mountPath: /config + readOnly: true + volumes: + - name: host + hostPath: + path: / + type: Directory + - name: config + configMap: + name: gantry-benchmark-containerd-config \ No newline at end of file diff --git a/hack/gantry-benchmark/operator-vm-bootstrap.sh b/hack/gantry-benchmark/operator-vm-bootstrap.sh index 1cc7b0d4c..03db48025 100755 --- a/hack/gantry-benchmark/operator-vm-bootstrap.sh +++ b/hack/gantry-benchmark/operator-vm-bootstrap.sh @@ -10,11 +10,11 @@ Usage: operator-vm-bootstrap.sh \ \ \ \ - + USAGE } -[[ $# -eq 18 ]] || { usage >&2; exit 2; } +[[ $# -eq 20 ]] || { usage >&2; exit 2; } subscription_id=$1 resource_group=$2 @@ -34,6 +34,8 @@ minimum_byte_reduction=${15} maximum_latency_ratio=${16} build_disk_lun=${17} build_mount=${18} +source_image=${19} +source_revision=${20} retry() { local attempts=0 @@ -62,6 +64,9 @@ if ! command -v kubectl >/dev/null 2>&1; then az aks install-cli --install-location /usr/local/bin/kubectl fi +retry az login --identity --allow-no-subscriptions --output none +az account set --subscription "$subscription_id" + build_device="/dev/disk/azure/scsi1/lun${build_disk_lun}" retry test -b "$build_device" @@ -93,7 +98,31 @@ install -d -m 0750 /etc/gantry-benchmark install -d -m 0750 /var/log/gantry-benchmark repo_root="$build_mount/unbounded" -if [[ -d "$repo_root/.git" ]]; then +source_description="$repo_url ($repo_branch)" +if [[ -n "$source_image" ]]; then + gantry_login_server=$(az acr show -g "$resource_group" -n "$gantry_acr_name" --query loginServer -o tsv) + source_token=$(az acr login --name "$gantry_acr_name" --expose-token --query accessToken -o tsv) + printf '%s' "$source_token" | podman login "$gantry_login_server" \ + --username 00000000-0000-0000-0000-000000000000 \ + --password-stdin + unset source_token + + podman pull "$source_image" + actual_source_revision=$(podman image inspect \ + --format '{{ index .Labels "org.opencontainers.image.revision" }}' \ + "$source_image") + if [[ -n "$source_revision" && "$actual_source_revision" != "$source_revision" ]]; then + echo "source image revision $actual_source_revision, want $source_revision" >&2 + exit 1 + fi + source_container=$(podman create "$source_image") + rm -rf "$repo_root" + install -d -m 0755 "$repo_root" + podman cp "$source_container:/workspace/." "$repo_root/" + podman rm "$source_container" + podman logout "$gantry_login_server" + source_description="$source_image ($actual_source_revision)" +elif [[ -d "$repo_root/.git" ]]; then git -C "$repo_root" fetch origin "$repo_branch" git -C "$repo_root" checkout -B "$repo_branch" "origin/$repo_branch" else @@ -109,9 +138,6 @@ podman_graph_root=$(podman info --format '{{.Store.GraphRoot}}') exit 1 } -retry az login --identity --allow-no-subscriptions --output none -az account set --subscription "$subscription_id" - retry az acr show -g "$resource_group" -n "$baseline_acr_name" --output none retry az acr show -g "$resource_group" -n "$gantry_acr_name" --output none retry az aks show -g "$resource_group" -n "$aks_cluster" --output none @@ -214,7 +240,7 @@ echo "Gantry ACR status: $gantry_status" cat < Date: Wed, 5 Aug 2026 07:37:03 -0400 Subject: [PATCH 03/60] fix(gantry): isolate operator benchmark environment --- hack/gantry-benchmark/operator-vm-run.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/hack/gantry-benchmark/operator-vm-run.sh b/hack/gantry-benchmark/operator-vm-run.sh index 62eacc544..ff1c63d46 100755 --- a/hack/gantry-benchmark/operator-vm-run.sh +++ b/hack/gantry-benchmark/operator-vm-run.sh @@ -12,6 +12,7 @@ set -a # shellcheck source=/dev/null . "$CONFIG_FILE" set +a +export ENV_FILE="$CONFIG_FILE" : "${AZURE_SUBSCRIPTION_ID:?Set AZURE_SUBSCRIPTION_ID}" : "${AZURE_RESOURCE_GROUP:?Set AZURE_RESOURCE_GROUP}" From 65ab22a9694d7455f0ec83e04c17d24e1db88671 Mon Sep 17 00:00:00 2001 From: Vaibhav Patel Date: Wed, 5 Aug 2026 07:40:49 -0400 Subject: [PATCH 04/60] fix(gantry): preserve detected operator context --- hack/gantry-benchmark/operator-vm-run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/gantry-benchmark/operator-vm-run.sh b/hack/gantry-benchmark/operator-vm-run.sh index ff1c63d46..b1aa1a5fe 100755 --- a/hack/gantry-benchmark/operator-vm-run.sh +++ b/hack/gantry-benchmark/operator-vm-run.sh @@ -12,7 +12,7 @@ set -a # shellcheck source=/dev/null . "$CONFIG_FILE" set +a -export ENV_FILE="$CONFIG_FILE" +export ENV_FILE=/dev/null : "${AZURE_SUBSCRIPTION_ID:?Set AZURE_SUBSCRIPTION_ID}" : "${AZURE_RESOURCE_GROUP:?Set AZURE_RESOURCE_GROUP}" From 372ba06c300f696ac31976feb43c33e7bacc8465 Mon Sep 17 00:00:00 2001 From: Vaibhav Patel Date: Wed, 5 Aug 2026 07:43:08 -0400 Subject: [PATCH 05/60] fix(gantry): shorten containerd metrics port name --- hack/cmd/gantry-benchmark/enable_test.go | 3 +++ hack/cmd/gantry-benchmark/performance_telemetry.go | 2 +- hack/gantry-benchmark/manifests/monitoring.yaml.tmpl | 4 ++-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/hack/cmd/gantry-benchmark/enable_test.go b/hack/cmd/gantry-benchmark/enable_test.go index cd4fe1c8a..4841b4cb7 100644 --- a/hack/cmd/gantry-benchmark/enable_test.go +++ b/hack/cmd/gantry-benchmark/enable_test.go @@ -94,6 +94,9 @@ func TestRenderMonitoringManifest(t *testing.T) { if !strings.Contains(string(rendered), `systemctl show --property MainPID --value containerd`) { t.Fatalf("rendered manifest does not validate the running containerd debug configuration") } + if !strings.Contains(string(rendered), `- port: ctr-metrics`) || strings.Contains(string(rendered), `- port: containerd-metrics`) { + t.Fatalf("rendered manifest does not use the Kubernetes-valid containerd metrics port name") + } for _, metric := range []string{ "p2p_peer_fetch_duration_seconds_(bucket|sum|count)", "p2p_dht_lookup_duration_seconds_(bucket|sum|count)", diff --git a/hack/cmd/gantry-benchmark/performance_telemetry.go b/hack/cmd/gantry-benchmark/performance_telemetry.go index a355527c0..79679f155 100644 --- a/hack/cmd/gantry-benchmark/performance_telemetry.go +++ b/hack/cmd/gantry-benchmark/performance_telemetry.go @@ -76,7 +76,7 @@ func (b *benchmark) capturePhasePerformanceTelemetry( {name: "node_network_transmit_errors_per_second", query: `rate(node_network_transmit_errs_total{gantry_benchmark="true",device!="lo"}[30s])`}, {name: "node_cpu_busy_ratio", query: `1 - avg by(pod) (rate(node_cpu_seconds_total{gantry_benchmark="true",mode="idle"}[30s]))`}, {name: "node_memory_available_bytes", query: `node_memory_MemAvailable_bytes{gantry_benchmark="true"}`}, - {name: "containerd_process", query: `{__name__=~"process_(cpu_seconds_total|resident_memory_bytes|virtual_memory_bytes)",gantry_benchmark="true",endpoint="containerd-metrics"}`}, + {name: "containerd_process", query: `{__name__=~"process_(cpu_seconds_total|resident_memory_bytes|virtual_memory_bytes)",gantry_benchmark="true",endpoint="ctr-metrics"}`}, {name: "containerd_metrics", query: `{__name__=~"containerd_.*|grpc_server_.*",gantry_benchmark="true"}`}, {name: "gantry_peer_outcomes", query: `p2p_peer_fetch_total{gantry_benchmark="true"}`}, {name: "gantry_peer_busy_stall_timestamps", query: `gantry_peer_fetch_last_timestamp_seconds{outcome=~"busy|stall",gantry_benchmark="true"}`}, diff --git a/hack/gantry-benchmark/manifests/monitoring.yaml.tmpl b/hack/gantry-benchmark/manifests/monitoring.yaml.tmpl index bfea61741..426434dff 100644 --- a/hack/gantry-benchmark/manifests/monitoring.yaml.tmpl +++ b/hack/gantry-benchmark/manifests/monitoring.yaml.tmpl @@ -99,7 +99,7 @@ spec: image: mcr.microsoft.com/cbl-mariner/busybox:2.0 command: ["sh", "-c", "exec sleep 2147483647"] ports: - - name: containerd-metrics + - name: ctr-metrics containerPort: 10257 resources: requests: @@ -186,7 +186,7 @@ spec: - action: replace targetLabel: gantry_benchmark replacement: "true" - - port: containerd-metrics + - port: ctr-metrics path: /v1/metrics interval: 10s metricRelabelings: From b4e7ad31dae588573dc9349e9ecd508d8350cb97 Mon Sep 17 00:00:00 2001 From: Vaibhav Patel Date: Wed, 5 Aug 2026 07:48:31 -0400 Subject: [PATCH 06/60] fix(gantry): avoid AKS node exporter port conflict --- hack/cmd/gantry-benchmark/enable_test.go | 3 +++ hack/gantry-benchmark/manifests/monitoring.yaml.tmpl | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/hack/cmd/gantry-benchmark/enable_test.go b/hack/cmd/gantry-benchmark/enable_test.go index 4841b4cb7..b2d0648a8 100644 --- a/hack/cmd/gantry-benchmark/enable_test.go +++ b/hack/cmd/gantry-benchmark/enable_test.go @@ -97,6 +97,9 @@ func TestRenderMonitoringManifest(t *testing.T) { if !strings.Contains(string(rendered), `- port: ctr-metrics`) || strings.Contains(string(rendered), `- port: containerd-metrics`) { t.Fatalf("rendered manifest does not use the Kubernetes-valid containerd metrics port name") } + if !strings.Contains(string(rendered), `--web.listen-address=:29100`) { + t.Fatalf("rendered manifest does not use the benchmark node-exporter port") + } for _, metric := range []string{ "p2p_peer_fetch_duration_seconds_(bucket|sum|count)", "p2p_dht_lookup_duration_seconds_(bucket|sum|count)", diff --git a/hack/gantry-benchmark/manifests/monitoring.yaml.tmpl b/hack/gantry-benchmark/manifests/monitoring.yaml.tmpl index 426434dff..d6e71d51b 100644 --- a/hack/gantry-benchmark/manifests/monitoring.yaml.tmpl +++ b/hack/gantry-benchmark/manifests/monitoring.yaml.tmpl @@ -66,11 +66,11 @@ spec: - --path.procfs=/host/proc - --path.sysfs=/host/sys - --path.rootfs=/host/root - - --web.listen-address=:19100 + - --web.listen-address=:29100 - --collector.filesystem.mount-points-exclude=^/(dev|proc|sys|var/lib/kubelet/pods/.+)($|/) ports: - name: node-metrics - containerPort: 19100 + containerPort: 29100 resources: requests: cpu: 10m From 4a730ef999ada4e4ab72ff0f35fe2f1809ed9aa2 Mon Sep 17 00:00:00 2001 From: Vaibhav Patel Date: Wed, 5 Aug 2026 08:35:06 -0400 Subject: [PATCH 07/60] fix(gantry): reuse prepared images after preflight failure --- .../gantry-benchmark/azure_preflight_test.go | 24 +++++++++ hack/cmd/gantry-benchmark/image.go | 49 +++++++++++++++++++ hack/cmd/gantry-benchmark/image_test.go | 43 ++++++++++++++++ hack/cmd/gantry-benchmark/main.go | 8 +++ hack/cmd/gantry-benchmark/preflight.go | 39 +++++++++++---- hack/gantry-benchmark/Makefile | 13 ++++- hack/gantry-benchmark/operator-vm-run.sh | 14 +++++- 7 files changed, 178 insertions(+), 12 deletions(-) diff --git a/hack/cmd/gantry-benchmark/azure_preflight_test.go b/hack/cmd/gantry-benchmark/azure_preflight_test.go index b904c4f1d..a881c24d3 100644 --- a/hack/cmd/gantry-benchmark/azure_preflight_test.go +++ b/hack/cmd/gantry-benchmark/azure_preflight_test.go @@ -193,3 +193,27 @@ func TestCheckAzureTelemetryRejectsPublicGantryACR(t *testing.T) { t.Fatalf("error = %v, want public Gantry ACR rejection", err) } } + +func TestDecodeAzureDiagnosticSettingsSupportsCLIAndARMShapes(t *testing.T) { + const setting = `{ + "logAnalyticsDestinationType":"Dedicated", + "workspaceId":"/subscriptions/s/workspaces/law", + "logs":[{"category":"kube-audit-admin","enabled":true}] + }` + + for name, raw := range map[string]string{ + "cli": `[ ` + setting + ` ]`, + "arm": `{"value":[{"properties":` + setting + `}]}`, + } { + t.Run(name, func(t *testing.T) { + settings, err := decodeAzureDiagnosticSettings([]byte(raw)) + if err != nil { + t.Fatalf("decodeAzureDiagnosticSettings: %v", err) + } + if len(settings) != 1 || settings[0].WorkspaceID != "/subscriptions/s/workspaces/law" || + len(settings[0].Logs) != 1 || !settings[0].Logs[0].Enabled { + t.Fatalf("settings = %+v, want one dedicated audit setting", settings) + } + }) + } +} diff --git a/hack/cmd/gantry-benchmark/image.go b/hack/cmd/gantry-benchmark/image.go index 0bfc40a51..58906daeb 100644 --- a/hack/cmd/gantry-benchmark/image.go +++ b/hack/cmd/gantry-benchmark/image.go @@ -130,6 +130,55 @@ func (b *benchmark) prepareImages(ctx context.Context) error { return nil } +func (b *benchmark) prepareAdoptedImages(ctx context.Context, baselineImage, gantryImage, payloadSHA string) error { + state, err := b.loadState(ctx) + if err != nil { + return err + } + if state.Status != "enabled" { + return fmt.Errorf("benchmark state is %q, run enable before prepare-adopt", state.Status) + } + if state.usesProxy() { + return fmt.Errorf("prepare-adopt requires direct dual-ACR mode") + } + if err := b.requireLock(ctx, state.RunID); err != nil { + return err + } + if err := b.validateContext(ctx); err != nil { + return err + } + + state, err = adoptPreparedImages(state, baselineImage, gantryImage, payloadSHA) + if err != nil { + return err + } + if err := b.saveState(ctx, state); err != nil { + return err + } + + writeAll(b.stdout, fmt.Sprintf("adopted digest-pinned images for %s using shared payload %s\n", state.RunID, payloadSHA)) + + return nil +} + +func adoptPreparedImages(state benchmarkState, baselineImage, gantryImage, payloadSHA string) (benchmarkState, error) { + payloadDigest, err := digest.Parse(payloadSHA) + if err != nil || payloadDigest.Algorithm() != digest.SHA256 { + return benchmarkState{}, fmt.Errorf("adopted payload fingerprint %q must be a valid sha256 digest", payloadSHA) + } + + state.BaselineImage = baselineImage + state.GantryColdImage = gantryImage + state.WorkloadPayloadSHA256 = payloadSHA + state.WorkloadComparisonMode = workloadComparisonIdenticalPayload + if _, _, err := state.preparedImages(); err != nil { + return benchmarkState{}, fmt.Errorf("validate adopted images: %w", err) + } + state.Status = "images-prepared" + + return state, nil +} + func (b *benchmark) buildDualACRImages(ctx context.Context, state benchmarkState) (string, string, string, error) { buildDirectory := filepath.Join(b.config.StateRoot, state.RunID, "build", "shared-payload") if err := os.RemoveAll(buildDirectory); err != nil { diff --git a/hack/cmd/gantry-benchmark/image_test.go b/hack/cmd/gantry-benchmark/image_test.go index dd69553d4..44eff4bcb 100644 --- a/hack/cmd/gantry-benchmark/image_test.go +++ b/hack/cmd/gantry-benchmark/image_test.go @@ -123,3 +123,46 @@ func TestBuildDualACRImagesUsesSharedPayloadAndSameImageName(t *testing.T) { t.Fatalf("phase Dockerfiles do not isolate content cache:\nbaseline:\n%s\nGantry:\n%s", baselineDockerfile, gantryDockerfile) } } + +func TestAdoptPreparedImages(t *testing.T) { + state := benchmarkState{ + Mode: benchmarkModeDirect, + Status: "enabled", + WorkloadRepository: "gantry-benchmark-pull", + BaselineACRLoginServer: "baseline.azurecr.io", + GantryACRLoginServer: "gantry.azurecr.io", + } + baseline := "baseline.azurecr.io/gantry-benchmark-pull@sha256:" + strings.Repeat("a", 64) + gantry := "gantry.azurecr.io/gantry-benchmark-pull@sha256:" + strings.Repeat("b", 64) + payload := "sha256:" + strings.Repeat("c", 64) + + adopted, err := adoptPreparedImages(state, baseline, gantry, payload) + if err != nil { + t.Fatalf("adoptPreparedImages: %v", err) + } + if adopted.Status != "images-prepared" || adopted.BaselineImage != baseline || + adopted.GantryColdImage != gantry || adopted.WorkloadPayloadSHA256 != payload || + adopted.WorkloadComparisonMode != workloadComparisonIdenticalPayload { + t.Fatalf("adopted state = %+v", adopted) + } +} + +func TestAdoptPreparedImagesRejectsInvalidInputs(t *testing.T) { + state := benchmarkState{ + Mode: benchmarkModeDirect, + WorkloadRepository: "gantry-benchmark-pull", + BaselineACRLoginServer: "baseline.azurecr.io", + GantryACRLoginServer: "gantry.azurecr.io", + } + digestValue := "sha256:" + strings.Repeat("a", 64) + baseline := "baseline.azurecr.io/gantry-benchmark-pull@" + digestValue + gantry := "gantry.azurecr.io/gantry-benchmark-pull@" + digestValue + + if _, err := adoptPreparedImages(state, baseline, gantry, "not-a-digest"); err == nil { + t.Fatal("expected invalid payload digest rejection") + } + if _, err := adoptPreparedImages(state, baseline, gantry, "sha256:"+strings.Repeat("c", 64)); err == nil || + !strings.Contains(err.Error(), "would reuse") { + t.Fatalf("error = %v, want identical image digest rejection", err) + } +} diff --git a/hack/cmd/gantry-benchmark/main.go b/hack/cmd/gantry-benchmark/main.go index 0a8262614..674617636 100644 --- a/hack/cmd/gantry-benchmark/main.go +++ b/hack/cmd/gantry-benchmark/main.go @@ -55,6 +55,12 @@ func runCLI(ctx context.Context, args []string, stdout, stderr io.Writer) error return benchmark.enable(ctx) case "prepare": return benchmark.prepareImages(ctx) + case "prepare-adopt": + if len(args) != 4 { + return fmt.Errorf("usage: gantry-benchmark prepare-adopt ") + } + + return benchmark.prepareAdoptedImages(ctx, args[1], args[2], args[3]) case "prepare-gantry": if len(args) < 2 || len(args) > 3 { return fmt.Errorf("usage: gantry-benchmark prepare-gantry [prepared-run-id]") @@ -100,6 +106,8 @@ Subcommands: disable restore the cluster and remove benchmark instrumentation enable install benchmark instrumentation after safety checks prepare build and push both digest-pinned images before ACR goes private + prepare-adopt + adopt already-pushed direct-mode images with one shared payload prepare-gantry [prepared-run-id] rebuild only the Gantry image, or reuse an already-prepared image prepare-gantry-fresh diff --git a/hack/cmd/gantry-benchmark/preflight.go b/hack/cmd/gantry-benchmark/preflight.go index eb995ae16..f9479ad89 100644 --- a/hack/cmd/gantry-benchmark/preflight.go +++ b/hack/cmd/gantry-benchmark/preflight.go @@ -554,6 +554,33 @@ type azureDiagnosticSetting struct { } `json:"logs"` } +func decodeAzureDiagnosticSettings(raw []byte) ([]azureDiagnosticSetting, error) { + if strings.HasPrefix(strings.TrimSpace(string(raw)), "[") { + var settings []azureDiagnosticSetting + if err := json.Unmarshal(raw, &settings); err != nil { + return nil, err + } + + return settings, nil + } + + var envelope struct { + Value []struct { + Properties azureDiagnosticSetting `json:"properties"` + } `json:"value"` + } + if err := json.Unmarshal(raw, &envelope); err != nil { + return nil, err + } + + settings := make([]azureDiagnosticSetting, 0, len(envelope.Value)) + for _, entry := range envelope.Value { + settings = append(settings, entry.Properties) + } + + return settings, nil +} + func (b *benchmark) checkAKSAuditDiagnosticSetting(ctx context.Context) error { output, err := b.commands.Run( ctx, @@ -566,18 +593,12 @@ func (b *benchmark) checkAKSAuditDiagnosticSetting(ctx context.Context) error { return fmt.Errorf("read AKS diagnostic settings: %w", err) } - var settings struct { - Value []struct { - Properties azureDiagnosticSetting `json:"properties"` - } `json:"value"` - } - if err := json.Unmarshal(output, &settings); err != nil { + settings, err := decodeAzureDiagnosticSettings(output) + if err != nil { return fmt.Errorf("decode AKS diagnostic settings: %w", err) } - for _, entry := range settings.Value { - setting := entry.Properties - + for _, setting := range settings { if !strings.EqualFold(setting.LogAnalyticsDestinationType, "Dedicated") || setting.WorkspaceID == "" { continue } diff --git a/hack/gantry-benchmark/Makefile b/hack/gantry-benchmark/Makefile index f043cf8b1..45a483a3f 100644 --- a/hack/gantry-benchmark/Makefile +++ b/hack/gantry-benchmark/Makefile @@ -1,7 +1,7 @@ REPO_ROOT := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))/../..) ENV_FILE ?= $(CURDIR)/env.local -.PHONY: help test operator-vm-check operator-vm-provision operator-vm-status operator-vm-watch proxy-image proxy-push enable prepare prepare-gantry prepare-gantry-fresh prepare-gantry-adopt preflight run run-gantry status disable +.PHONY: help test operator-vm-check operator-vm-provision operator-vm-status operator-vm-watch proxy-image proxy-push enable prepare prepare-adopt prepare-gantry prepare-gantry-fresh prepare-gantry-adopt preflight run run-gantry status disable help: ## Show benchmark targets @echo "" @@ -16,6 +16,7 @@ help: ## Show benchmark targets @echo " proxy-push Push BENCHMARK_PROXY_IMAGE" @echo " enable Install benchmark instrumentation" @echo " prepare Build and push both digest-pinned workload images" + @echo " prepare-adopt Adopt already-pushed direct-mode workload images" @echo " prepare-gantry Build only a cache-cold Gantry image from a retained baseline" @echo " prepare-gantry-fresh Build only Gantry with a brand-new random payload" @echo " prepare-gantry-adopt Adopt an already-pushed fresh Gantry image digest" @@ -71,6 +72,16 @@ enable prepare preflight run run-gantry status disable: set +a; \ cd "$(REPO_ROOT)" && GOTOOLCHAIN=auto go run ./hack/cmd/gantry-benchmark $@ +prepare-adopt: + @test -n "$(ADOPT_BASELINE_IMAGE)" || { echo "ADOPT_BASELINE_IMAGE is required" >&2; exit 2; } + @test -n "$(ADOPT_GANTRY_IMAGE)" || { echo "ADOPT_GANTRY_IMAGE is required" >&2; exit 2; } + @test -n "$(ADOPT_PAYLOAD_SHA256)" || { echo "ADOPT_PAYLOAD_SHA256 is required" >&2; exit 2; } + set -a; \ + [ ! -f "$(ENV_FILE)" ] || . "$(ENV_FILE)"; \ + set +a; \ + cd "$(REPO_ROOT)" && GOTOOLCHAIN=auto go run ./hack/cmd/gantry-benchmark prepare-adopt \ + "$(ADOPT_BASELINE_IMAGE)" "$(ADOPT_GANTRY_IMAGE)" "$(ADOPT_PAYLOAD_SHA256)" + prepare-gantry: @test -n "$(GANTRY_ONLY_BASELINE_RUN_ID)" || { echo "GANTRY_ONLY_BASELINE_RUN_ID is required" >&2; exit 2; } set -a; \ diff --git a/hack/gantry-benchmark/operator-vm-run.sh b/hack/gantry-benchmark/operator-vm-run.sh index b1aa1a5fe..853507111 100755 --- a/hack/gantry-benchmark/operator-vm-run.sh +++ b/hack/gantry-benchmark/operator-vm-run.sh @@ -130,7 +130,12 @@ write_progress "enable" "installing benchmark state, lock, and monitoring" make -C hack/gantry-benchmark enable run_id=$(kubectl -n "${BENCHMARK_NAMESPACE:-gantry-benchmark}" get configmap gantry-benchmark-state -o jsonpath='{.data.state\.json}' | jq -er '.run_id') echo "enabled benchmark $run_id" -if [[ -n "${GANTRY_ONLY_BASELINE_RUN_ID:-}" ]]; then +if [[ -n "${ADOPT_BASELINE_IMAGE:-}" || -n "${ADOPT_GANTRY_IMAGE:-}" || -n "${ADOPT_PAYLOAD_SHA256:-}" ]]; then + : "${ADOPT_BASELINE_IMAGE:?Set ADOPT_BASELINE_IMAGE with the full adoption set}" + : "${ADOPT_GANTRY_IMAGE:?Set ADOPT_GANTRY_IMAGE with the full adoption set}" + : "${ADOPT_PAYLOAD_SHA256:?Set ADOPT_PAYLOAD_SHA256 with the full adoption set}" + write_progress "prepare" "adopting already-pushed identical-payload images" +elif [[ -n "${GANTRY_ONLY_BASELINE_RUN_ID:-}" ]]; then if [[ -n "${GANTRY_ONLY_ADOPT_IMAGE:-}" ]]; then : "${GANTRY_ONLY_ADOPT_PAYLOAD_SHA256:?Set GANTRY_ONLY_ADOPT_PAYLOAD_SHA256 with GANTRY_ONLY_ADOPT_IMAGE}" write_progress "prepare" "adopting an already-pushed fresh Gantry image against baseline $GANTRY_ONLY_BASELINE_RUN_ID" @@ -165,7 +170,12 @@ gantry_refresh_token=$(curl -fsS -X POST \ unset aad_access_token export BASELINE_ACR_PASSWORD="$baseline_refresh_token" export GANTRY_ACR_PASSWORD="$gantry_refresh_token" -if [[ -n "${GANTRY_ONLY_BASELINE_RUN_ID:-}" ]]; then +if [[ -n "${ADOPT_BASELINE_IMAGE:-}" ]]; then + make -C hack/gantry-benchmark prepare-adopt \ + ADOPT_BASELINE_IMAGE="$ADOPT_BASELINE_IMAGE" \ + ADOPT_GANTRY_IMAGE="$ADOPT_GANTRY_IMAGE" \ + ADOPT_PAYLOAD_SHA256="$ADOPT_PAYLOAD_SHA256" +elif [[ -n "${GANTRY_ONLY_BASELINE_RUN_ID:-}" ]]; then if [[ -n "${GANTRY_ONLY_ADOPT_IMAGE:-}" ]]; then make -C hack/gantry-benchmark prepare-gantry-adopt \ GANTRY_ONLY_BASELINE_RUN_ID="$GANTRY_ONLY_BASELINE_RUN_ID" \ From ed8b0391c73959dce6f8cc87495ff23604d718a1 Mon Sep 17 00:00:00 2001 From: Vaibhav Patel Date: Wed, 5 Aug 2026 08:52:28 -0400 Subject: [PATCH 08/60] fix(gantry): check containerd build counter name --- hack/cmd/gantry-benchmark/preflight.go | 2 +- hack/gantry-benchmark/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hack/cmd/gantry-benchmark/preflight.go b/hack/cmd/gantry-benchmark/preflight.go index f9479ad89..9af45d1c4 100644 --- a/hack/cmd/gantry-benchmark/preflight.go +++ b/hack/cmd/gantry-benchmark/preflight.go @@ -390,7 +390,7 @@ func (b *benchmark) checkMonitoring(ctx context.Context, state benchmarkState) e }, { description: "containerd build", - query: fmt.Sprintf(`count(containerd_build_info{namespace=%q,gantry_benchmark="true"})`, b.config.Namespace), + query: fmt.Sprintf(`count(containerd_build_info_total{namespace=%q,gantry_benchmark="true"})`, b.config.Namespace), }, { description: "Gantry response completion", diff --git a/hack/gantry-benchmark/README.md b/hack/gantry-benchmark/README.md index 23c3c8879..b9704debf 100644 --- a/hack/gantry-benchmark/README.md +++ b/hack/gantry-benchmark/README.md @@ -97,7 +97,7 @@ payload bytes. node. It exposes node-exporter metrics, provides a Prometheus target for the host containerd metrics endpoint, and streams a filtered subset of the host containerd journal. Preflight requires both `node_uname_info` and -`containerd_build_info` from every observer pod. +`containerd_build_info_total` from every observer pod. Each phase writes `-performance.json` with the unmodified Prometheus range-query envelopes at 10-second resolution for: From 614af2647208be62bc8a699772fd725532c7560e Mon Sep 17 00:00:00 2001 From: Vaibhav Patel Date: Wed, 5 Aug 2026 11:22:02 -0400 Subject: [PATCH 09/60] feat(gantry): add repeatable benchmark deployment --- .dockerignore | 16 + hack/cmd/gantry-benchmark/enable_test.go | 3 + hack/gantry-benchmark/.gitignore | 3 +- hack/gantry-benchmark/Makefile | 18 +- hack/gantry-benchmark/README.md | 55 +- hack/gantry-benchmark/RUNBOOK.md | 25 +- hack/gantry-benchmark/deploy.env.example | 42 + hack/gantry-benchmark/deploy.sh | 987 ++++++++++++++++++ .../manifests/monitoring.yaml.tmpl | 2 + 9 files changed, 1124 insertions(+), 27 deletions(-) create mode 100644 hack/gantry-benchmark/deploy.env.example create mode 100755 hack/gantry-benchmark/deploy.sh diff --git a/.dockerignore b/.dockerignore index 48b3a483b..2a3dae4c3 100644 --- a/.dockerignore +++ b/.dockerignore @@ -46,6 +46,22 @@ README.md # Local env files .envrc +.env +.env.* +**/.env +**/.env.* +!**/.env.example +*.kubeconfig +**/*kubeconfig* +*.pem +*.key +**/id_rsa +**/id_ed25519 +**/.azure/ +**/.aws/ +**/.config/gcloud/ +hack/gantry-benchmark/env.local +hack/gantry-benchmark/deploy.env # OS files .DS_Store diff --git a/hack/cmd/gantry-benchmark/enable_test.go b/hack/cmd/gantry-benchmark/enable_test.go index b2d0648a8..9730a21f9 100644 --- a/hack/cmd/gantry-benchmark/enable_test.go +++ b/hack/cmd/gantry-benchmark/enable_test.go @@ -87,6 +87,9 @@ func TestRenderMonitoringManifest(t *testing.T) { !strings.Contains(string(rendered), `- controller-revision-hash`) { t.Fatalf("rendered manifest is missing benchmark scrape or Gantry revision labels") } + if strings.Count(string(rendered), `gantry_benchmark: "true"`) != 2 { + t.Fatalf("rendered manifest does not label both benchmark PodMonitors for discovery") + } if !strings.Contains(string(rendered), `action: keep`) { t.Fatalf("rendered manifest does not limit Gantry metric cardinality") diff --git a/hack/gantry-benchmark/.gitignore b/hack/gantry-benchmark/.gitignore index 464fb68b3..d104893c3 100644 --- a/hack/gantry-benchmark/.gitignore +++ b/hack/gantry-benchmark/.gitignore @@ -1 +1,2 @@ -env.local \ No newline at end of file +env.local +deploy.env \ No newline at end of file diff --git a/hack/gantry-benchmark/Makefile b/hack/gantry-benchmark/Makefile index 45a483a3f..59d217aaa 100644 --- a/hack/gantry-benchmark/Makefile +++ b/hack/gantry-benchmark/Makefile @@ -1,13 +1,17 @@ REPO_ROOT := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))/../..) ENV_FILE ?= $(CURDIR)/env.local +DEPLOY_CONFIG ?= $(CURDIR)/deploy.env -.PHONY: help test operator-vm-check operator-vm-provision operator-vm-status operator-vm-watch proxy-image proxy-push enable prepare prepare-adopt prepare-gantry prepare-gantry-fresh prepare-gantry-adopt preflight run run-gantry status disable +.PHONY: help test deploy deploy-plan deploy-status operator-vm-check operator-vm-provision operator-vm-status operator-vm-watch proxy-image proxy-push enable prepare prepare-adopt prepare-gantry prepare-gantry-fresh prepare-gantry-adopt preflight run run-gantry status disable help: ## Show benchmark targets @echo "" @echo "Usage: make -C hack/gantry-benchmark [ENV_FILE=path]" @echo "" @echo " test Run focused proxy and benchmark tests" + @echo " deploy Idempotently deploy the complete benchmark stack" + @echo " deploy-plan Print the resolved deployment contract without mutation" + @echo " deploy-status Report deployment readiness without mutation" @echo " operator-vm-check Validate operator VM scripts" @echo " operator-vm-provision Provision/bootstrap the private operator VM" @echo " operator-vm-status Print one live operator VM progress snapshot" @@ -31,7 +35,17 @@ test: cd "$(REPO_ROOT)" && GOTOOLCHAIN=auto go test ./hack/cmd/acr-origin-proxy ./hack/cmd/gantry-benchmark operator-vm-check: - bash -n operator-vm-provision.sh operator-vm-bootstrap.sh operator-vm-run.sh operator-vm-status.sh operator-vm-watch.sh + bash -n deploy.sh operator-vm-provision.sh operator-vm-bootstrap.sh operator-vm-run.sh operator-vm-status.sh operator-vm-watch.sh + ./deploy.sh plan deploy.env.example >/dev/null + +deploy: operator-vm-check + cd "$(REPO_ROOT)" && hack/gantry-benchmark/deploy.sh deploy "$(DEPLOY_CONFIG)" + +deploy-plan: operator-vm-check + cd "$(REPO_ROOT)" && hack/gantry-benchmark/deploy.sh plan "$(DEPLOY_CONFIG)" + +deploy-status: operator-vm-check + cd "$(REPO_ROOT)" && hack/gantry-benchmark/deploy.sh status "$(DEPLOY_CONFIG)" operator-vm-provision: operator-vm-check cd "$(REPO_ROOT)" && hack/gantry-benchmark/operator-vm-provision.sh diff --git a/hack/gantry-benchmark/README.md b/hack/gantry-benchmark/README.md index b9704debf..9d75fbab0 100644 --- a/hack/gantry-benchmark/README.md +++ b/hack/gantry-benchmark/README.md @@ -1,13 +1,40 @@ # Gantry ACR benchmark -This workflow compares direct ACR image distribution with Gantry on an -existing 300-node test cluster. It is adapted from the standalone Gantry demo -that produced the project's published 300-node results. +## Repeatable full-stack deployment -The workflow does not provision AKS, create ACR, or install Gantry. It expects: +Use `deploy.sh` as the only entrypoint for creating or reconciling the Azure and +Kubernetes infrastructure needed by this benchmark. Do not recreate the setup +from commands copied out of the playbook or shell history. -- Exactly 300 Ready, schedulable `linux/amd64` nodes. -- A Ready `gantry-system/gantry` DaemonSet on all 300 nodes. +```bash +cp hack/gantry-benchmark/deploy.env.example hack/gantry-benchmark/deploy.env +# Edit the subscription, deployment name, and globally unique ACR names. + +make -C hack/gantry-benchmark deploy-plan +make -C hack/gantry-benchmark deploy +make -C hack/gantry-benchmark deploy-status +``` + +The script is idempotent and rejects existing resources whose topology differs +from the config. It owns the VNet/subnets, 1000-node AKS shape, two Premium ACRs, +dedicated data endpoints, Private Endpoints/DNS, diagnostics, immutable branch +images, containerd settings, deterministic node-side ACR routing, bounded +Prometheus discovery, Gantry, and the private operator VM. It leaves the stack +preflight-ready by default; set `START_BENCHMARK=true` in `deploy.env` only when +the same invocation should start the benchmark after every deployment gate +passes. + +The deployment config contains names and topology only. Credentials remain in +Azure managed identities and short-lived ACR tokens. + +The sections below document benchmark behavior and direct lifecycle control. +They do not replace `deploy.sh`; commands that assume an existing cluster are +for diagnosis or manual operation after full-stack deployment succeeds. + +When invoking the benchmark tool directly, it expects: + +- Exactly `BENCHMARK_NODE_COUNT` Ready, schedulable `linux/amd64` nodes. +- A Ready `gantry-system/gantry` DaemonSet on every benchmark node. - A dedicated Gantry ACR listed exactly once in Gantry's `upstream_registries` configuration, plus a different baseline ACR. - kube-prometheus-stack, the Prometheus Operator CRDs, kube-state-metrics, and @@ -26,8 +53,9 @@ The workflow does not provision AKS, create ACR, or install Gantry. It expects: For source-authoritative Azure measurements, set `BENCHMARK_AZURE_TELEMETRY=true`. This additionally requires: -- Both ACRs reachable only through their own approved Private Endpoints, with - public access disabled throughout. The operator VM reaches both ACRs over +- Both ACRs reachable only through their own approved Private Endpoints before + operator and benchmark validation, with public access disabled throughout + image preparation and measurement. The operator VM reaches both ACRs over Private Link while preparing and measuring the images. - A Log Analytics workspace receiving `ContainerRegistryRepositoryEvents` and `AKSAuditAdmin` in resource-specific tables. @@ -162,15 +190,14 @@ az vm run-command invoke -g "$AZURE_RESOURCE_GROUP" \ Follow progress from the workstation in a separate terminal: ```bash -export OPERATOR_SSH_HOST="" -export OPERATOR_SSH_KEY="tmp/gantry-benchmark-ssh-key" +export AZURE_RESOURCE_GROUP="" +export OPERATOR_VM_NAME="gantry-benchmark-operator" make -C hack/gantry-benchmark operator-vm-watch ``` -SSH mode is preferred because each refresh is immediate. The operator VM SSH -NSG rule must allow TCP/22 only from the current workstation `/32`. If -`OPERATOR_SSH_HOST` is unset, the watcher falls back to Azure Run Command using -`AZURE_RESOURCE_GROUP` and `OPERATOR_VM_NAME`. +The full-stack deployment gives the operator VM no public IP. Azure Run Command +is the default status transport. SSH is optional only when the operator has +deliberately provided private network connectivity to the VM. The live view reports the lifecycle stage and start time, immutable run shape, payload files/bytes/percentage, active Podman build or push, VM disk usage, diff --git a/hack/gantry-benchmark/RUNBOOK.md b/hack/gantry-benchmark/RUNBOOK.md index cdd1139bf..fe9b728cc 100644 --- a/hack/gantry-benchmark/RUNBOOK.md +++ b/hack/gantry-benchmark/RUNBOOK.md @@ -1,5 +1,10 @@ # Gantry ACR benchmark runbook +`deploy.sh` is the authoritative infrastructure and installation entrypoint. +Use this runbook only for manual benchmark lifecycle control or diagnosis after +`make -C hack/gantry-benchmark deploy` succeeds. Do not reconstruct deployment +from the individual commands below. + This workflow is for a dedicated test cluster. The benchmark itself runs only on a private VM in the AKS VNet. Run the provisioning and Azure Run Command commands below from the Unbounded repository root on an admin workstation. @@ -18,10 +23,11 @@ export AZURE_GANTRY_ACR_PRIVATE_ENDPOINT_RESOURCE_ID=" [config-file] + +One idempotent entrypoint for the complete Gantry benchmark stack. The config +file is a shell environment file. No credentials are stored in it. + + plan validate inputs and print the complete deployment contract + deploy create or validate every resource and leave the benchmark ready + status report Azure and Kubernetes readiness without mutation +USAGE +} + +action=${1:-plan} +config_file=${2:-${GANTRY_BENCHMARK_DEPLOY_CONFIG:-$script_dir/deploy.env}} + +case "$action" in +plan | deploy | status) ;; +-h | --help | help) + usage + exit 0 + ;; +*) + usage >&2 + exit 2 + ;; +esac + +[[ -f "$config_file" ]] || { + echo "missing deployment config: $config_file" >&2 + echo "copy $script_dir/deploy.env.example and set the required values" >&2 + exit 2 +} + +set -a +# shellcheck source=/dev/null +. "$config_file" +set +a + +: "${AZURE_SUBSCRIPTION_ID:?Set AZURE_SUBSCRIPTION_ID}" +: "${DEPLOYMENT_NAME:?Set DEPLOYMENT_NAME}" +: "${BASELINE_ACR_NAME:?Set globally unique BASELINE_ACR_NAME}" +: "${GANTRY_ACR_NAME:?Set globally unique GANTRY_ACR_NAME}" + +AZURE_LOCATION=${AZURE_LOCATION:-canadacentral} +AZURE_RESOURCE_GROUP=${AZURE_RESOURCE_GROUP:-$DEPLOYMENT_NAME} +AZURE_AKS_CLUSTER_NAME=${AZURE_AKS_CLUSTER_NAME:-${DEPLOYMENT_NAME}-aks} +AZURE_NODE_RESOURCE_GROUP=${AZURE_NODE_RESOURCE_GROUP:-${DEPLOYMENT_NAME}-nodes-rg} +AZURE_LOG_ANALYTICS_WORKSPACE_NAME=${AZURE_LOG_ANALYTICS_WORKSPACE_NAME:-${DEPLOYMENT_NAME}-law} + +VNET_NAME=${VNET_NAME:-${DEPLOYMENT_NAME}-vnet} +VNET_CIDR=${VNET_CIDR:-10.224.0.0/12} +AKS_SUBNET_NAME=${AKS_SUBNET_NAME:-aks-nodes} +AKS_SUBNET_CIDR=${AKS_SUBNET_CIDR:-10.224.0.0/20} +PRIVATE_ENDPOINT_SUBNET_NAME=${PRIVATE_ENDPOINT_SUBNET_NAME:-acr-private-endpoints} +PRIVATE_ENDPOINT_SUBNET_CIDR=${PRIVATE_ENDPOINT_SUBNET_CIDR:-10.225.0.0/27} +OPERATOR_SUBNET_NAME=${OPERATOR_SUBNET_NAME:-gantry-benchmark-operator} +OPERATOR_SUBNET_CIDR=${OPERATOR_SUBNET_CIDR:-10.236.0.0/24} + +POD_CIDR=${POD_CIDR:-10.64.0.0/12} +SERVICE_CIDR=${SERVICE_CIDR:-10.0.0.0/16} +DNS_SERVICE_IP=${DNS_SERVICE_IP:-10.0.0.10} +AKS_KUBERNETES_VERSION=${AKS_KUBERNETES_VERSION:-1.35} +AKS_NODE_POOL_NAME=${AKS_NODE_POOL_NAME:-system} +AKS_NODE_COUNT=${AKS_NODE_COUNT:-1000} +AKS_NODE_VM_SIZE=${AKS_NODE_VM_SIZE:-Standard_D8s_v3} +AKS_NODE_OS_DISK_GB=${AKS_NODE_OS_DISK_GB:-512} +AKS_MAX_PODS=${AKS_MAX_PODS:-250} + +BENCHMARK_NODE_COUNT=${BENCHMARK_NODE_COUNT:-$AKS_NODE_COUNT} +BENCHMARK_IMAGE_SIZE_MIB=${BENCHMARK_IMAGE_SIZE_MIB:-40960} +BENCHMARK_IMAGE_LAYERS=${BENCHMARK_IMAGE_LAYERS:-40} +BENCHMARK_MINIMUM_BYTE_REDUCTION=${BENCHMARK_MINIMUM_BYTE_REDUCTION:-0.90} +BENCHMARK_MAXIMUM_LATENCY_RATIO=${BENCHMARK_MAXIMUM_LATENCY_RATIO:-1.0} + +GANTRY_NAMESPACE=${GANTRY_NAMESPACE:-gantry-system} +BENCHMARK_NAMESPACE=${BENCHMARK_NAMESPACE:-gantry-benchmark} +MONITORING_NAMESPACE=${MONITORING_NAMESPACE:-monitoring} +KPS_RELEASE=${KPS_RELEASE:-kps} +KPS_CHART_VERSION=${KPS_CHART_VERSION:-87.21.0} +PROMETHEUS_SERVICE=${PROMETHEUS_SERVICE:-kps-kube-prometheus-stack-prometheus} + +OPERATOR_VM_NAME=${OPERATOR_VM_NAME:-gantry-benchmark-operator} +OPERATOR_VM_SIZE=${OPERATOR_VM_SIZE:-Standard_D32ds_v5} +OPERATOR_VM_ZONE=${OPERATOR_VM_ZONE:-1} +OPERATOR_OS_DISK_GB=${OPERATOR_OS_DISK_GB:-128} +OPERATOR_BUILD_DISK_GB=${OPERATOR_BUILD_DISK_GB:-512} +OPERATOR_BUILD_DISK_SKU=${OPERATOR_BUILD_DISK_SKU:-PremiumV2_LRS} +OPERATOR_BUILD_DISK_IOPS=${OPERATOR_BUILD_DISK_IOPS:-20000} +OPERATOR_BUILD_DISK_MBPS=${OPERATOR_BUILD_DISK_MBPS:-750} + +START_BENCHMARK=${START_BENCHMARK:-false} +DEPLOY_CONFIRM=${DEPLOY_CONFIRM:-} +DEPLOY_STATE_DIR=${DEPLOY_STATE_DIR:-$repo_root/tmp/$DEPLOYMENT_NAME} +KUBECONFIG=${DEPLOY_KUBECONFIG:-$DEPLOY_STATE_DIR/kubeconfig} + +BASELINE_PRIVATE_ENDPOINT_NAME=${BASELINE_PRIVATE_ENDPOINT_NAME:-${DEPLOYMENT_NAME}-baseline-acr-pe} +GANTRY_PRIVATE_ENDPOINT_NAME=${GANTRY_PRIVATE_ENDPOINT_NAME:-${DEPLOYMENT_NAME}-gantry-acr-pe} +PRIVATE_DNS_ZONE=privatelink.azurecr.io +PRIVATE_DNS_LINK_NAME=${PRIVATE_DNS_LINK_NAME:-${DEPLOYMENT_NAME}-acr-link} + +BASELINE_ACR_LOGIN_SERVER=${BASELINE_ACR_NAME}.azurecr.io +GANTRY_ACR_LOGIN_SERVER=${GANTRY_ACR_NAME}.azurecr.io +BASELINE_ACR_DATA_HOST=${BASELINE_ACR_NAME}.${AZURE_LOCATION}.data.azurecr.io +GANTRY_ACR_DATA_HOST=${GANTRY_ACR_NAME}.${AZURE_LOCATION}.data.azurecr.io + +[[ "$AKS_NODE_COUNT" =~ ^[1-9][0-9]*$ ]] || { echo "AKS_NODE_COUNT must be positive" >&2; exit 2; } +[[ "$BENCHMARK_NODE_COUNT" == "$AKS_NODE_COUNT" ]] || { + echo "BENCHMARK_NODE_COUNT must equal AKS_NODE_COUNT for this topology" >&2 + exit 2 +} +[[ "$BENCHMARK_IMAGE_LAYERS" =~ ^[1-9][0-9]*$ ]] || { echo "BENCHMARK_IMAGE_LAYERS must be positive" >&2; exit 2; } +((BENCHMARK_IMAGE_LAYERS <= BENCHMARK_IMAGE_SIZE_MIB)) || { + echo "BENCHMARK_IMAGE_LAYERS cannot exceed BENCHMARK_IMAGE_SIZE_MIB" >&2 + exit 2 +} +for acr_name in "$BASELINE_ACR_NAME" "$GANTRY_ACR_NAME"; do + [[ "$acr_name" =~ ^[a-z0-9]{5,50}$ ]] || { + echo "invalid ACR name $acr_name: use 5-50 lowercase alphanumeric characters" >&2 + exit 2 + } +done +[[ "$START_BENCHMARK" == true || "$START_BENCHMARK" == false ]] || { + echo "START_BENCHMARK must be true or false" >&2 + exit 2 +} +assert_default() { + local name=$1 + local actual=$2 + local expected=$3 + [[ "$actual" == "$expected" ]] || { + echo "$name=$actual is unsupported; the operator contract requires $expected" >&2 + exit 2 + } +} +assert_default GANTRY_NAMESPACE "$GANTRY_NAMESPACE" gantry-system +assert_default BENCHMARK_NAMESPACE "$BENCHMARK_NAMESPACE" gantry-benchmark +assert_default MONITORING_NAMESPACE "$MONITORING_NAMESPACE" monitoring +assert_default KPS_RELEASE "$KPS_RELEASE" kps +assert_default PROMETHEUS_SERVICE "$PROMETHEUS_SERVICE" kps-kube-prometheus-stack-prometheus + +log() { printf '[deploy] %s\n' "$*"; } + +require_command() { + command -v "$1" >/dev/null 2>&1 || { echo "required command not found: $1" >&2; exit 1; } +} + +print_plan() { + cat </dev/null || echo unknown) + +Azure + subscription: $AZURE_SUBSCRIPTION_ID + location: $AZURE_LOCATION + resource group: $AZURE_RESOURCE_GROUP + AKS: $AZURE_AKS_CLUSTER_NAME + node resource group: $AZURE_NODE_RESOURCE_GROUP + node pool: $AKS_NODE_POOL_NAME ($AKS_NODE_COUNT x $AKS_NODE_VM_SIZE) + node OS disk: ${AKS_NODE_OS_DISK_GB} GiB managed + Kubernetes: $AKS_KUBERNETES_VERSION + +Network + VNet: $VNET_NAME $VNET_CIDR + AKS subnet: $AKS_SUBNET_NAME $AKS_SUBNET_CIDR + pod CIDR: $POD_CIDR + service CIDR: $SERVICE_CIDR + PE subnet: $PRIVATE_ENDPOINT_SUBNET_NAME $PRIVATE_ENDPOINT_SUBNET_CIDR + operator subnet: $OPERATOR_SUBNET_NAME $OPERATOR_SUBNET_CIDR + +Registries + baseline: $BASELINE_ACR_LOGIN_SERVER + Gantry: $GANTRY_ACR_LOGIN_SERVER + access: Premium, dedicated data endpoint, Private Endpoint, public disabled at completion + +Benchmark + nodes: $BENCHMARK_NODE_COUNT + payload: ${BENCHMARK_IMAGE_SIZE_MIB} MiB in $BENCHMARK_IMAGE_LAYERS layers + monitoring: kube-prometheus-stack $KPS_CHART_VERSION with benchmark-only discovery + operator: $OPERATOR_VM_SIZE with ${OPERATOR_BUILD_DISK_GB} GiB $OPERATOR_BUILD_DISK_SKU + start benchmark: $START_BENCHMARK +PLAN +} + +if [[ "$action" == plan ]]; then + print_plan + exit 0 +fi + +for command in az jq kubectl helm podman git make sha256sum; do + require_command "$command" +done + +az account set --subscription "$AZURE_SUBSCRIPTION_ID" + +if [[ "$action" == status ]]; then + az group show -g "$AZURE_RESOURCE_GROUP" --query '{name:name,location:location,state:properties.provisioningState}' -o json + az aks show -g "$AZURE_RESOURCE_GROUP" -n "$AZURE_AKS_CLUSTER_NAME" \ + --query '{state:provisioningState,version:kubernetesVersion,nodeResourceGroup:nodeResourceGroup}' -o json + az acr list -g "$AZURE_RESOURCE_GROUP" \ + --query '[].{name:name,publicNetworkAccess:publicNetworkAccess,dataEndpointEnabled:dataEndpointEnabled}' -o json + mkdir -p "$(dirname "$KUBECONFIG")" + az aks get-credentials -g "$AZURE_RESOURCE_GROUP" -n "$AZURE_AKS_CLUSTER_NAME" \ + --admin --file "$KUBECONFIG" --overwrite-existing --only-show-errors + chmod 0600 "$KUBECONFIG" + export KUBECONFIG + kubectl get nodes -o json | jq '{total:(.items|length),ready:([.items[]|select(any(.status.conditions[];.type=="Ready" and .status=="True"))]|length),unschedulable:([.items[]|select(.spec.unschedulable==true)]|length)}' + kubectl get daemonset -A -o json | jq '[.items[]|select(.metadata.name|test("gantry|benchmark"))|{namespace:.metadata.namespace,name:.metadata.name,desired:.status.desiredNumberScheduled,ready:.status.numberReady}]' + exit 0 +fi + +[[ "$DEPLOY_CONFIRM" == "$AZURE_RESOURCE_GROUP" ]] || { + echo "set DEPLOY_CONFIRM=$AZURE_RESOURCE_GROUP to authorize deployment" >&2 + exit 2 +} + +[[ -z "$(git -C "$repo_root" status --porcelain)" ]] || { + echo "deployment requires a clean Git worktree" >&2 + exit 1 +} + +source_revision=$(git -C "$repo_root" rev-parse HEAD) +source_short=$(git -C "$repo_root" rev-parse --short=12 HEAD) + +mkdir -p "$DEPLOY_STATE_DIR" +chmod 0700 "$DEPLOY_STATE_DIR" + +public_restore_needed=false +set_acrs_private() { + for acr in "$BASELINE_ACR_NAME" "$GANTRY_ACR_NAME"; do + if az acr show -g "$AZURE_RESOURCE_GROUP" -n "$acr" --output none >/dev/null 2>&1; then + az acr update -g "$AZURE_RESOURCE_GROUP" -n "$acr" \ + --data-endpoint-enabled true --public-network-enabled false \ + --only-show-errors -o none + fi + done +} + +restore_private_access() { + local status=$? + if [[ "$public_restore_needed" == true ]]; then + set_acrs_private || true + fi + exit "$status" +} +trap restore_private_access EXIT INT TERM + +assert_equal() { + local description=$1 + local actual=$2 + local expected=$3 + [[ "$actual" == "$expected" ]] || { + echo "$description is $actual, want $expected" >&2 + exit 1 + } +} + +guard_active_benchmark() { + if ! az aks show -g "$AZURE_RESOURCE_GROUP" -n "$AZURE_AKS_CLUSTER_NAME" --output none 2>/dev/null; then + return + fi + + mkdir -p "$(dirname "$KUBECONFIG")" + az aks get-credentials -g "$AZURE_RESOURCE_GROUP" -n "$AZURE_AKS_CLUSTER_NAME" \ + --admin --file "$KUBECONFIG" --overwrite-existing --only-show-errors + chmod 0600 "$KUBECONFIG" + export KUBECONFIG + if kubectl -n "$BENCHMARK_NAMESPACE" get configmap gantry-benchmark-state >/dev/null 2>&1 || + kubectl -n "$GANTRY_NAMESPACE" get configmap gantry-benchmark-lock >/dev/null 2>&1; then + echo "an active benchmark state or lock exists; finish or disable it before deployment" >&2 + exit 1 + fi +} + +ensure_group() { + if [[ $(az group exists -n "$AZURE_RESOURCE_GROUP") == false ]]; then + log "creating resource group $AZURE_RESOURCE_GROUP" + az group create -n "$AZURE_RESOURCE_GROUP" -l "$AZURE_LOCATION" --only-show-errors -o none + fi + assert_equal "resource group location" \ + "$(az group show -n "$AZURE_RESOURCE_GROUP" --query location -o tsv)" "$AZURE_LOCATION" +} + +ensure_vnet() { + if ! az network vnet show -g "$AZURE_RESOURCE_GROUP" -n "$VNET_NAME" --output none 2>/dev/null; then + log "creating VNet $VNET_NAME" + az network vnet create -g "$AZURE_RESOURCE_GROUP" -n "$VNET_NAME" -l "$AZURE_LOCATION" \ + --address-prefixes "$VNET_CIDR" --subnet-name "$AKS_SUBNET_NAME" \ + --subnet-prefixes "$AKS_SUBNET_CIDR" --only-show-errors -o none + fi + local actual_prefix + actual_prefix=$(az network vnet show -g "$AZURE_RESOURCE_GROUP" -n "$VNET_NAME" --query 'addressSpace.addressPrefixes[0]' -o tsv) + assert_equal "VNet prefix" "$actual_prefix" "$VNET_CIDR" + + if ! az network vnet subnet show -g "$AZURE_RESOURCE_GROUP" --vnet-name "$VNET_NAME" -n "$AKS_SUBNET_NAME" --output none 2>/dev/null; then + az network vnet subnet create -g "$AZURE_RESOURCE_GROUP" --vnet-name "$VNET_NAME" \ + -n "$AKS_SUBNET_NAME" --address-prefixes "$AKS_SUBNET_CIDR" --only-show-errors -o none + fi + assert_equal "AKS subnet prefix" \ + "$(az network vnet subnet show -g "$AZURE_RESOURCE_GROUP" --vnet-name "$VNET_NAME" -n "$AKS_SUBNET_NAME" --query addressPrefix -o tsv)" \ + "$AKS_SUBNET_CIDR" + + if ! az network vnet subnet show -g "$AZURE_RESOURCE_GROUP" --vnet-name "$VNET_NAME" -n "$PRIVATE_ENDPOINT_SUBNET_NAME" --output none 2>/dev/null; then + az network vnet subnet create -g "$AZURE_RESOURCE_GROUP" --vnet-name "$VNET_NAME" \ + -n "$PRIVATE_ENDPOINT_SUBNET_NAME" --address-prefixes "$PRIVATE_ENDPOINT_SUBNET_CIDR" \ + --disable-private-endpoint-network-policies true --only-show-errors -o none + fi + assert_equal "Private Endpoint subnet prefix" \ + "$(az network vnet subnet show -g "$AZURE_RESOURCE_GROUP" --vnet-name "$VNET_NAME" -n "$PRIVATE_ENDPOINT_SUBNET_NAME" --query addressPrefix -o tsv)" \ + "$PRIVATE_ENDPOINT_SUBNET_CIDR" + + if ! az network vnet subnet show -g "$AZURE_RESOURCE_GROUP" --vnet-name "$VNET_NAME" -n "$OPERATOR_SUBNET_NAME" --output none 2>/dev/null; then + az network vnet subnet create -g "$AZURE_RESOURCE_GROUP" --vnet-name "$VNET_NAME" \ + -n "$OPERATOR_SUBNET_NAME" --address-prefixes "$OPERATOR_SUBNET_CIDR" \ + --only-show-errors -o none + fi + assert_equal "operator subnet prefix" \ + "$(az network vnet subnet show -g "$AZURE_RESOURCE_GROUP" --vnet-name "$VNET_NAME" -n "$OPERATOR_SUBNET_NAME" --query addressPrefix -o tsv)" \ + "$OPERATOR_SUBNET_CIDR" +} + +ensure_acr() { + local name=$1 + if ! az acr show -g "$AZURE_RESOURCE_GROUP" -n "$name" --output none 2>/dev/null; then + log "creating Premium ACR $name" + az acr create -g "$AZURE_RESOURCE_GROUP" -n "$name" -l "$AZURE_LOCATION" \ + --sku Premium --public-network-enabled false --only-show-errors -o none + fi + assert_equal "$name SKU" "$(az acr show -g "$AZURE_RESOURCE_GROUP" -n "$name" --query sku.name -o tsv)" Premium + assert_equal "$name location" "$(az acr show -g "$AZURE_RESOURCE_GROUP" -n "$name" --query location -o tsv)" "$AZURE_LOCATION" + az acr update -g "$AZURE_RESOURCE_GROUP" -n "$name" \ + --data-endpoint-enabled true --only-show-errors -o none +} + +acr_image_digest() { + local registry=$1 + local image=$2 + az acr repository show -n "$registry" --image "$image" --query digest -o tsv 2>/dev/null || true +} + +build_branch_images() { + log "building branch artifacts from $source_revision" + + SOURCE_IMAGE=$GANTRY_ACR_LOGIN_SERVER/gantry-benchmark-source:$source_revision + GANTRY_IMAGE_TAG=$GANTRY_ACR_LOGIN_SERVER/gantry:benchmark-$source_short + BASELINE_PROBE_TAG=$BASELINE_ACR_LOGIN_SERVER/gantry-deploy-probe:$source_revision + local current_baseline_acr_id current_gantry_acr_id + current_baseline_acr_id=$(az acr show -g "$AZURE_RESOURCE_GROUP" -n "$BASELINE_ACR_NAME" --query id -o tsv) + current_gantry_acr_id=$(az acr show -g "$AZURE_RESOURCE_GROUP" -n "$GANTRY_ACR_NAME" --query id -o tsv) + + local image_state=$DEPLOY_STATE_DIR/images.env + if [[ -f "$image_state" ]]; then + local recorded_revision recorded_baseline_acr_id recorded_gantry_acr_id + recorded_revision=$(sed -n "s/^SOURCE_REVISION='\([^']*\)'$/\1/p" "$image_state") + recorded_baseline_acr_id=$(sed -n "s/^BASELINE_ACR_RESOURCE_ID='\([^']*\)'$/\1/p" "$image_state") + recorded_gantry_acr_id=$(sed -n "s/^GANTRY_ACR_RESOURCE_ID='\([^']*\)'$/\1/p" "$image_state") + if [[ "$recorded_revision" == "$source_revision" && + "$recorded_baseline_acr_id" == "$current_baseline_acr_id" && + "$recorded_gantry_acr_id" == "$current_gantry_acr_id" ]]; then + # shellcheck source=/dev/null + . "$image_state" + [[ "$SOURCE_IMAGE" == "$GANTRY_ACR_LOGIN_SERVER/gantry-benchmark-source:$source_revision" ]] || { + echo "recorded source image does not match this deployment" >&2 + exit 1 + } + [[ "$GANTRY_IMAGE" == "$GANTRY_ACR_LOGIN_SERVER/gantry@sha256:"* ]] || { + echo "recorded Gantry image is not an immutable deployment reference" >&2 + exit 1 + } + [[ "$BASELINE_PROBE_IMAGE" == "$BASELINE_ACR_LOGIN_SERVER/gantry-deploy-probe@sha256:"* ]] || { + echo "recorded baseline probe image is not an immutable deployment reference" >&2 + exit 1 + } + log "reusing locally recorded immutable branch artifacts" + return + fi + fi + + local source_digest gantry_digest baseline_probe_digest + source_digest=$(acr_image_digest "$GANTRY_ACR_NAME" "gantry-benchmark-source:$source_revision") + gantry_digest=$(acr_image_digest "$GANTRY_ACR_NAME" "gantry:benchmark-$source_short") + baseline_probe_digest=$(acr_image_digest "$BASELINE_ACR_NAME" "gantry-deploy-probe:$source_revision") + + if [[ -n "$source_digest" && -n "$gantry_digest" && -n "$baseline_probe_digest" ]]; then + log "reusing immutable branch artifacts already present in ACR" + GANTRY_IMAGE=$GANTRY_ACR_LOGIN_SERVER/gantry@$gantry_digest + BASELINE_PROBE_IMAGE=$BASELINE_ACR_LOGIN_SERVER/gantry-deploy-probe@$baseline_probe_digest + cat >"$image_state" </dev/null + unset token + + if [[ -z "$source_digest" ]]; then + podman build --isolation chroot --build-arg "SOURCE_REVISION=$source_revision" \ + -t "$SOURCE_IMAGE" -f "$repo_root/images/gantry-benchmark-source/Containerfile" "$repo_root" + podman push "$SOURCE_IMAGE" >/dev/null + source_digest=$(acr_image_digest "$GANTRY_ACR_NAME" "gantry-benchmark-source:$source_revision") + else + podman pull "$SOURCE_IMAGE" >/dev/null + fi + + if [[ -z "$gantry_digest" ]]; then + podman build --isolation chroot \ + --build-arg "VERSION=benchmark-$source_short" \ + --build-arg "GIT_COMMIT=$source_revision" \ + -t "$GANTRY_IMAGE_TAG" -f "$repo_root/images/gantry/Containerfile" "$repo_root" + local digest_file=$DEPLOY_STATE_DIR/gantry.digest + podman push --digestfile "$digest_file" "$GANTRY_IMAGE_TAG" >/dev/null + gantry_digest=$(tr -d '[:space:]' <"$digest_file") + fi + podman logout "$GANTRY_ACR_LOGIN_SERVER" >/dev/null + + if [[ -z "$baseline_probe_digest" ]]; then + token=$(az acr login --name "$BASELINE_ACR_NAME" --expose-token --query accessToken -o tsv) + printf '%s' "$token" | podman login "$BASELINE_ACR_LOGIN_SERVER" \ + --username 00000000-0000-0000-0000-000000000000 --password-stdin >/dev/null + unset token + podman pull mcr.microsoft.com/cbl-mariner/busybox:2.0 >/dev/null + podman tag mcr.microsoft.com/cbl-mariner/busybox:2.0 "$BASELINE_PROBE_TAG" + local probe_digest_file=$DEPLOY_STATE_DIR/baseline-probe.digest + podman push --digestfile "$probe_digest_file" "$BASELINE_PROBE_TAG" >/dev/null + baseline_probe_digest=$(tr -d '[:space:]' <"$probe_digest_file") + podman logout "$BASELINE_ACR_LOGIN_SERVER" >/dev/null + fi + + GANTRY_IMAGE=$GANTRY_ACR_LOGIN_SERVER/gantry@$gantry_digest + BASELINE_PROBE_IMAGE=$BASELINE_ACR_LOGIN_SERVER/gantry-deploy-probe@$baseline_probe_digest + cat >"$image_state" </dev/null; then + log "creating AKS cluster $AZURE_AKS_CLUSTER_NAME" + az aks create -g "$AZURE_RESOURCE_GROUP" -n "$AZURE_AKS_CLUSTER_NAME" -l "$AZURE_LOCATION" \ + --tier standard --enable-managed-identity --node-resource-group "$AZURE_NODE_RESOURCE_GROUP" \ + --nodepool-name "$AKS_NODE_POOL_NAME" --node-count "$AKS_NODE_COUNT" \ + --node-vm-size "$AKS_NODE_VM_SIZE" --node-osdisk-type Managed \ + --node-osdisk-size "$AKS_NODE_OS_DISK_GB" --max-pods "$AKS_MAX_PODS" \ + --os-sku Ubuntu --network-plugin azure --network-plugin-mode overlay \ + --network-dataplane azure --pod-cidr "$POD_CIDR" --service-cidr "$SERVICE_CIDR" \ + --dns-service-ip "$DNS_SERVICE_IP" --vnet-subnet-id "$subnet_id" \ + --load-balancer-sku standard --outbound-type loadBalancer \ + --kubernetes-version "$AKS_KUBERNETES_VERSION" --no-ssh-key --only-show-errors -o none + fi + + az aks wait -g "$AZURE_RESOURCE_GROUP" -n "$AZURE_AKS_CLUSTER_NAME" \ + --created --interval 30 --timeout 7200 + + local cluster_json pool_json + cluster_json=$(az aks show -g "$AZURE_RESOURCE_GROUP" -n "$AZURE_AKS_CLUSTER_NAME" -o json) + assert_equal "AKS location" "$(jq -r .location <<<"$cluster_json")" "$AZURE_LOCATION" + assert_equal "AKS Kubernetes version" "$(jq -r .kubernetesVersion <<<"$cluster_json")" "$AKS_KUBERNETES_VERSION" + assert_equal "AKS pod CIDR" "$(jq -r .networkProfile.podCidr <<<"$cluster_json")" "$POD_CIDR" + assert_equal "AKS service CIDR" "$(jq -r .networkProfile.serviceCidr <<<"$cluster_json")" "$SERVICE_CIDR" + assert_equal "AKS node resource group" "$(jq -r .nodeResourceGroup <<<"$cluster_json")" "$AZURE_NODE_RESOURCE_GROUP" + + pool_json=$(az aks nodepool show -g "$AZURE_RESOURCE_GROUP" --cluster-name "$AZURE_AKS_CLUSTER_NAME" \ + -n "$AKS_NODE_POOL_NAME" -o json) + assert_equal "AKS node count" "$(jq -r .count <<<"$pool_json")" "$AKS_NODE_COUNT" + assert_equal "AKS node VM size" "$(jq -r .vmSize <<<"$pool_json")" "$AKS_NODE_VM_SIZE" + assert_equal "AKS max pods" "$(jq -r .maxPods <<<"$pool_json")" "$AKS_MAX_PODS" + assert_equal "AKS node OS disk" "$(jq -r .osDiskSizeGb <<<"$pool_json")" "$AKS_NODE_OS_DISK_GB" + assert_equal "AKS node OS SKU" "$(jq -r .osSku <<<"$pool_json")" Ubuntu + assert_equal "AKS node-pool mode" "$(jq -r .mode <<<"$pool_json")" System + assert_equal "AKS node subnet" "$(jq -r .vnetSubnetId <<<"$pool_json")" "$subnet_id" +} + +ensure_role() { + local principal=$1 + local role=$2 + local scope=$3 + if [[ $(az role assignment list --assignee-object-id "$principal" --scope "$scope" --role "$role" --query 'length(@)' -o tsv) == 0 ]]; then + az role assignment create --assignee-object-id "$principal" \ + --assignee-principal-type ServicePrincipal --role "$role" --scope "$scope" \ + --only-show-errors -o none + fi +} + +ensure_diagnostics() { + local law_id aks_id baseline_id gantry_id + if ! az monitor log-analytics workspace show -g "$AZURE_RESOURCE_GROUP" -n "$AZURE_LOG_ANALYTICS_WORKSPACE_NAME" --output none 2>/dev/null; then + az monitor log-analytics workspace create -g "$AZURE_RESOURCE_GROUP" \ + -n "$AZURE_LOG_ANALYTICS_WORKSPACE_NAME" -l "$AZURE_LOCATION" \ + --retention-time 30 --only-show-errors -o none + fi + assert_equal "Log Analytics location" \ + "$(az monitor log-analytics workspace show -g "$AZURE_RESOURCE_GROUP" -n "$AZURE_LOG_ANALYTICS_WORKSPACE_NAME" --query location -o tsv)" \ + "$AZURE_LOCATION" + law_id=$(az monitor log-analytics workspace show -g "$AZURE_RESOURCE_GROUP" \ + -n "$AZURE_LOG_ANALYTICS_WORKSPACE_NAME" --query id -o tsv) + aks_id=$(az aks show -g "$AZURE_RESOURCE_GROUP" -n "$AZURE_AKS_CLUSTER_NAME" --query id -o tsv) + baseline_id=$(az acr show -g "$AZURE_RESOURCE_GROUP" -n "$BASELINE_ACR_NAME" --query id -o tsv) + gantry_id=$(az acr show -g "$AZURE_RESOURCE_GROUP" -n "$GANTRY_ACR_NAME" --query id -o tsv) + + az monitor diagnostic-settings create --name "${DEPLOYMENT_NAME}-baseline-acr-diag" \ + --resource "$baseline_id" --workspace "$law_id" --export-to-resource-specific true \ + --logs '[{"category":"ContainerRegistryRepositoryEvents","enabled":true},{"category":"ContainerRegistryLoginEvents","enabled":true}]' \ + --metrics '[{"category":"AllMetrics","enabled":true}]' --only-show-errors -o none + az monitor diagnostic-settings create --name "${DEPLOYMENT_NAME}-gantry-acr-diag" \ + --resource "$gantry_id" --workspace "$law_id" --export-to-resource-specific true \ + --logs '[{"category":"ContainerRegistryRepositoryEvents","enabled":true},{"category":"ContainerRegistryLoginEvents","enabled":true}]' \ + --metrics '[{"category":"AllMetrics","enabled":true}]' --only-show-errors -o none + az monitor diagnostic-settings create --name "${DEPLOYMENT_NAME}-aks-diag" \ + --resource "$aks_id" --workspace "$law_id" --export-to-resource-specific true \ + --logs '[{"category":"kube-audit-admin","enabled":true},{"category":"kube-apiserver","enabled":true},{"category":"kube-scheduler","enabled":true}]' \ + --metrics '[{"category":"AllMetrics","enabled":true}]' --only-show-errors -o none +} + +ensure_private_endpoint() { + local name=$1 + local acr_id=$2 + local connection_name=$3 + local subnet_id zone_id + subnet_id=$(az network vnet subnet show -g "$AZURE_RESOURCE_GROUP" --vnet-name "$VNET_NAME" \ + -n "$PRIVATE_ENDPOINT_SUBNET_NAME" --query id -o tsv) + zone_id=$(az network private-dns zone show -g "$AZURE_RESOURCE_GROUP" -n "$PRIVATE_DNS_ZONE" --query id -o tsv) + + if ! az network private-endpoint show -g "$AZURE_RESOURCE_GROUP" -n "$name" --output none 2>/dev/null; then + az network private-endpoint create -g "$AZURE_RESOURCE_GROUP" -n "$name" -l "$AZURE_LOCATION" \ + --subnet "$subnet_id" --private-connection-resource-id "$acr_id" \ + --group-ids registry --connection-name "$connection_name" --only-show-errors -o none + fi + if ! az network private-endpoint dns-zone-group show -g "$AZURE_RESOURCE_GROUP" \ + --endpoint-name "$name" -n acr --output none 2>/dev/null; then + az network private-endpoint dns-zone-group create -g "$AZURE_RESOURCE_GROUP" \ + --endpoint-name "$name" -n acr --private-dns-zone "$zone_id" \ + --zone-name "$PRIVATE_DNS_ZONE" --only-show-errors -o none + fi + assert_equal "$name connection state" \ + "$(az network private-endpoint show -g "$AZURE_RESOURCE_GROUP" -n "$name" --query 'privateLinkServiceConnections[0].privateLinkServiceConnectionState.status' -o tsv)" \ + Approved + assert_equal "$name target resource" \ + "$(az network private-endpoint show -g "$AZURE_RESOURCE_GROUP" -n "$name" --query 'privateLinkServiceConnections[0].privateLinkServiceId' -o tsv)" \ + "$acr_id" + assert_equal "$name subnet" \ + "$(az network private-endpoint show -g "$AZURE_RESOURCE_GROUP" -n "$name" --query subnet.id -o tsv)" \ + "$subnet_id" +} + +ensure_private_network() { + if ! az network private-dns zone show -g "$AZURE_RESOURCE_GROUP" -n "$PRIVATE_DNS_ZONE" --output none 2>/dev/null; then + az network private-dns zone create -g "$AZURE_RESOURCE_GROUP" -n "$PRIVATE_DNS_ZONE" --only-show-errors -o none + fi + local vnet_id + vnet_id=$(az network vnet show -g "$AZURE_RESOURCE_GROUP" -n "$VNET_NAME" --query id -o tsv) + if ! az network private-dns link vnet show -g "$AZURE_RESOURCE_GROUP" -z "$PRIVATE_DNS_ZONE" \ + -n "$PRIVATE_DNS_LINK_NAME" --output none 2>/dev/null; then + az network private-dns link vnet create -g "$AZURE_RESOURCE_GROUP" -z "$PRIVATE_DNS_ZONE" \ + -n "$PRIVATE_DNS_LINK_NAME" -v "$vnet_id" -e false --only-show-errors -o none + fi + assert_equal "private DNS VNet link" \ + "$(az network private-dns link vnet show -g "$AZURE_RESOURCE_GROUP" -z "$PRIVATE_DNS_ZONE" -n "$PRIVATE_DNS_LINK_NAME" --query virtualNetwork.id -o tsv)" \ + "$vnet_id" + + local baseline_id gantry_id + baseline_id=$(az acr show -g "$AZURE_RESOURCE_GROUP" -n "$BASELINE_ACR_NAME" --query id -o tsv) + gantry_id=$(az acr show -g "$AZURE_RESOURCE_GROUP" -n "$GANTRY_ACR_NAME" --query id -o tsv) + ensure_private_endpoint "$BASELINE_PRIVATE_ENDPOINT_NAME" "$baseline_id" "${DEPLOYMENT_NAME}-baseline-acr" + ensure_private_endpoint "$GANTRY_PRIVATE_ENDPOINT_NAME" "$gantry_id" "${DEPLOYMENT_NAME}-gantry-acr" +} + +wait_for_nodes() { + mkdir -p "$(dirname "$KUBECONFIG")" + rm -f "$KUBECONFIG" + az aks get-credentials -g "$AZURE_RESOURCE_GROUP" -n "$AZURE_AKS_CLUSTER_NAME" \ + --admin --file "$KUBECONFIG" --overwrite-existing --only-show-errors + chmod 0600 "$KUBECONFIG" + export KUBECONFIG + + if kubectl -n "$BENCHMARK_NAMESPACE" get configmap gantry-benchmark-state >/dev/null 2>&1 || + kubectl -n "$GANTRY_NAMESPACE" get configmap gantry-benchmark-lock >/dev/null 2>&1; then + echo "an active benchmark state or lock exists; finish or disable it before deployment" >&2 + exit 1 + fi + + local attempt total ready unschedulable + for attempt in $(seq 1 120); do + local nodes + nodes=$(kubectl get nodes -o json) + total=$(jq '.items|length' <<<"$nodes") + ready=$(jq '[.items[]|select(any(.status.conditions[];.type=="Ready" and .status=="True"))]|length' <<<"$nodes") + unschedulable=$(jq '[.items[]|select(.spec.unschedulable==true)]|length' <<<"$nodes") + if [[ "$total" == "$AKS_NODE_COUNT" && "$ready" == "$AKS_NODE_COUNT" && "$unschedulable" == 0 ]]; then + log "AKS nodes ready: $ready/$total" + return + fi + log "waiting for nodes: total=$total ready=$ready unschedulable=$unschedulable" + sleep 30 + done + echo "AKS did not reach $AKS_NODE_COUNT Ready schedulable nodes" >&2 + exit 1 +} + +install_monitoring() { + export KUBECONFIG + local values=$DEPLOY_STATE_DIR/kps-values.yaml + cat >"$values" </dev/null || true) + if [[ -n "$value" ]]; then + printf '%s' "$value" + return + fi + sleep 10 + done + echo "private DNS record $record did not appear" >&2 + exit 1 +} + +install_node_configuration() { + export KUBECONFIG + kubectl create namespace "$GANTRY_NAMESPACE" --dry-run=client -o yaml | kubectl apply -f - + kubectl apply -f "$repo_root/hack/gantry-benchmark/manifests/containerd.yaml" + kubectl -n "$GANTRY_NAMESPACE" rollout status \ + daemonset/gantry-benchmark-containerd-config --timeout=45m + + local baseline_login_ip baseline_data_ip gantry_login_ip gantry_data_ip + baseline_login_ip=$(private_dns_ip "$BASELINE_ACR_NAME") + baseline_data_ip=$(private_dns_ip "$BASELINE_ACR_NAME.$AZURE_LOCATION.data") + gantry_login_ip=$(private_dns_ip "$GANTRY_ACR_NAME") + gantry_data_ip=$(private_dns_ip "$GANTRY_ACR_NAME.$AZURE_LOCATION.data") + + local guard=$DEPLOY_STATE_DIR/acr-private-dns-guard.yaml + cat >"$guard" <"\$temp" + cat >>"\$temp" <<'HOSTS' + # BEGIN GANTRY BENCHMARK ACR PRIVATE DNS + $baseline_login_ip $BASELINE_ACR_LOGIN_SERVER + $baseline_data_ip $BASELINE_ACR_DATA_HOST + $gantry_login_ip $GANTRY_ACR_LOGIN_SERVER + $gantry_data_ip $GANTRY_ACR_DATA_HOST + # END GANTRY BENCHMARK ACR PRIVATE DNS + HOSTS + cat "\$temp" >"\$hosts" + rm "\$temp" + resolvectl flush-caches + exec sleep 2147483647 + readinessProbe: + exec: + command: + - chroot + - /host + - sh + - -c + - | + set -eu + check() { getent ahostsv4 "\$1" | awk '{print \$1}' | grep -Fxq "\$2"; } + check $BASELINE_ACR_LOGIN_SERVER $baseline_login_ip + check $BASELINE_ACR_DATA_HOST $baseline_data_ip + check $GANTRY_ACR_LOGIN_SERVER $gantry_login_ip + check $GANTRY_ACR_DATA_HOST $gantry_data_ip + initialDelaySeconds: 2 + timeoutSeconds: 10 + periodSeconds: 15 + failureThreshold: 3 + resources: + requests: {cpu: 1m, memory: 4Mi} + limits: {cpu: 20m, memory: 16Mi} + securityContext: + privileged: true + runAsUser: 0 + volumeMounts: + - name: host + mountPath: /host + volumes: + - name: host + hostPath: + path: / + type: Directory +GUARD + kubectl apply -f "$guard" + kubectl -n "$GANTRY_NAMESPACE" rollout status daemonset/gantry-acr-private-dns-guard --timeout=30m +} + +verify_private_baseline_pull() { + export KUBECONFIG + local manifest=$DEPLOY_STATE_DIR/baseline-private-pull-probe.yaml + cat >"$manifest" <&2 + return 1 + fi + kubectl -n "$GANTRY_NAMESPACE" delete daemonset gantry-baseline-acr-pull-probe --wait=true +} + +deploy_gantry() { + export KUBECONFIG + local rendered=$DEPLOY_STATE_DIR/gantry-rendered + rm -rf "$rendered" + GOTOOLCHAIN=auto go run "$repo_root/hack/cmd/render-manifests" \ + --templates-dir "$repo_root/deploy/gantry" --output-dir "$rendered" \ + --set "Namespace=$GANTRY_NAMESPACE" --set "Image=$GANTRY_IMAGE" + sed -i "s/registry\.example\.com/$GANTRY_ACR_LOGIN_SERVER/g" "$rendered/configmap.yaml" + + kubectl apply -f "$rendered/serviceaccount.yaml" + kubectl apply -f "$rendered/configmap.yaml" + kubectl apply -f "$rendered/node-config.yaml" + kubectl apply -f "$rendered/daemonset.yaml" + kubectl -n "$GANTRY_NAMESPACE" rollout status daemonset/gantry-containerd-config --timeout=30m + kubectl -n "$GANTRY_NAMESPACE" rollout status daemonset/gantry --timeout=45m +} + +provision_operator() { + export AZURE_SUBSCRIPTION_ID AZURE_RESOURCE_GROUP AZURE_AKS_CLUSTER_NAME + export BASELINE_ACR_NAME GANTRY_ACR_NAME AZURE_LOG_ANALYTICS_WORKSPACE_NAME + export OPERATOR_VNET_RESOURCE_GROUP=$AZURE_RESOURCE_GROUP OPERATOR_VNET_NAME=$VNET_NAME + export AZURE_LOCATION OPERATOR_VM_NAME OPERATOR_VM_SIZE OPERATOR_VM_ZONE + export OPERATOR_OS_DISK_GB OPERATOR_BUILD_DISK_GB OPERATOR_BUILD_DISK_SKU + export OPERATOR_BUILD_DISK_IOPS OPERATOR_BUILD_DISK_MBPS OPERATOR_SUBNET_NAME OPERATOR_SUBNET_CIDR + export BENCHMARK_SOURCE_IMAGE=$SOURCE_IMAGE BENCHMARK_SOURCE_REVISION=$source_revision + export BENCHMARK_NODE_COUNT BENCHMARK_IMAGE_SIZE_MIB BENCHMARK_IMAGE_LAYERS + export BENCHMARK_AZURE_TELEMETRY=true BENCHMARK_MINIMUM_BYTE_REDUCTION BENCHMARK_MAXIMUM_LATENCY_RATIO + export START_BENCHMARK=false + AZURE_BASELINE_ACR_PRIVATE_ENDPOINT_RESOURCE_ID=$(az network private-endpoint show \ + -g "$AZURE_RESOURCE_GROUP" -n "$BASELINE_PRIVATE_ENDPOINT_NAME" --query id -o tsv) + AZURE_GANTRY_ACR_PRIVATE_ENDPOINT_RESOURCE_ID=$(az network private-endpoint show \ + -g "$AZURE_RESOURCE_GROUP" -n "$GANTRY_PRIVATE_ENDPOINT_NAME" --query id -o tsv) + export AZURE_BASELINE_ACR_PRIVATE_ENDPOINT_RESOURCE_ID AZURE_GANTRY_ACR_PRIVATE_ENDPOINT_RESOURCE_ID + "$repo_root/hack/gantry-benchmark/operator-vm-provision.sh" + + assert_equal "operator VM size" \ + "$(az vm show -g "$AZURE_RESOURCE_GROUP" -n "$OPERATOR_VM_NAME" --query hardwareProfile.vmSize -o tsv)" \ + "$OPERATOR_VM_SIZE" + local public_ip_id + public_ip_id=$(az vm nic list -g "$AZURE_RESOURCE_GROUP" --vm-name "$OPERATOR_VM_NAME" \ + --query '[].ipConfigurations[].publicIPAddress.id | [0]' -o tsv) + [[ -z "$public_ip_id" ]] || { echo "operator VM unexpectedly has public IP resource $public_ip_id" >&2; exit 1; } + local build_disk_name + build_disk_name=${OPERATOR_BUILD_DISK_NAME:-${OPERATOR_VM_NAME}-build} + assert_equal "operator build disk size" \ + "$(az disk show -g "$AZURE_RESOURCE_GROUP" -n "$build_disk_name" --query diskSizeGb -o tsv)" \ + "$OPERATOR_BUILD_DISK_GB" + assert_equal "operator build disk SKU" \ + "$(az disk show -g "$AZURE_RESOURCE_GROUP" -n "$build_disk_name" --query sku.name -o tsv)" \ + "$OPERATOR_BUILD_DISK_SKU" +} + +verify_operator_private_pushes() { + log "verifying private operator pushes to both ACR data endpoints" + az vm run-command invoke -g "$AZURE_RESOURCE_GROUP" -n "$OPERATOR_VM_NAME" \ + --command-id RunShellScript --scripts "set -eu +podman pull mcr.microsoft.com/cbl-mariner/busybox:2.0 >/dev/null +for acr in '$BASELINE_ACR_NAME' '$GANTRY_ACR_NAME'; do + login=\"\${acr}.azurecr.io\" + token=\$(az acr login --name \"\$acr\" --expose-token --query accessToken -o tsv) + printf '%s' \"\$token\" | podman login \"\$login\" \\ + --username 00000000-0000-0000-0000-000000000000 --password-stdin >/dev/null + unset token + target=\"\$login/gantry-deploy-operator-probe:$source_revision\" + podman tag mcr.microsoft.com/cbl-mariner/busybox:2.0 \"\$target\" + podman push \"\$target\" >/dev/null + podman logout \"\$login\" >/dev/null + podman image rm \"\$target\" >/dev/null +done" --only-show-errors -o none +} + +guard_active_benchmark +ensure_group +ensure_vnet +ensure_acr "$BASELINE_ACR_NAME" +ensure_acr "$GANTRY_ACR_NAME" +build_branch_images +if [[ "$public_restore_needed" == true ]]; then + set_acrs_private + public_restore_needed=false +fi +ensure_private_network +ensure_aks +ensure_diagnostics + +kubelet_object_id=$(az aks show -g "$AZURE_RESOURCE_GROUP" -n "$AZURE_AKS_CLUSTER_NAME" \ + --query identityProfile.kubeletidentity.objectId -o tsv) +ensure_role "$kubelet_object_id" AcrPull \ + "$(az acr show -g "$AZURE_RESOURCE_GROUP" -n "$BASELINE_ACR_NAME" --query id -o tsv)" +ensure_role "$kubelet_object_id" AcrPull \ + "$(az acr show -g "$AZURE_RESOURCE_GROUP" -n "$GANTRY_ACR_NAME" --query id -o tsv)" + +wait_for_nodes +install_node_configuration +install_monitoring + +set_acrs_private + +verify_private_baseline_pull +deploy_gantry +provision_operator +verify_operator_private_pushes + +log "validating final deployment" +assert_equal "baseline ACR public access" \ + "$(az acr show -g "$AZURE_RESOURCE_GROUP" -n "$BASELINE_ACR_NAME" --query publicNetworkAccess -o tsv)" Disabled +assert_equal "Gantry ACR public access" \ + "$(az acr show -g "$AZURE_RESOURCE_GROUP" -n "$GANTRY_ACR_NAME" --query publicNetworkAccess -o tsv)" Disabled + +kubectl -n "$MONITORING_NAMESPACE" get endpoints "$PROMETHEUS_SERVICE" -o json | \ + jq -e '.subsets | any(.addresses | length > 0)' >/dev/null +for daemonset in gantry-benchmark-containerd-config gantry-acr-private-dns-guard gantry-containerd-config gantry; do + namespace=$GANTRY_NAMESPACE + desired=$(kubectl -n "$namespace" get daemonset "$daemonset" -o jsonpath='{.status.desiredNumberScheduled}') + ready=$(kubectl -n "$namespace" get daemonset "$daemonset" -o jsonpath='{.status.numberReady}') + assert_equal "$daemonset readiness" "$ready" "$desired" +done + +if [[ "$START_BENCHMARK" == true ]]; then + log "starting benchmark operator service" + az vm run-command invoke -g "$AZURE_RESOURCE_GROUP" -n "$OPERATOR_VM_NAME" \ + --command-id RunShellScript \ + --scripts 'systemctl reset-failed gantry-benchmark-operator.service; systemctl start --no-block gantry-benchmark-operator.service' \ + --only-show-errors -o none +fi + +trap - EXIT INT TERM +log "deployment complete" +print_plan +cat < Date: Wed, 5 Aug 2026 11:37:33 -0400 Subject: [PATCH 10/60] fix(gantry): open ACR before artifact lookup --- hack/gantry-benchmark/deploy.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/hack/gantry-benchmark/deploy.sh b/hack/gantry-benchmark/deploy.sh index 84a2ee9d9..2ed1ca493 100755 --- a/hack/gantry-benchmark/deploy.sh +++ b/hack/gantry-benchmark/deploy.sh @@ -388,6 +388,12 @@ build_branch_images() { fi fi + public_restore_needed=true + for registry in "$BASELINE_ACR_NAME" "$GANTRY_ACR_NAME"; do + az acr update -g "$AZURE_RESOURCE_GROUP" -n "$registry" \ + --public-network-enabled true --only-show-errors -o none + done + local source_digest gantry_digest baseline_probe_digest source_digest=$(acr_image_digest "$GANTRY_ACR_NAME" "gantry-benchmark-source:$source_revision") gantry_digest=$(acr_image_digest "$GANTRY_ACR_NAME" "gantry:benchmark-$source_short") @@ -409,12 +415,6 @@ IMAGES return fi - public_restore_needed=true - for registry in "$BASELINE_ACR_NAME" "$GANTRY_ACR_NAME"; do - az acr update -g "$AZURE_RESOURCE_GROUP" -n "$registry" \ - --public-network-enabled true --only-show-errors -o none - done - local token token=$(az acr login --name "$GANTRY_ACR_NAME" --expose-token --query accessToken -o tsv) printf '%s' "$token" | podman login "$GANTRY_ACR_LOGIN_SERVER" \ From ebabed01b9d9939312392d192aea291cd246fa58 Mon Sep 17 00:00:00 2001 From: Vaibhav Patel Date: Wed, 5 Aug 2026 11:45:06 -0400 Subject: [PATCH 11/60] fix(gantry): publish artifacts without ACR lookup --- hack/gantry-benchmark/deploy.sh | 80 ++++++++++----------------------- 1 file changed, 23 insertions(+), 57 deletions(-) diff --git a/hack/gantry-benchmark/deploy.sh b/hack/gantry-benchmark/deploy.sh index 2ed1ca493..5b0042594 100755 --- a/hack/gantry-benchmark/deploy.sh +++ b/hack/gantry-benchmark/deploy.sh @@ -344,12 +344,6 @@ ensure_acr() { --data-endpoint-enabled true --only-show-errors -o none } -acr_image_digest() { - local registry=$1 - local image=$2 - az acr repository show -n "$registry" --image "$image" --query digest -o tsv 2>/dev/null || true -} - build_branch_images() { log "building branch artifacts from $source_revision" @@ -394,65 +388,37 @@ build_branch_images() { --public-network-enabled true --only-show-errors -o none done - local source_digest gantry_digest baseline_probe_digest - source_digest=$(acr_image_digest "$GANTRY_ACR_NAME" "gantry-benchmark-source:$source_revision") - gantry_digest=$(acr_image_digest "$GANTRY_ACR_NAME" "gantry:benchmark-$source_short") - baseline_probe_digest=$(acr_image_digest "$BASELINE_ACR_NAME" "gantry-deploy-probe:$source_revision") - - if [[ -n "$source_digest" && -n "$gantry_digest" && -n "$baseline_probe_digest" ]]; then - log "reusing immutable branch artifacts already present in ACR" - GANTRY_IMAGE=$GANTRY_ACR_LOGIN_SERVER/gantry@$gantry_digest - BASELINE_PROBE_IMAGE=$BASELINE_ACR_LOGIN_SERVER/gantry-deploy-probe@$baseline_probe_digest - cat >"$image_state" </dev/null unset token - if [[ -z "$source_digest" ]]; then - podman build --isolation chroot --build-arg "SOURCE_REVISION=$source_revision" \ - -t "$SOURCE_IMAGE" -f "$repo_root/images/gantry-benchmark-source/Containerfile" "$repo_root" - podman push "$SOURCE_IMAGE" >/dev/null - source_digest=$(acr_image_digest "$GANTRY_ACR_NAME" "gantry-benchmark-source:$source_revision") - else - podman pull "$SOURCE_IMAGE" >/dev/null - fi - - if [[ -z "$gantry_digest" ]]; then - podman build --isolation chroot \ - --build-arg "VERSION=benchmark-$source_short" \ - --build-arg "GIT_COMMIT=$source_revision" \ - -t "$GANTRY_IMAGE_TAG" -f "$repo_root/images/gantry/Containerfile" "$repo_root" - local digest_file=$DEPLOY_STATE_DIR/gantry.digest - podman push --digestfile "$digest_file" "$GANTRY_IMAGE_TAG" >/dev/null - gantry_digest=$(tr -d '[:space:]' <"$digest_file") - fi + podman build --isolation chroot --build-arg "SOURCE_REVISION=$source_revision" \ + -t "$SOURCE_IMAGE" -f "$repo_root/images/gantry-benchmark-source/Containerfile" "$repo_root" + podman push "$SOURCE_IMAGE" >/dev/null + + podman build --isolation chroot \ + --build-arg "VERSION=benchmark-$source_short" \ + --build-arg "GIT_COMMIT=$source_revision" \ + -t "$GANTRY_IMAGE_TAG" -f "$repo_root/images/gantry/Containerfile" "$repo_root" + local digest_file=$DEPLOY_STATE_DIR/gantry.digest + podman push --digestfile "$digest_file" "$GANTRY_IMAGE_TAG" >/dev/null + local gantry_digest + gantry_digest=$(tr -d '[:space:]' <"$digest_file") podman logout "$GANTRY_ACR_LOGIN_SERVER" >/dev/null - if [[ -z "$baseline_probe_digest" ]]; then - token=$(az acr login --name "$BASELINE_ACR_NAME" --expose-token --query accessToken -o tsv) - printf '%s' "$token" | podman login "$BASELINE_ACR_LOGIN_SERVER" \ - --username 00000000-0000-0000-0000-000000000000 --password-stdin >/dev/null - unset token - podman pull mcr.microsoft.com/cbl-mariner/busybox:2.0 >/dev/null - podman tag mcr.microsoft.com/cbl-mariner/busybox:2.0 "$BASELINE_PROBE_TAG" - local probe_digest_file=$DEPLOY_STATE_DIR/baseline-probe.digest - podman push --digestfile "$probe_digest_file" "$BASELINE_PROBE_TAG" >/dev/null - baseline_probe_digest=$(tr -d '[:space:]' <"$probe_digest_file") - podman logout "$BASELINE_ACR_LOGIN_SERVER" >/dev/null - fi + token=$(az acr login --name "$BASELINE_ACR_NAME" --expose-token --query accessToken -o tsv) + printf '%s' "$token" | podman login "$BASELINE_ACR_LOGIN_SERVER" \ + --username 00000000-0000-0000-0000-000000000000 --password-stdin >/dev/null + unset token + podman pull mcr.microsoft.com/cbl-mariner/busybox:2.0 >/dev/null + podman tag mcr.microsoft.com/cbl-mariner/busybox:2.0 "$BASELINE_PROBE_TAG" + local probe_digest_file=$DEPLOY_STATE_DIR/baseline-probe.digest + podman push --digestfile "$probe_digest_file" "$BASELINE_PROBE_TAG" >/dev/null + local baseline_probe_digest + baseline_probe_digest=$(tr -d '[:space:]' <"$probe_digest_file") + podman logout "$BASELINE_ACR_LOGIN_SERVER" >/dev/null GANTRY_IMAGE=$GANTRY_ACR_LOGIN_SERVER/gantry@$gantry_digest BASELINE_PROBE_IMAGE=$BASELINE_ACR_LOGIN_SERVER/gantry-deploy-probe@$baseline_probe_digest From 2a70cb5d12f172444ab425ef5430399246973a48 Mon Sep 17 00:00:00 2001 From: Vaibhav Patel Date: Wed, 5 Aug 2026 12:21:06 -0400 Subject: [PATCH 12/60] refactor(gantry): build deployment images privately --- hack/gantry-benchmark/Makefile | 4 +- hack/gantry-benchmark/README.md | 7 + hack/gantry-benchmark/deploy.sh | 190 ++++++++---------- .../gantry-benchmark/operator-vm-bootstrap.sh | 80 +++++++- .../operator-vm-build-images.sh | 96 +++++++++ 5 files changed, 265 insertions(+), 112 deletions(-) create mode 100755 hack/gantry-benchmark/operator-vm-build-images.sh diff --git a/hack/gantry-benchmark/Makefile b/hack/gantry-benchmark/Makefile index 59d217aaa..5ef4f7beb 100644 --- a/hack/gantry-benchmark/Makefile +++ b/hack/gantry-benchmark/Makefile @@ -35,8 +35,10 @@ test: cd "$(REPO_ROOT)" && GOTOOLCHAIN=auto go test ./hack/cmd/acr-origin-proxy ./hack/cmd/gantry-benchmark operator-vm-check: - bash -n deploy.sh operator-vm-provision.sh operator-vm-bootstrap.sh operator-vm-run.sh operator-vm-status.sh operator-vm-watch.sh + bash -n deploy.sh operator-vm-provision.sh operator-vm-bootstrap.sh operator-vm-build-images.sh operator-vm-run.sh operator-vm-status.sh operator-vm-watch.sh ./deploy.sh plan deploy.env.example >/dev/null + ! grep -Eq 'az acr login|podman (build|push|login|pull|tag)' deploy.sh + ! grep -Eq '^[[:space:]]*az login([[:space:]]|$$)' deploy.sh deploy: operator-vm-check cd "$(REPO_ROOT)" && hack/gantry-benchmark/deploy.sh deploy "$(DEPLOY_CONFIG)" diff --git a/hack/gantry-benchmark/README.md b/hack/gantry-benchmark/README.md index 9d75fbab0..bce60f959 100644 --- a/hack/gantry-benchmark/README.md +++ b/hack/gantry-benchmark/README.md @@ -27,6 +27,13 @@ passes. The deployment config contains names and topology only. Credentials remain in Azure managed identities and short-lived ACR tokens. +The workstation needs one valid Azure management-plane login before invoking +`deploy.sh`; the script never invokes `az login`, `az acr login`, or workstation +Podman. It publishes only the revision-labelled source carrier through an ACR +Task, creates Private Endpoints and disables public registry access, then +bootstraps the private operator VM. Gantry and pull-probe images are built and +pushed from that VM with its managed identity over Private Link. + The sections below document benchmark behavior and direct lifecycle control. They do not replace `deploy.sh`; commands that assume an existing cluster are for diagnosis or manual operation after full-stack deployment succeeds. diff --git a/hack/gantry-benchmark/deploy.sh b/hack/gantry-benchmark/deploy.sh index 5b0042594..19c8370bc 100755 --- a/hack/gantry-benchmark/deploy.sh +++ b/hack/gantry-benchmark/deploy.sh @@ -154,6 +154,22 @@ require_command() { command -v "$1" >/dev/null 2>&1 || { echo "required command not found: $1" >&2; exit 1; } } +retry_command() { + local attempts=$1 + local delay=$2 + shift 2 + local attempt + for attempt in $(seq 1 "$attempts"); do + if "$@"; then + return 0 + fi + if ((attempt == attempts)); then + return 1 + fi + sleep "$delay" + done +} + print_plan() { cat </dev/null || echo unknown) + source carrier: ACR Task before registry privatization + runtime images: managed-identity operator over Private Link Azure subscription: $AZURE_SUBSCRIPTION_ID @@ -199,11 +217,15 @@ if [[ "$action" == plan ]]; then exit 0 fi -for command in az jq kubectl helm podman git make sha256sum; do +for command in az jq kubectl helm git make sha256sum timeout; do require_command "$command" done az account set --subscription "$AZURE_SUBSCRIPTION_ID" +if ! timeout 60s az account get-access-token --resource https://management.core.windows.net/ --output none; then + echo "Azure management authentication is unavailable; run az login once before deploy.sh" >&2 + exit 1 +fi if [[ "$action" == status ]]; then az group show -g "$AZURE_RESOURCE_GROUP" --query '{name:name,location:location,state:properties.provisioningState}' -o json @@ -344,93 +366,23 @@ ensure_acr() { --data-endpoint-enabled true --only-show-errors -o none } -build_branch_images() { - log "building branch artifacts from $source_revision" - +build_source_image() { + log "publishing private source carrier from $source_revision" SOURCE_IMAGE=$GANTRY_ACR_LOGIN_SERVER/gantry-benchmark-source:$source_revision - GANTRY_IMAGE_TAG=$GANTRY_ACR_LOGIN_SERVER/gantry:benchmark-$source_short - BASELINE_PROBE_TAG=$BASELINE_ACR_LOGIN_SERVER/gantry-deploy-probe:$source_revision - local current_baseline_acr_id current_gantry_acr_id - current_baseline_acr_id=$(az acr show -g "$AZURE_RESOURCE_GROUP" -n "$BASELINE_ACR_NAME" --query id -o tsv) - current_gantry_acr_id=$(az acr show -g "$AZURE_RESOURCE_GROUP" -n "$GANTRY_ACR_NAME" --query id -o tsv) - - local image_state=$DEPLOY_STATE_DIR/images.env - if [[ -f "$image_state" ]]; then - local recorded_revision recorded_baseline_acr_id recorded_gantry_acr_id - recorded_revision=$(sed -n "s/^SOURCE_REVISION='\([^']*\)'$/\1/p" "$image_state") - recorded_baseline_acr_id=$(sed -n "s/^BASELINE_ACR_RESOURCE_ID='\([^']*\)'$/\1/p" "$image_state") - recorded_gantry_acr_id=$(sed -n "s/^GANTRY_ACR_RESOURCE_ID='\([^']*\)'$/\1/p" "$image_state") - if [[ "$recorded_revision" == "$source_revision" && - "$recorded_baseline_acr_id" == "$current_baseline_acr_id" && - "$recorded_gantry_acr_id" == "$current_gantry_acr_id" ]]; then - # shellcheck source=/dev/null - . "$image_state" - [[ "$SOURCE_IMAGE" == "$GANTRY_ACR_LOGIN_SERVER/gantry-benchmark-source:$source_revision" ]] || { - echo "recorded source image does not match this deployment" >&2 - exit 1 - } - [[ "$GANTRY_IMAGE" == "$GANTRY_ACR_LOGIN_SERVER/gantry@sha256:"* ]] || { - echo "recorded Gantry image is not an immutable deployment reference" >&2 - exit 1 - } - [[ "$BASELINE_PROBE_IMAGE" == "$BASELINE_ACR_LOGIN_SERVER/gantry-deploy-probe@sha256:"* ]] || { - echo "recorded baseline probe image is not an immutable deployment reference" >&2 - exit 1 - } - log "reusing locally recorded immutable branch artifacts" - return - fi - fi public_restore_needed=true - for registry in "$BASELINE_ACR_NAME" "$GANTRY_ACR_NAME"; do - az acr update -g "$AZURE_RESOURCE_GROUP" -n "$registry" \ - --public-network-enabled true --only-show-errors -o none - done + az acr update -g "$AZURE_RESOURCE_GROUP" -n "$GANTRY_ACR_NAME" \ + --public-network-enabled true --only-show-errors -o none - local token - token=$(az acr login --name "$GANTRY_ACR_NAME" --expose-token --query accessToken -o tsv) - printf '%s' "$token" | podman login "$GANTRY_ACR_LOGIN_SERVER" \ - --username 00000000-0000-0000-0000-000000000000 --password-stdin >/dev/null - unset token - - podman build --isolation chroot --build-arg "SOURCE_REVISION=$source_revision" \ - -t "$SOURCE_IMAGE" -f "$repo_root/images/gantry-benchmark-source/Containerfile" "$repo_root" - podman push "$SOURCE_IMAGE" >/dev/null - - podman build --isolation chroot \ - --build-arg "VERSION=benchmark-$source_short" \ - --build-arg "GIT_COMMIT=$source_revision" \ - -t "$GANTRY_IMAGE_TAG" -f "$repo_root/images/gantry/Containerfile" "$repo_root" - local digest_file=$DEPLOY_STATE_DIR/gantry.digest - podman push --digestfile "$digest_file" "$GANTRY_IMAGE_TAG" >/dev/null - local gantry_digest - gantry_digest=$(tr -d '[:space:]' <"$digest_file") - podman logout "$GANTRY_ACR_LOGIN_SERVER" >/dev/null - - token=$(az acr login --name "$BASELINE_ACR_NAME" --expose-token --query accessToken -o tsv) - printf '%s' "$token" | podman login "$BASELINE_ACR_LOGIN_SERVER" \ - --username 00000000-0000-0000-0000-000000000000 --password-stdin >/dev/null - unset token - podman pull mcr.microsoft.com/cbl-mariner/busybox:2.0 >/dev/null - podman tag mcr.microsoft.com/cbl-mariner/busybox:2.0 "$BASELINE_PROBE_TAG" - local probe_digest_file=$DEPLOY_STATE_DIR/baseline-probe.digest - podman push --digestfile "$probe_digest_file" "$BASELINE_PROBE_TAG" >/dev/null - local baseline_probe_digest - baseline_probe_digest=$(tr -d '[:space:]' <"$probe_digest_file") - podman logout "$BASELINE_ACR_LOGIN_SERVER" >/dev/null - - GANTRY_IMAGE=$GANTRY_ACR_LOGIN_SERVER/gantry@$gantry_digest - BASELINE_PROBE_IMAGE=$BASELINE_ACR_LOGIN_SERVER/gantry-deploy-probe@$baseline_probe_digest - cat >"$image_state" <&2 return 1 @@ -866,23 +831,32 @@ provision_operator() { "$OPERATOR_BUILD_DISK_SKU" } -verify_operator_private_pushes() { - log "verifying private operator pushes to both ACR data endpoints" - az vm run-command invoke -g "$AZURE_RESOURCE_GROUP" -n "$OPERATOR_VM_NAME" \ - --command-id RunShellScript --scripts "set -eu -podman pull mcr.microsoft.com/cbl-mariner/busybox:2.0 >/dev/null -for acr in '$BASELINE_ACR_NAME' '$GANTRY_ACR_NAME'; do - login=\"\${acr}.azurecr.io\" - token=\$(az acr login --name \"\$acr\" --expose-token --query accessToken -o tsv) - printf '%s' \"\$token\" | podman login \"\$login\" \\ - --username 00000000-0000-0000-0000-000000000000 --password-stdin >/dev/null - unset token - target=\"\$login/gantry-deploy-operator-probe:$source_revision\" - podman tag mcr.microsoft.com/cbl-mariner/busybox:2.0 \"\$target\" - podman push \"\$target\" >/dev/null - podman logout \"\$login\" >/dev/null - podman image rm \"\$target\" >/dev/null -done" --only-show-errors -o none +build_operator_images() { + log "building Gantry and pull-probe images inside the private operator VM" + local output + output=$(az vm run-command invoke -g "$AZURE_RESOURCE_GROUP" -n "$OPERATOR_VM_NAME" \ + --command-id RunShellScript \ + --scripts @"$repo_root/hack/gantry-benchmark/operator-vm-build-images.sh" \ + --parameters "$AZURE_SUBSCRIPTION_ID" "$BASELINE_ACR_NAME" "$GANTRY_ACR_NAME" \ + "$source_revision" "$source_short" \ + --query 'value[0].message' -o tsv) + local result_json + result_json=$(tr -d '\r' <<<"$output" | sed -n 's/^DEPLOYMENT_IMAGES_JSON=//p' | tail -1) + jq -e 'type == "object" and (.gantry_image | type == "string") and (.baseline_probe_image | type == "string")' \ + <<<"$result_json" >/dev/null || { + echo "operator did not return valid deployment image JSON" >&2 + return 1 + } + GANTRY_IMAGE=$(jq -r .gantry_image <<<"$result_json") + BASELINE_PROBE_IMAGE=$(jq -r .baseline_probe_image <<<"$result_json") + [[ "$GANTRY_IMAGE" == "$GANTRY_ACR_LOGIN_SERVER/gantry@sha256:"* ]] || { + echo "operator did not return an immutable Gantry image" >&2 + return 1 + } + [[ "$BASELINE_PROBE_IMAGE" == "$BASELINE_ACR_LOGIN_SERVER/gantry-deploy-probe@sha256:"* ]] || { + echo "operator did not return an immutable baseline probe image" >&2 + return 1 + } } guard_active_benchmark @@ -890,11 +864,7 @@ ensure_group ensure_vnet ensure_acr "$BASELINE_ACR_NAME" ensure_acr "$GANTRY_ACR_NAME" -build_branch_images -if [[ "$public_restore_needed" == true ]]; then - set_acrs_private - public_restore_needed=false -fi +build_source_image ensure_private_network ensure_aks ensure_diagnostics @@ -912,10 +882,10 @@ install_monitoring set_acrs_private +provision_operator +build_operator_images verify_private_baseline_pull deploy_gantry -provision_operator -verify_operator_private_pushes log "validating final deployment" assert_equal "baseline ACR public access" \ diff --git a/hack/gantry-benchmark/operator-vm-bootstrap.sh b/hack/gantry-benchmark/operator-vm-bootstrap.sh index 03db48025..cd8c893cd 100755 --- a/hack/gantry-benchmark/operator-vm-bootstrap.sh +++ b/hack/gantry-benchmark/operator-vm-bootstrap.sh @@ -52,6 +52,64 @@ retry() { done } +acr_access_token() { + local acr_name=$1 + local attempts=0 + local maximum=60 + local token + + until token=$(az acr login --name "$acr_name" --expose-token --query accessToken -o tsv); do + attempts=$((attempts + 1)) + if ((attempts >= maximum)); then + echo "failed to obtain managed-identity ACR token for $acr_name after $attempts attempts" >&2 + return 1 + fi + sleep 10 + done + + printf '%s' "$token" +} + +private_dns_ip() { + local record_name=$1 + local attempts=0 + local maximum=60 + local ip + + until ip=$(az network private-dns record-set a show \ + --resource-group "$resource_group" \ + --zone-name privatelink.azurecr.io \ + --name "$record_name" \ + --query 'aRecords[0].ipv4Address' \ + --output tsv 2>/dev/null) && [[ -n "$ip" ]]; do + attempts=$((attempts + 1)) + if ((attempts >= maximum)); then + echo "private DNS record $record_name did not become readable" >&2 + return 1 + fi + sleep 10 + done + + printf '%s' "$ip" +} + +require_private_resolution() { + local host=$1 + local expected_ip=$2 + local attempts=0 + local maximum=60 + + local resolved + until resolved=$(getent ahostsv4 "$host" | awk '{print $1}' | sort -u) && [[ "$resolved" == "$expected_ip" ]]; do + attempts=$((attempts + 1)) + if ((attempts >= maximum)); then + echo "$host did not resolve to private IP $expected_ip" >&2 + return 1 + fi + sleep 10 + done +} + export DEBIAN_FRONTEND=noninteractive apt-get update apt-get install -y ca-certificates curl e2fsprogs git gnupg jq make podman golang-go @@ -97,11 +155,31 @@ install -d -m 0700 /var/lib/gantry-benchmark install -d -m 0750 /etc/gantry-benchmark install -d -m 0750 /var/log/gantry-benchmark +retry az acr show -g "$resource_group" -n "$baseline_acr_name" --output none +retry az acr show -g "$resource_group" -n "$gantry_acr_name" --output none +for acr_name in "$baseline_acr_name" "$gantry_acr_name"; do + public_access=$(az acr show -g "$resource_group" -n "$acr_name" --query publicNetworkAccess -o tsv) + data_endpoint=$(az acr show -g "$resource_group" -n "$acr_name" --query dataEndpointEnabled -o tsv) + [[ "$public_access" == Disabled ]] || { echo "$acr_name public access is $public_access, want Disabled" >&2; exit 1; } + [[ "$data_endpoint" == true ]] || { echo "$acr_name dedicated data endpoint is not enabled" >&2; exit 1; } +done + +baseline_location=$(az acr show -g "$resource_group" -n "$baseline_acr_name" --query location -o tsv) +gantry_location=$(az acr show -g "$resource_group" -n "$gantry_acr_name" --query location -o tsv) +baseline_login_ip=$(private_dns_ip "$baseline_acr_name") +baseline_data_ip=$(private_dns_ip "$baseline_acr_name.$baseline_location.data") +gantry_login_ip=$(private_dns_ip "$gantry_acr_name") +gantry_data_ip=$(private_dns_ip "$gantry_acr_name.$gantry_location.data") +require_private_resolution "$baseline_acr_name.azurecr.io" "$baseline_login_ip" +require_private_resolution "$baseline_acr_name.$baseline_location.data.azurecr.io" "$baseline_data_ip" +require_private_resolution "$gantry_acr_name.azurecr.io" "$gantry_login_ip" +require_private_resolution "$gantry_acr_name.$gantry_location.data.azurecr.io" "$gantry_data_ip" + repo_root="$build_mount/unbounded" source_description="$repo_url ($repo_branch)" if [[ -n "$source_image" ]]; then gantry_login_server=$(az acr show -g "$resource_group" -n "$gantry_acr_name" --query loginServer -o tsv) - source_token=$(az acr login --name "$gantry_acr_name" --expose-token --query accessToken -o tsv) + source_token=$(acr_access_token "$gantry_acr_name") printf '%s' "$source_token" | podman login "$gantry_login_server" \ --username 00000000-0000-0000-0000-000000000000 \ --password-stdin diff --git a/hack/gantry-benchmark/operator-vm-build-images.sh b/hack/gantry-benchmark/operator-vm-build-images.sh new file mode 100755 index 000000000..8c6a8b1c7 --- /dev/null +++ b/hack/gantry-benchmark/operator-vm-build-images.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +set -Eeuo pipefail + +if [[ $# -ne 5 ]]; then + echo "usage: operator-vm-build-images.sh " >&2 + exit 2 +fi + +subscription_id=$1 +baseline_acr=$2 +gantry_acr=$3 +source_revision=$4 +source_short=$5 +baseline_login="${baseline_acr}.azurecr.io" +gantry_login="${gantry_acr}.azurecr.io" +log_file=/var/log/gantry-benchmark/deployment-images.log + +. /etc/gantry-benchmark/env +export HOME="${BENCHMARK_OPERATOR_HOME:-/var/lib/gantry-benchmark}" +cd "$BENCHMARK_REPO_ROOT" + +az login --identity --allow-no-subscriptions --output none >>"$log_file" 2>&1 +az account set --subscription "$subscription_id" + +image_state=/var/lib/gantry-benchmark/deployment-images.env +if [[ -f "$image_state" ]]; then + recorded_revision=$(sed -n "s/^SOURCE_REVISION='\([^']*\)'$/\1/p" "$image_state") + if [[ "$recorded_revision" == "$source_revision" ]]; then + # shellcheck source=/dev/null + . "$image_state" + if [[ "$GANTRY_IMAGE" == "$gantry_login/gantry@sha256:"* && + "$BASELINE_PROBE_IMAGE" == "$baseline_login/gantry-deploy-probe@sha256:"* ]]; then + jq -cn --arg gantry_image "$GANTRY_IMAGE" --arg baseline_probe_image "$BASELINE_PROBE_IMAGE" \ + '{gantry_image:$gantry_image,baseline_probe_image:$baseline_probe_image}' | \ + sed 's/^/DEPLOYMENT_IMAGES_JSON=/' + exit 0 + fi + fi +fi + +registry_login() { + local acr=$1 + local login=$2 + local token + local attempt + + for attempt in $(seq 1 18); do + if token=$(az acr login --name "$acr" --expose-token --query accessToken -o tsv 2>>"$log_file"); then + printf '%s' "$token" | podman login "$login" \ + --username 00000000-0000-0000-0000-000000000000 \ + --password-stdin >>"$log_file" 2>&1 + unset token + return + fi + sleep 10 + done + + echo "failed to authenticate to $login" >&2 + return 1 +} + +registry_login "$gantry_acr" "$gantry_login" +gantry_tag="$gantry_login/gantry:benchmark-$source_short" +podman build --isolation chroot --platform linux/amd64 \ + --build-arg "VERSION=benchmark-$source_short" \ + --build-arg "GIT_COMMIT=$source_revision" \ + --tag "$gantry_tag" --file images/gantry/Containerfile . >>"$log_file" 2>&1 +gantry_digest_file=/var/lib/gantry-benchmark/gantry-deploy.digest +podman push --digestfile "$gantry_digest_file" "$gantry_tag" >>"$log_file" 2>&1 +gantry_digest=$(tr -d '[:space:]' <"$gantry_digest_file") +podman logout "$gantry_login" >>"$log_file" 2>&1 + +registry_login "$baseline_acr" "$baseline_login" +podman pull mcr.microsoft.com/cbl-mariner/busybox:2.0 >>"$log_file" 2>&1 +probe_tag="$baseline_login/gantry-deploy-probe:$source_revision" +podman tag mcr.microsoft.com/cbl-mariner/busybox:2.0 "$probe_tag" +probe_digest_file=/var/lib/gantry-benchmark/baseline-probe.digest +podman push --digestfile "$probe_digest_file" "$probe_tag" >>"$log_file" 2>&1 +probe_digest=$(tr -d '[:space:]' <"$probe_digest_file") +podman logout "$baseline_login" >>"$log_file" 2>&1 + +GANTRY_IMAGE="$gantry_login/gantry@$gantry_digest" +BASELINE_PROBE_IMAGE="$baseline_login/gantry-deploy-probe@$probe_digest" +cat >"$image_state" < Date: Wed, 5 Aug 2026 12:42:21 -0400 Subject: [PATCH 13/60] fix(gantry): wait for ACR task access --- hack/gantry-benchmark/deploy.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/hack/gantry-benchmark/deploy.sh b/hack/gantry-benchmark/deploy.sh index 19c8370bc..16d1894c9 100755 --- a/hack/gantry-benchmark/deploy.sh +++ b/hack/gantry-benchmark/deploy.sh @@ -166,6 +166,7 @@ retry_command() { if ((attempt == attempts)); then return 1 fi + log "attempt $attempt/$attempts failed; retrying in ${delay}s" sleep "$delay" done } @@ -366,6 +367,10 @@ ensure_acr() { --data-endpoint-enabled true --only-show-errors -o none } +acr_public_access_enabled() { + [[ $(az acr show -g "$AZURE_RESOURCE_GROUP" -n "$GANTRY_ACR_NAME" --query publicNetworkAccess -o tsv) == Enabled ]] +} + build_source_image() { log "publishing private source carrier from $source_revision" SOURCE_IMAGE=$GANTRY_ACR_LOGIN_SERVER/gantry-benchmark-source:$source_revision @@ -374,7 +379,9 @@ build_source_image() { az acr update -g "$AZURE_RESOURCE_GROUP" -n "$GANTRY_ACR_NAME" \ --public-network-enabled true --only-show-errors -o none - retry_command 6 20 az acr build \ + retry_command 30 10 acr_public_access_enabled + + retry_command 18 30 az acr build \ --registry "$GANTRY_ACR_NAME" \ --image "gantry-benchmark-source:$source_revision" \ --file "$repo_root/images/gantry-benchmark-source/Containerfile" \ From fb97d78e726cbdb97edded9b929b6a9d35536b52 Mon Sep 17 00:00:00 2001 From: Vaibhav Patel Date: Wed, 5 Aug 2026 12:44:50 -0400 Subject: [PATCH 14/60] fix(gantry): wait quietly for ACR task propagation --- hack/gantry-benchmark/deploy.sh | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/hack/gantry-benchmark/deploy.sh b/hack/gantry-benchmark/deploy.sh index 16d1894c9..415435542 100755 --- a/hack/gantry-benchmark/deploy.sh +++ b/hack/gantry-benchmark/deploy.sh @@ -381,12 +381,27 @@ build_source_image() { retry_command 30 10 acr_public_access_enabled - retry_command 18 30 az acr build \ - --registry "$GANTRY_ACR_NAME" \ - --image "gantry-benchmark-source:$source_revision" \ - --file "$repo_root/images/gantry-benchmark-source/Containerfile" \ - --build-arg "SOURCE_REVISION=$source_revision" \ - "$repo_root" --only-show-errors -o none + local build_log=$DEPLOY_STATE_DIR/source-carrier-build.log + local built=false + local attempt + for attempt in $(seq 1 18); do + if az acr build \ + --registry "$GANTRY_ACR_NAME" \ + --image "gantry-benchmark-source:$source_revision" \ + --file "$repo_root/images/gantry-benchmark-source/Containerfile" \ + --build-arg "SOURCE_REVISION=$source_revision" \ + "$repo_root" --only-show-errors -o none >"$build_log" 2>&1; then + built=true + break + fi + log "source-carrier ACR Task is waiting for firewall propagation ($attempt/18)" + sleep 30 + done + if [[ "$built" != true ]]; then + cat "$build_log" >&2 + return 1 + fi + cat "$build_log" set_acrs_private public_restore_needed=false From 02ecb4de122b8a1aa7596d8431c4563fb304d0b8 Mon Sep 17 00:00:00 2001 From: Vaibhav Patel Date: Wed, 5 Aug 2026 13:02:35 -0400 Subject: [PATCH 15/60] fix(gantry): open ACR firewall for source build --- hack/gantry-benchmark/deploy.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/hack/gantry-benchmark/deploy.sh b/hack/gantry-benchmark/deploy.sh index 415435542..90c4120fa 100755 --- a/hack/gantry-benchmark/deploy.sh +++ b/hack/gantry-benchmark/deploy.sh @@ -265,7 +265,7 @@ set_acrs_private() { for acr in "$BASELINE_ACR_NAME" "$GANTRY_ACR_NAME"; do if az acr show -g "$AZURE_RESOURCE_GROUP" -n "$acr" --output none >/dev/null 2>&1; then az acr update -g "$AZURE_RESOURCE_GROUP" -n "$acr" \ - --data-endpoint-enabled true --public-network-enabled false \ + --data-endpoint-enabled true --default-action Deny --public-network-enabled false \ --only-show-errors -o none fi done @@ -368,7 +368,8 @@ ensure_acr() { } acr_public_access_enabled() { - [[ $(az acr show -g "$AZURE_RESOURCE_GROUP" -n "$GANTRY_ACR_NAME" --query publicNetworkAccess -o tsv) == Enabled ]] + [[ $(az acr show -g "$AZURE_RESOURCE_GROUP" -n "$GANTRY_ACR_NAME" \ + --query '[publicNetworkAccess,networkRuleSet.defaultAction]' -o tsv) == $'Enabled\tAllow' ]] } build_source_image() { @@ -377,7 +378,7 @@ build_source_image() { public_restore_needed=true az acr update -g "$AZURE_RESOURCE_GROUP" -n "$GANTRY_ACR_NAME" \ - --public-network-enabled true --only-show-errors -o none + --default-action Allow --public-network-enabled true --only-show-errors -o none retry_command 30 10 acr_public_access_enabled From 777d178a050e4641a7952d3e8fe65a825a6b5f13 Mon Sep 17 00:00:00 2001 From: Vaibhav Patel Date: Wed, 5 Aug 2026 13:07:17 -0400 Subject: [PATCH 16/60] fix(gantry): parse ACR readiness as JSON --- hack/gantry-benchmark/deploy.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/hack/gantry-benchmark/deploy.sh b/hack/gantry-benchmark/deploy.sh index 90c4120fa..90dc6cb19 100755 --- a/hack/gantry-benchmark/deploy.sh +++ b/hack/gantry-benchmark/deploy.sh @@ -368,8 +368,10 @@ ensure_acr() { } acr_public_access_enabled() { - [[ $(az acr show -g "$AZURE_RESOURCE_GROUP" -n "$GANTRY_ACR_NAME" \ - --query '[publicNetworkAccess,networkRuleSet.defaultAction]' -o tsv) == $'Enabled\tAllow' ]] + local state + state=$(az acr show -g "$AZURE_RESOURCE_GROUP" -n "$GANTRY_ACR_NAME" -o json) + [[ $(jq -r .publicNetworkAccess <<<"$state") == Enabled && + $(jq -r .networkRuleSet.defaultAction <<<"$state") == Allow ]] } build_source_image() { From dffb5415fb023f9b3554fc3d7f079625ee65743e Mon Sep 17 00:00:00 2001 From: Vaibhav Patel Date: Wed, 5 Aug 2026 13:39:47 -0400 Subject: [PATCH 17/60] fix(gantry): ensure ACR private DNS zone groups --- hack/gantry-benchmark/deploy.sh | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/hack/gantry-benchmark/deploy.sh b/hack/gantry-benchmark/deploy.sh index 90dc6cb19..9caea5e48 100755 --- a/hack/gantry-benchmark/deploy.sh +++ b/hack/gantry-benchmark/deploy.sh @@ -495,7 +495,7 @@ ensure_private_endpoint() { local name=$1 local acr_id=$2 local connection_name=$3 - local subnet_id zone_id + local subnet_id zone_id zone_group_count subnet_id=$(az network vnet subnet show -g "$AZURE_RESOURCE_GROUP" --vnet-name "$VNET_NAME" \ -n "$PRIVATE_ENDPOINT_SUBNET_NAME" --query id -o tsv) zone_id=$(az network private-dns zone show -g "$AZURE_RESOURCE_GROUP" -n "$PRIVATE_DNS_ZONE" --query id -o tsv) @@ -505,12 +505,17 @@ ensure_private_endpoint() { --subnet "$subnet_id" --private-connection-resource-id "$acr_id" \ --group-ids registry --connection-name "$connection_name" --only-show-errors -o none fi - if ! az network private-endpoint dns-zone-group show -g "$AZURE_RESOURCE_GROUP" \ - --endpoint-name "$name" -n acr --output none 2>/dev/null; then + zone_group_count=$(az network private-endpoint dns-zone-group list -g "$AZURE_RESOURCE_GROUP" \ + --endpoint-name "$name" --query 'length(@)' -o tsv) + if [[ "$zone_group_count" == 0 ]]; then az network private-endpoint dns-zone-group create -g "$AZURE_RESOURCE_GROUP" \ --endpoint-name "$name" -n acr --private-dns-zone "$zone_id" \ --zone-name "$PRIVATE_DNS_ZONE" --only-show-errors -o none fi + assert_equal "$name private DNS zone" \ + "$(az network private-endpoint dns-zone-group show -g "$AZURE_RESOURCE_GROUP" \ + --endpoint-name "$name" -n acr --query 'privateDnsZoneConfigs[0].privateDnsZoneId' -o tsv)" \ + "$zone_id" assert_equal "$name connection state" \ "$(az network private-endpoint show -g "$AZURE_RESOURCE_GROUP" -n "$name" --query 'privateLinkServiceConnections[0].privateLinkServiceConnectionState.status' -o tsv)" \ Approved From 9a58b985874a15eb2d39b142e33f22035b8ae3da Mon Sep 17 00:00:00 2001 From: Vaibhav Patel Date: Wed, 5 Aug 2026 13:51:07 -0400 Subject: [PATCH 18/60] fix(gantry): serialize operator VM commands --- hack/gantry-benchmark/deploy.sh | 16 ++++++++++++++++ hack/gantry-benchmark/operator-vm-watch.sh | 6 ++++++ 2 files changed, 22 insertions(+) diff --git a/hack/gantry-benchmark/deploy.sh b/hack/gantry-benchmark/deploy.sh index 9caea5e48..5d80d954e 100755 --- a/hack/gantry-benchmark/deploy.sh +++ b/hack/gantry-benchmark/deploy.sh @@ -102,6 +102,7 @@ START_BENCHMARK=${START_BENCHMARK:-false} DEPLOY_CONFIRM=${DEPLOY_CONFIRM:-} DEPLOY_STATE_DIR=${DEPLOY_STATE_DIR:-$repo_root/tmp/$DEPLOYMENT_NAME} KUBECONFIG=${DEPLOY_KUBECONFIG:-$DEPLOY_STATE_DIR/kubeconfig} +OPERATOR_RUN_COMMAND_LOCK=${OPERATOR_RUN_COMMAND_LOCK:-${TMPDIR:-/tmp}/gantry-benchmark-${AZURE_RESOURCE_GROUP}-${OPERATOR_VM_NAME}.run-command.lock} BASELINE_PRIVATE_ENDPOINT_NAME=${BASELINE_PRIVATE_ENDPOINT_NAME:-${DEPLOYMENT_NAME}-baseline-acr-pe} GANTRY_PRIVATE_ENDPOINT_NAME=${GANTRY_PRIVATE_ENDPOINT_NAME:-${DEPLOYMENT_NAME}-gantry-acr-pe} @@ -889,6 +890,17 @@ build_operator_images() { } } +acquire_operator_run_command_lock() { + log "waiting for exclusive operator VM Run Command access" + exec {operator_run_command_lock_fd}>"$OPERATOR_RUN_COMMAND_LOCK" + flock "$operator_run_command_lock_fd" +} + +release_operator_run_command_lock() { + flock -u "$operator_run_command_lock_fd" + exec {operator_run_command_lock_fd}>&- +} + guard_active_benchmark ensure_group ensure_vnet @@ -912,8 +924,10 @@ install_monitoring set_acrs_private +acquire_operator_run_command_lock provision_operator build_operator_images +release_operator_run_command_lock verify_private_baseline_pull deploy_gantry @@ -934,10 +948,12 @@ done if [[ "$START_BENCHMARK" == true ]]; then log "starting benchmark operator service" + acquire_operator_run_command_lock az vm run-command invoke -g "$AZURE_RESOURCE_GROUP" -n "$OPERATOR_VM_NAME" \ --command-id RunShellScript \ --scripts 'systemctl reset-failed gantry-benchmark-operator.service; systemctl start --no-block gantry-benchmark-operator.service' \ --only-show-errors -o none + release_operator_run_command_lock fi trap - EXIT INT TERM diff --git a/hack/gantry-benchmark/operator-vm-watch.sh b/hack/gantry-benchmark/operator-vm-watch.sh index 34e7f3d40..a80469ffd 100755 --- a/hack/gantry-benchmark/operator-vm-watch.sh +++ b/hack/gantry-benchmark/operator-vm-watch.sh @@ -10,6 +10,7 @@ OPERATOR_SSH_HOST="${OPERATOR_SSH_HOST:-}" OPERATOR_SSH_KEY="${OPERATOR_SSH_KEY:-}" OPERATOR_SSH_USER="${OPERATOR_SSH_USER:-benchmark}" WATCH_INTERVAL_SECONDS="${WATCH_INTERVAL_SECONDS:-30}" +OPERATOR_RUN_COMMAND_LOCK="${OPERATOR_RUN_COMMAND_LOCK:-${TMPDIR:-/tmp}/gantry-benchmark-${AZURE_RESOURCE_GROUP}-${OPERATOR_VM_NAME}.run-command.lock}" follow=false usage() { @@ -63,6 +64,9 @@ status_once() { 'sudo -n /opt/gantry-benchmark/unbounded/hack/gantry-benchmark/operator-vm-status.sh') else : "${AZURE_RESOURCE_GROUP:?Set AZURE_RESOURCE_GROUP when OPERATOR_SSH_HOST is not set}" + local run_command_lock_fd + exec {run_command_lock_fd}>"$OPERATOR_RUN_COMMAND_LOCK" + flock "$run_command_lock_fd" output=$(az vm run-command invoke \ -g "$AZURE_RESOURCE_GROUP" \ -n "$OPERATOR_VM_NAME" \ @@ -71,6 +75,8 @@ status_once() { --only-show-errors \ --query 'value[0].message' \ -o tsv) + flock -u "$run_command_lock_fd" + exec {run_command_lock_fd}>&- fi printf '%s\n' "$output" | sed \ From 4b8446239bcb59fc1846af8670998a5a263e188d Mon Sep 17 00:00:00 2001 From: Vaibhav Patel Date: Wed, 5 Aug 2026 14:07:17 -0400 Subject: [PATCH 19/60] fix(gantry): validate operator disk size --- hack/gantry-benchmark/deploy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/gantry-benchmark/deploy.sh b/hack/gantry-benchmark/deploy.sh index 5d80d954e..b9f83847a 100755 --- a/hack/gantry-benchmark/deploy.sh +++ b/hack/gantry-benchmark/deploy.sh @@ -855,7 +855,7 @@ provision_operator() { local build_disk_name build_disk_name=${OPERATOR_BUILD_DISK_NAME:-${OPERATOR_VM_NAME}-build} assert_equal "operator build disk size" \ - "$(az disk show -g "$AZURE_RESOURCE_GROUP" -n "$build_disk_name" --query diskSizeGb -o tsv)" \ + "$(az disk show -g "$AZURE_RESOURCE_GROUP" -n "$build_disk_name" --query diskSizeGB -o tsv)" \ "$OPERATOR_BUILD_DISK_GB" assert_equal "operator build disk SKU" \ "$(az disk show -g "$AZURE_RESOURCE_GROUP" -n "$build_disk_name" --query sku.name -o tsv)" \ From 6266dc0f2e3b4b4c72154c2cf7080b642591fc3b Mon Sep 17 00:00:00 2001 From: Vaibhav Patel Date: Wed, 5 Aug 2026 14:50:06 -0400 Subject: [PATCH 20/60] fix(gantry): recover isolated private pull failures --- hack/gantry-benchmark/deploy.sh | 57 +++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/hack/gantry-benchmark/deploy.sh b/hack/gantry-benchmark/deploy.sh index b9f83847a..59c2e569b 100755 --- a/hack/gantry-benchmark/deploy.sh +++ b/hack/gantry-benchmark/deploy.sh @@ -81,6 +81,7 @@ BENCHMARK_IMAGE_SIZE_MIB=${BENCHMARK_IMAGE_SIZE_MIB:-40960} BENCHMARK_IMAGE_LAYERS=${BENCHMARK_IMAGE_LAYERS:-40} BENCHMARK_MINIMUM_BYTE_REDUCTION=${BENCHMARK_MINIMUM_BYTE_REDUCTION:-0.90} BENCHMARK_MAXIMUM_LATENCY_RATIO=${BENCHMARK_MAXIMUM_LATENCY_RATIO:-1.0} +BASELINE_PULL_MAX_NODE_RESTARTS=${BASELINE_PULL_MAX_NODE_RESTARTS:-5} GANTRY_NAMESPACE=${GANTRY_NAMESPACE:-gantry-system} BENCHMARK_NAMESPACE=${BENCHMARK_NAMESPACE:-gantry-benchmark} @@ -124,6 +125,10 @@ GANTRY_ACR_DATA_HOST=${GANTRY_ACR_NAME}.${AZURE_LOCATION}.data.azurecr.io echo "BENCHMARK_IMAGE_LAYERS cannot exceed BENCHMARK_IMAGE_SIZE_MIB" >&2 exit 2 } +[[ "$BASELINE_PULL_MAX_NODE_RESTARTS" =~ ^[1-9][0-9]*$ ]] || { + echo "BASELINE_PULL_MAX_NODE_RESTARTS must be positive" >&2 + exit 2 +} for acr_name in "$BASELINE_ACR_NAME" "$GANTRY_ACR_NAME"; do [[ "$acr_name" =~ ^[a-z0-9]{5,50}$ ]] || { echo "invalid ACR name $acr_name: use 5-50 lowercase alphanumeric characters" >&2 @@ -754,6 +759,53 @@ GUARD kubectl -n "$GANTRY_NAMESPACE" rollout status daemonset/gantry-acr-private-dns-guard --timeout=30m } +restart_private_pull_tls_nodes() { + local pods_json node provider_id vmss instance_id old_boot_id + local -a nodes + pods_json=$(kubectl -n "$GANTRY_NAMESPACE" get pods \ + -l app.kubernetes.io/name=gantry-baseline-acr-pull-probe -o json) + mapfile -t nodes < <(jq -r '.items[] | + select(any(.status.containerStatuses[]?; ((.state.waiting.message? // "") | contains("TLS handshake timeout")))) | + .spec.nodeName' <<<"$pods_json" | sort -u) + ((${#nodes[@]} > 0)) || return 1 + ((${#nodes[@]} <= BASELINE_PULL_MAX_NODE_RESTARTS)) || { + echo "refusing to restart ${#nodes[@]} nodes with ACR TLS handshake timeouts; limit is $BASELINE_PULL_MAX_NODE_RESTARTS" >&2 + return 1 + } + + for node in "${nodes[@]}"; do + provider_id=$(kubectl get node "$node" -o jsonpath='{.spec.providerID}') + vmss=$(sed -n 's#^.*/virtualMachineScaleSets/\([^/]*\)/virtualMachines/[^/]*$#\1#p' <<<"$provider_id") + instance_id=$(sed -n 's#^.*/virtualMachineScaleSets/[^/]*/virtualMachines/\([^/]*\)$#\1#p' <<<"$provider_id") + [[ -n "$vmss" && -n "$instance_id" ]] || { + echo "cannot parse VMSS instance from provider ID for $node: $provider_id" >&2 + return 1 + } + old_boot_id=$(kubectl get node "$node" -o jsonpath='{.status.nodeInfo.bootID}') + log "restarting $node (VMSS $vmss instance $instance_id) after ACR Private Endpoint TLS timeouts" + az vmss restart -g "$AZURE_NODE_RESOURCE_GROUP" -n "$vmss" \ + --instance-ids "$instance_id" --only-show-errors -o none + + local attempt current_boot_id ready + for attempt in $(seq 1 90); do + current_boot_id=$(kubectl get node "$node" -o jsonpath='{.status.nodeInfo.bootID}' 2>/dev/null || true) + ready=$(kubectl get node "$node" -o json | \ + jq -r 'any(.status.conditions[]; .type == "Ready" and .status == "True")' 2>/dev/null || true) + if [[ -n "$current_boot_id" && "$current_boot_id" != "$old_boot_id" && "$ready" == true ]]; then + break + fi + sleep 10 + done + [[ -n "$current_boot_id" && "$current_boot_id" != "$old_boot_id" && "$ready" == true ]] || { + echo "$node did not return Ready with a new boot ID after restart" >&2 + return 1 + } + kubectl -n "$GANTRY_NAMESPACE" delete pod \ + -l app.kubernetes.io/name=gantry-baseline-acr-pull-probe \ + --field-selector "spec.nodeName=$node" --wait=false + done +} + verify_private_baseline_pull() { export KUBECONFIG local manifest=$DEPLOY_STATE_DIR/baseline-private-pull-probe.yaml @@ -793,6 +845,7 @@ spec: PROBE kubectl apply -f "$manifest" local ready=false + local repair_attempted=false local attempt for attempt in $(seq 1 6); do if kubectl -n "$GANTRY_NAMESPACE" rollout status \ @@ -800,6 +853,10 @@ PROBE ready=true break fi + if [[ "$repair_attempted" == false ]] && restart_private_pull_tls_nodes; then + repair_attempted=true + continue + fi log "waiting for baseline AcrPull propagation ($attempt/6)" done if [[ "$ready" != true ]]; then From 46bf1d75869e8b486331b224a51af83c9bdfbdf1 Mon Sep 17 00:00:00 2001 From: Vaibhav Patel Date: Wed, 5 Aug 2026 15:30:07 -0400 Subject: [PATCH 21/60] fix(gantry): reimage persistent private pull failures --- hack/gantry-benchmark/deploy.sh | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/hack/gantry-benchmark/deploy.sh b/hack/gantry-benchmark/deploy.sh index 59c2e569b..1ea5d3563 100755 --- a/hack/gantry-benchmark/deploy.sh +++ b/hack/gantry-benchmark/deploy.sh @@ -81,7 +81,7 @@ BENCHMARK_IMAGE_SIZE_MIB=${BENCHMARK_IMAGE_SIZE_MIB:-40960} BENCHMARK_IMAGE_LAYERS=${BENCHMARK_IMAGE_LAYERS:-40} BENCHMARK_MINIMUM_BYTE_REDUCTION=${BENCHMARK_MINIMUM_BYTE_REDUCTION:-0.90} BENCHMARK_MAXIMUM_LATENCY_RATIO=${BENCHMARK_MAXIMUM_LATENCY_RATIO:-1.0} -BASELINE_PULL_MAX_NODE_RESTARTS=${BASELINE_PULL_MAX_NODE_RESTARTS:-5} +BASELINE_PULL_MAX_NODE_REIMAGES=${BASELINE_PULL_MAX_NODE_REIMAGES:-5} GANTRY_NAMESPACE=${GANTRY_NAMESPACE:-gantry-system} BENCHMARK_NAMESPACE=${BENCHMARK_NAMESPACE:-gantry-benchmark} @@ -125,8 +125,8 @@ GANTRY_ACR_DATA_HOST=${GANTRY_ACR_NAME}.${AZURE_LOCATION}.data.azurecr.io echo "BENCHMARK_IMAGE_LAYERS cannot exceed BENCHMARK_IMAGE_SIZE_MIB" >&2 exit 2 } -[[ "$BASELINE_PULL_MAX_NODE_RESTARTS" =~ ^[1-9][0-9]*$ ]] || { - echo "BASELINE_PULL_MAX_NODE_RESTARTS must be positive" >&2 +[[ "$BASELINE_PULL_MAX_NODE_REIMAGES" =~ ^[1-9][0-9]*$ ]] || { + echo "BASELINE_PULL_MAX_NODE_REIMAGES must be positive" >&2 exit 2 } for acr_name in "$BASELINE_ACR_NAME" "$GANTRY_ACR_NAME"; do @@ -759,7 +759,7 @@ GUARD kubectl -n "$GANTRY_NAMESPACE" rollout status daemonset/gantry-acr-private-dns-guard --timeout=30m } -restart_private_pull_tls_nodes() { +reimage_private_pull_tls_nodes() { local pods_json node provider_id vmss instance_id old_boot_id local -a nodes pods_json=$(kubectl -n "$GANTRY_NAMESPACE" get pods \ @@ -768,8 +768,8 @@ restart_private_pull_tls_nodes() { select(any(.status.containerStatuses[]?; ((.state.waiting.message? // "") | contains("TLS handshake timeout")))) | .spec.nodeName' <<<"$pods_json" | sort -u) ((${#nodes[@]} > 0)) || return 1 - ((${#nodes[@]} <= BASELINE_PULL_MAX_NODE_RESTARTS)) || { - echo "refusing to restart ${#nodes[@]} nodes with ACR TLS handshake timeouts; limit is $BASELINE_PULL_MAX_NODE_RESTARTS" >&2 + ((${#nodes[@]} <= BASELINE_PULL_MAX_NODE_REIMAGES)) || { + echo "refusing to reimage ${#nodes[@]} nodes with ACR TLS handshake timeouts; limit is $BASELINE_PULL_MAX_NODE_REIMAGES" >&2 return 1 } @@ -782,8 +782,8 @@ restart_private_pull_tls_nodes() { return 1 } old_boot_id=$(kubectl get node "$node" -o jsonpath='{.status.nodeInfo.bootID}') - log "restarting $node (VMSS $vmss instance $instance_id) after ACR Private Endpoint TLS timeouts" - az vmss restart -g "$AZURE_NODE_RESOURCE_GROUP" -n "$vmss" \ + log "reimaging $node (VMSS $vmss instance $instance_id) after ACR Private Endpoint TLS timeouts" + az vmss reimage -g "$AZURE_NODE_RESOURCE_GROUP" -n "$vmss" \ --instance-ids "$instance_id" --only-show-errors -o none local attempt current_boot_id ready @@ -797,7 +797,7 @@ restart_private_pull_tls_nodes() { sleep 10 done [[ -n "$current_boot_id" && "$current_boot_id" != "$old_boot_id" && "$ready" == true ]] || { - echo "$node did not return Ready with a new boot ID after restart" >&2 + echo "$node did not return Ready with a new boot ID after reimage" >&2 return 1 } kubectl -n "$GANTRY_NAMESPACE" delete pod \ @@ -853,7 +853,7 @@ PROBE ready=true break fi - if [[ "$repair_attempted" == false ]] && restart_private_pull_tls_nodes; then + if [[ "$repair_attempted" == false ]] && reimage_private_pull_tls_nodes; then repair_attempted=true continue fi From 7cb5375d5ecbed433fdc84f44dbf336abae2763c Mon Sep 17 00:00:00 2001 From: Vaibhav Patel Date: Wed, 5 Aug 2026 16:13:51 -0400 Subject: [PATCH 22/60] fix(gantry): replace persistent private pull failures --- hack/gantry-benchmark/deploy.sh | 81 +++++++++++++++++++-------------- 1 file changed, 48 insertions(+), 33 deletions(-) diff --git a/hack/gantry-benchmark/deploy.sh b/hack/gantry-benchmark/deploy.sh index 1ea5d3563..54e98a2a4 100755 --- a/hack/gantry-benchmark/deploy.sh +++ b/hack/gantry-benchmark/deploy.sh @@ -81,7 +81,7 @@ BENCHMARK_IMAGE_SIZE_MIB=${BENCHMARK_IMAGE_SIZE_MIB:-40960} BENCHMARK_IMAGE_LAYERS=${BENCHMARK_IMAGE_LAYERS:-40} BENCHMARK_MINIMUM_BYTE_REDUCTION=${BENCHMARK_MINIMUM_BYTE_REDUCTION:-0.90} BENCHMARK_MAXIMUM_LATENCY_RATIO=${BENCHMARK_MAXIMUM_LATENCY_RATIO:-1.0} -BASELINE_PULL_MAX_NODE_REIMAGES=${BASELINE_PULL_MAX_NODE_REIMAGES:-5} +BASELINE_PULL_MAX_NODE_REPLACEMENTS=${BASELINE_PULL_MAX_NODE_REPLACEMENTS:-5} GANTRY_NAMESPACE=${GANTRY_NAMESPACE:-gantry-system} BENCHMARK_NAMESPACE=${BENCHMARK_NAMESPACE:-gantry-benchmark} @@ -125,8 +125,8 @@ GANTRY_ACR_DATA_HOST=${GANTRY_ACR_NAME}.${AZURE_LOCATION}.data.azurecr.io echo "BENCHMARK_IMAGE_LAYERS cannot exceed BENCHMARK_IMAGE_SIZE_MIB" >&2 exit 2 } -[[ "$BASELINE_PULL_MAX_NODE_REIMAGES" =~ ^[1-9][0-9]*$ ]] || { - echo "BASELINE_PULL_MAX_NODE_REIMAGES must be positive" >&2 +[[ "$BASELINE_PULL_MAX_NODE_REPLACEMENTS" =~ ^[1-9][0-9]*$ ]] || { + echo "BASELINE_PULL_MAX_NODE_REPLACEMENTS must be positive" >&2 exit 2 } for acr_name in "$BASELINE_ACR_NAME" "$GANTRY_ACR_NAME"; do @@ -759,8 +759,8 @@ GUARD kubectl -n "$GANTRY_NAMESPACE" rollout status daemonset/gantry-acr-private-dns-guard --timeout=30m } -reimage_private_pull_tls_nodes() { - local pods_json node provider_id vmss instance_id old_boot_id +replace_private_pull_tls_nodes() { + local pods_json node provider_id machine_name current_count local -a nodes pods_json=$(kubectl -n "$GANTRY_NAMESPACE" get pods \ -l app.kubernetes.io/name=gantry-baseline-acr-pull-probe -o json) @@ -768,42 +768,57 @@ reimage_private_pull_tls_nodes() { select(any(.status.containerStatuses[]?; ((.state.waiting.message? // "") | contains("TLS handshake timeout")))) | .spec.nodeName' <<<"$pods_json" | sort -u) ((${#nodes[@]} > 0)) || return 1 - ((${#nodes[@]} <= BASELINE_PULL_MAX_NODE_REIMAGES)) || { - echo "refusing to reimage ${#nodes[@]} nodes with ACR TLS handshake timeouts; limit is $BASELINE_PULL_MAX_NODE_REIMAGES" >&2 + ((${#nodes[@]} <= BASELINE_PULL_MAX_NODE_REPLACEMENTS)) || { + echo "refusing to replace ${#nodes[@]} nodes with ACR TLS handshake timeouts; limit is $BASELINE_PULL_MAX_NODE_REPLACEMENTS" >&2 return 1 } for node in "${nodes[@]}"; do provider_id=$(kubectl get node "$node" -o jsonpath='{.spec.providerID}') - vmss=$(sed -n 's#^.*/virtualMachineScaleSets/\([^/]*\)/virtualMachines/[^/]*$#\1#p' <<<"$provider_id") - instance_id=$(sed -n 's#^.*/virtualMachineScaleSets/[^/]*/virtualMachines/\([^/]*\)$#\1#p' <<<"$provider_id") - [[ -n "$vmss" && -n "$instance_id" ]] || { - echo "cannot parse VMSS instance from provider ID for $node: $provider_id" >&2 + machine_name=$(az aks machine list -g "$AZURE_RESOURCE_GROUP" \ + --cluster-name "$AZURE_AKS_CLUSTER_NAME" --nodepool-name "$AKS_NODE_POOL_NAME" -o json | \ + jq -r --arg resource_id "${provider_id#azure://}" \ + '.[] | select((.properties.resourceId | ascii_downcase) == ($resource_id | ascii_downcase)) | .name') + [[ -n "$machine_name" ]] || { + echo "cannot resolve AKS machine for $node provider ID $provider_id" >&2 return 1 } - old_boot_id=$(kubectl get node "$node" -o jsonpath='{.status.nodeInfo.bootID}') - log "reimaging $node (VMSS $vmss instance $instance_id) after ACR Private Endpoint TLS timeouts" - az vmss reimage -g "$AZURE_NODE_RESOURCE_GROUP" -n "$vmss" \ - --instance-ids "$instance_id" --only-show-errors -o none - - local attempt current_boot_id ready - for attempt in $(seq 1 90); do - current_boot_id=$(kubectl get node "$node" -o jsonpath='{.status.nodeInfo.bootID}' 2>/dev/null || true) - ready=$(kubectl get node "$node" -o json | \ - jq -r 'any(.status.conditions[]; .type == "Ready" and .status == "True")' 2>/dev/null || true) - if [[ -n "$current_boot_id" && "$current_boot_id" != "$old_boot_id" && "$ready" == true ]]; then - break - fi - sleep 10 + log "replacing $node (AKS machine $machine_name) after persistent ACR Private Endpoint TLS timeouts" + az aks nodepool delete-machines -g "$AZURE_RESOURCE_GROUP" \ + --cluster-name "$AZURE_AKS_CLUSTER_NAME" -n "$AKS_NODE_POOL_NAME" \ + --machine-names "$machine_name" --only-show-errors -o none + done + + current_count=$(az aks nodepool show -g "$AZURE_RESOURCE_GROUP" \ + --cluster-name "$AZURE_AKS_CLUSTER_NAME" -n "$AKS_NODE_POOL_NAME" --query count -o tsv) + if [[ "$current_count" != "$AKS_NODE_COUNT" ]]; then + log "restoring AKS node pool count from $current_count to $AKS_NODE_COUNT" + az aks nodepool scale -g "$AZURE_RESOURCE_GROUP" --cluster-name "$AZURE_AKS_CLUSTER_NAME" \ + -n "$AKS_NODE_POOL_NAME" --node-count "$AKS_NODE_COUNT" --only-show-errors -o none + fi + + local attempt total ready old_nodes_remaining + for attempt in $(seq 1 180); do + total=$(kubectl get nodes -l "agentpool=$AKS_NODE_POOL_NAME" -o json | jq '.items | length') + ready=$(kubectl get nodes -l "agentpool=$AKS_NODE_POOL_NAME" -o json | \ + jq '[.items[] | select(any(.status.conditions[]; .type == "Ready" and .status == "True"))] | length') + old_nodes_remaining=0 + for node in "${nodes[@]}"; do + kubectl get node "$node" >/dev/null 2>&1 && ((old_nodes_remaining += 1)) done - [[ -n "$current_boot_id" && "$current_boot_id" != "$old_boot_id" && "$ready" == true ]] || { - echo "$node did not return Ready with a new boot ID after reimage" >&2 - return 1 - } - kubectl -n "$GANTRY_NAMESPACE" delete pod \ - -l app.kubernetes.io/name=gantry-baseline-acr-pull-probe \ - --field-selector "spec.nodeName=$node" --wait=false + if [[ "$total" == "$AKS_NODE_COUNT" && "$ready" == "$AKS_NODE_COUNT" && "$old_nodes_remaining" == 0 ]]; then + break + fi + sleep 10 done + [[ "$total" == "$AKS_NODE_COUNT" && "$ready" == "$AKS_NODE_COUNT" && "$old_nodes_remaining" == 0 ]] || { + echo "AKS node replacement did not restore $AKS_NODE_COUNT Ready nodes or remove all failed nodes" >&2 + return 1 + } + kubectl -n "$GANTRY_NAMESPACE" rollout status \ + daemonset/gantry-benchmark-containerd-config --timeout=30m + kubectl -n "$GANTRY_NAMESPACE" rollout status \ + daemonset/gantry-acr-private-dns-guard --timeout=30m } verify_private_baseline_pull() { @@ -853,7 +868,7 @@ PROBE ready=true break fi - if [[ "$repair_attempted" == false ]] && reimage_private_pull_tls_nodes; then + if [[ "$repair_attempted" == false ]] && replace_private_pull_tls_nodes; then repair_attempted=true continue fi From 29f6a2104b667a32fb3c45f98d47cd43fc6d51c4 Mon Sep 17 00:00:00 2001 From: Vaibhav Patel Date: Wed, 5 Aug 2026 16:48:22 -0400 Subject: [PATCH 23/60] fix(gantry): preserve benchmark start request --- hack/gantry-benchmark/Makefile | 1 + hack/gantry-benchmark/deploy.sh | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hack/gantry-benchmark/Makefile b/hack/gantry-benchmark/Makefile index 5ef4f7beb..2ee15ad28 100644 --- a/hack/gantry-benchmark/Makefile +++ b/hack/gantry-benchmark/Makefile @@ -39,6 +39,7 @@ operator-vm-check: ./deploy.sh plan deploy.env.example >/dev/null ! grep -Eq 'az acr login|podman (build|push|login|pull|tag)' deploy.sh ! grep -Eq '^[[:space:]]*az login([[:space:]]|$$)' deploy.sh + ! grep -Eq '^[[:space:]]*export START_BENCHMARK=false' deploy.sh deploy: operator-vm-check cd "$(REPO_ROOT)" && hack/gantry-benchmark/deploy.sh deploy "$(DEPLOY_CONFIG)" diff --git a/hack/gantry-benchmark/deploy.sh b/hack/gantry-benchmark/deploy.sh index 54e98a2a4..b6d2f3e66 100755 --- a/hack/gantry-benchmark/deploy.sh +++ b/hack/gantry-benchmark/deploy.sh @@ -909,13 +909,12 @@ provision_operator() { export BENCHMARK_SOURCE_IMAGE=$SOURCE_IMAGE BENCHMARK_SOURCE_REVISION=$source_revision export BENCHMARK_NODE_COUNT BENCHMARK_IMAGE_SIZE_MIB BENCHMARK_IMAGE_LAYERS export BENCHMARK_AZURE_TELEMETRY=true BENCHMARK_MINIMUM_BYTE_REDUCTION BENCHMARK_MAXIMUM_LATENCY_RATIO - export START_BENCHMARK=false AZURE_BASELINE_ACR_PRIVATE_ENDPOINT_RESOURCE_ID=$(az network private-endpoint show \ -g "$AZURE_RESOURCE_GROUP" -n "$BASELINE_PRIVATE_ENDPOINT_NAME" --query id -o tsv) AZURE_GANTRY_ACR_PRIVATE_ENDPOINT_RESOURCE_ID=$(az network private-endpoint show \ -g "$AZURE_RESOURCE_GROUP" -n "$GANTRY_PRIVATE_ENDPOINT_NAME" --query id -o tsv) export AZURE_BASELINE_ACR_PRIVATE_ENDPOINT_RESOURCE_ID AZURE_GANTRY_ACR_PRIVATE_ENDPOINT_RESOURCE_ID - "$repo_root/hack/gantry-benchmark/operator-vm-provision.sh" + START_BENCHMARK=false "$repo_root/hack/gantry-benchmark/operator-vm-provision.sh" assert_equal "operator VM size" \ "$(az vm show -g "$AZURE_RESOURCE_GROUP" -n "$OPERATOR_VM_NAME" --query hardwareProfile.vmSize -o tsv)" \ From f37910d3560eaf0a1ee271fea8bd4de128acef97 Mon Sep 17 00:00:00 2001 From: Vaibhav Patel Date: Wed, 5 Aug 2026 17:31:38 -0400 Subject: [PATCH 24/60] feat(gantry): show detailed benchmark progress --- hack/gantry-benchmark/README.md | 11 ++- hack/gantry-benchmark/operator-vm-status.sh | 94 ++++++++++++++++++++- hack/gantry-benchmark/operator-vm-watch.sh | 4 +- 3 files changed, 100 insertions(+), 9 deletions(-) diff --git a/hack/gantry-benchmark/README.md b/hack/gantry-benchmark/README.md index bce60f959..19ea128db 100644 --- a/hack/gantry-benchmark/README.md +++ b/hack/gantry-benchmark/README.md @@ -207,10 +207,13 @@ is the default status transport. SSH is optional only when the operator has deliberately provided private network connectivity to the VM. The live view reports the lifecycle stage and start time, immutable run shape, -payload files/bytes/percentage, active Podman build or push, VM disk usage, -Kubernetes Job completion, Gantry readiness, recent logs, and the final report. -Use `operator-vm-status` for a single snapshot. Override the refresh cadence -with `WATCH_INTERVAL_SECONDS` (default 30). +payload files/bytes/percentage, each baseline and Gantry-cold image reference, +image size, layer count, build/push state, completed digest, active image +operation and elapsed time, per-phase Kubernetes Job completion, Gantry +readiness, recent logs, and the final report. Podman 4.9 does not expose a +machine-readable live push byte percentage, so the view reports that limitation +instead of estimating it. Use `operator-vm-status` for a single snapshot. +Override the refresh cadence with `WATCH_INTERVAL_SECONDS` (default 30). Artifacts persist on the VM under `/var/lib/gantry-benchmark/artifacts//`; `latest` points at the newest diff --git a/hack/gantry-benchmark/operator-vm-status.sh b/hack/gantry-benchmark/operator-vm-status.sh index 94e0a5656..b99d42c4b 100755 --- a/hack/gantry-benchmark/operator-vm-status.sh +++ b/hack/gantry-benchmark/operator-vm-status.sh @@ -51,6 +51,68 @@ if [[ -z "$run_id" ]]; then run_id=$(jq -r '.run_id // empty' <<<"${last_run_json:-{}}" 2>/dev/null || true) fi +build_process=$(pgrep -af 'podman (pull|build|push|create|cp)' 2>/dev/null || true) +completed_image_steps=0 +active_image_operation="" + +print_prepared_image_status() { + local label=$1 + local phase=$2 + local target=$3 + local build_dir=$4 + local digest_file="$build_dir/push-digest.$phase.txt" + local build_state=waiting + local push_state=waiting + local digest="" + local layers="unknown" + local image_bytes="unknown" + local image_size="unknown" + local active_pid="" + local elapsed="" + + if podman image exists "$target" 2>/dev/null; then + build_state=complete + layers=$(podman image inspect --format '{{ len .RootFS.Layers }}' "$target" 2>/dev/null || printf 'unknown') + image_bytes=$(podman image inspect --format '{{ .Size }}' "$target" 2>/dev/null || printf 'unknown') + if [[ "$image_bytes" =~ ^[0-9]+$ ]]; then + image_size=$(numfmt --to=iec-i --suffix=B "$image_bytes" 2>/dev/null || printf '%s bytes' "$image_bytes") + fi + fi + if grep -Fq "podman build" <<<"$build_process" && grep -Fq -- "--tag $target" <<<"$build_process"; then + build_state=active + active_pid=$(awk -v target="$target" 'index($0, "podman build") && index($0, target) {print $1; exit}' <<<"$build_process") + active_image_operation="$label build" + fi + if grep -Fq "podman push" <<<"$build_process" && grep -Fq "$target" <<<"$build_process"; then + build_state=complete + push_state=active + active_pid=$(awk -v target="$target" 'index($0, "podman push") && index($0, target) {print $1; exit}' <<<"$build_process") + active_image_operation="$label push" + fi + if [[ -s "$digest_file" ]]; then + digest=$(tr -d '[:space:]' <"$digest_file") + build_state=complete + push_state=complete + fi + if [[ -n "$active_pid" ]]; then + elapsed=$(ps -o etime= -p "$active_pid" 2>/dev/null | xargs) + fi + + [[ "$build_state" == complete ]] && ((completed_image_steps += 1)) + [[ "$push_state" == complete ]] && ((completed_image_steps += 1)) + + printf '%s image:\n' "$label" + printf ' target: %s\n' "$target" + printf ' size: %s (%s bytes)\n' "$image_size" "$image_bytes" + printf ' build: %s (%s image layers)\n' "$build_state" "$layers" + printf ' push: %s' "$push_state" + [[ -z "$elapsed" ]] || printf ' (elapsed %s)' "$elapsed" + printf '\n' + if [[ -n "$digest" ]]; then + printf ' digest: %s\n' "$digest" + fi +} + printf '=== Gantry benchmark operator ===\n' printf 'time: %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" printf 'service: %s\n' "$service_state" @@ -86,10 +148,22 @@ if [[ -n "$run_id" ]]; then printf 'payload generation: %d/%s files, %d%% (%s bytes)\n' \ "$payload_files" "$payload_layers" "$percent" "$payload_bytes" fi + + image_tag=${run_id//_/-} + workload_repository=${BENCHMARK_WORKLOAD_REPOSITORY:-gantry-benchmark-pull} + printf '\n=== Image preparation ===\n' + print_prepared_image_status \ + baseline baseline "$BASELINE_ACR_LOGIN_SERVER/$workload_repository:$image_tag" "$build_dir" + print_prepared_image_status \ + Gantry-cold gantry_cold "$GANTRY_ACR_LOGIN_SERVER/$workload_repository:$image_tag" "$build_dir" + printf 'image steps: %d/4 complete\n' "$completed_image_steps" + [[ -z "$active_image_operation" ]] || printf 'active image operation: %s\n' "$active_image_operation" + if [[ "$active_image_operation" == *" push" ]]; then + printf 'push byte progress: unavailable from Podman 4.9; total image size and elapsed time shown above\n' + fi fi fi -build_process=$(pgrep -af 'podman (pull|build|push|create|cp)' 2>/dev/null || true) if [[ -n "$build_process" ]]; then printf 'image process:\n%s\n' "$build_process" fi @@ -106,9 +180,21 @@ podman system df 2>/dev/null || true if [[ -f "$KUBECONFIG" ]]; then printf '\n=== Kubernetes ===\n' - kubectl -n "$NAMESPACE" get jobs \ - -o custom-columns=NAME:.metadata.name,ACTIVE:.status.active,SUCCEEDED:.status.succeeded,FAILED:.status.failed,COMPLETIONS:.spec.completions \ - 2>/dev/null || true + jobs_json=$(kubectl -n "$NAMESPACE" get jobs -o json 2>/dev/null || true) + if [[ -n "$jobs_json" ]]; then + jq -r --argjson now "$(date -u +%s)" ' + .items[] | + (.spec.completions // 1) as $desired | + (.status.succeeded // 0) as $succeeded | + (.status.active // 0) as $active | + (.status.failed // 0) as $failed | + (if $desired > 0 then (($succeeded * 100 / $desired) | floor) else 0 end) as $percent | + (if .status.startTime then ($now - (.status.startTime | fromdateiso8601)) else 0 end) as $elapsed | + "phase: \(.metadata.name | sub("^gantry-benchmark-"; "") | split("-run-")[0])\n" + + " job: \(.metadata.name)\n" + + " pods: \($succeeded)/\($desired) complete (\($percent)%), \($active) active, \($failed) failed\n" + + " elapsed: \($elapsed)s"' <<<"$jobs_json" 2>/dev/null || true + fi kubectl -n "$GANTRY_NS" get daemonset gantry \ -o custom-columns=DESIRED:.status.desiredNumberScheduled,READY:.status.numberReady,UPDATED:.status.updatedNumberScheduled,AVAILABLE:.status.numberAvailable \ 2>/dev/null || true diff --git a/hack/gantry-benchmark/operator-vm-watch.sh b/hack/gantry-benchmark/operator-vm-watch.sh index a80469ffd..ff724ac23 100755 --- a/hack/gantry-benchmark/operator-vm-watch.sh +++ b/hack/gantry-benchmark/operator-vm-watch.sh @@ -4,6 +4,8 @@ set -Eeuo pipefail +script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) + AZURE_RESOURCE_GROUP="${AZURE_RESOURCE_GROUP:-}" OPERATOR_VM_NAME="${OPERATOR_VM_NAME:-gantry-benchmark-operator}" OPERATOR_SSH_HOST="${OPERATOR_SSH_HOST:-}" @@ -71,7 +73,7 @@ status_once() { -g "$AZURE_RESOURCE_GROUP" \ -n "$OPERATOR_VM_NAME" \ --command-id RunShellScript \ - --scripts '/opt/gantry-benchmark/unbounded/hack/gantry-benchmark/operator-vm-status.sh' \ + --scripts @"$script_dir/operator-vm-status.sh" \ --only-show-errors \ --query 'value[0].message' \ -o tsv) From 1fe1a51a0592dc6bb571e6a05679595bd772eab9 Mon Sep 17 00:00:00 2001 From: Vaibhav Patel Date: Wed, 5 Aug 2026 19:16:58 -0400 Subject: [PATCH 25/60] fix(gantry): bound containerd telemetry capture --- .../gantry-benchmark/performance_telemetry.go | 78 +++++++++++++------ .../performance_telemetry_test.go | 32 ++++++++ 2 files changed, 87 insertions(+), 23 deletions(-) diff --git a/hack/cmd/gantry-benchmark/performance_telemetry.go b/hack/cmd/gantry-benchmark/performance_telemetry.go index 79679f155..74bbaea02 100644 --- a/hack/cmd/gantry-benchmark/performance_telemetry.go +++ b/hack/cmd/gantry-benchmark/performance_telemetry.go @@ -13,7 +13,11 @@ import ( "time" ) -const performanceTelemetryStep = 10 * time.Second +const ( + performanceTelemetryStep = 10 * time.Second + grpcHandledTelemetryStep = 5 * time.Minute + maxPrometheusRangeResponseBytes = 256 * 1024 * 1024 +) type prometheusRangeCapture struct { Name string `json:"name"` @@ -22,6 +26,12 @@ type prometheusRangeCapture struct { Response json.RawMessage `json:"response"` } +type performanceTelemetryQuery struct { + name string + query string + step time.Duration +} + type containerdJournalEvent struct { ObserverPod string `json:"observer_pod"` NodeName string `json:"node_name"` @@ -43,25 +53,8 @@ type phasePerformanceTelemetry struct { var journalFieldPattern = regexp.MustCompile(`(?:^|[[:space:]])([a-zA-Z_]+)="?([^"[:space:]]+)"?`) -func (b *benchmark) capturePhasePerformanceTelemetry( - ctx context.Context, - phase proxyPhase, - job jobObservation, -) (phasePerformanceTelemetry, error) { - window := telemetryWindow{ - StartedAt: job.PhaseStartedAt, - FinishedAt: job.PhaseFinishedAt, - } - - observerPods, err := b.observerPodNodes(ctx) - if err != nil { - return phasePerformanceTelemetry{}, err - } - - queries := []struct { - name string - query string - }{ +func performanceTelemetryQueries() []performanceTelemetryQuery { + return []performanceTelemetryQuery{ {name: "node_disk_read_bytes_per_second", query: `rate(node_disk_read_bytes_total{gantry_benchmark="true"}[30s])`}, {name: "node_disk_written_bytes_per_second", query: `rate(node_disk_written_bytes_total{gantry_benchmark="true"}[30s])`}, {name: "node_disk_busy_ratio", query: `rate(node_disk_io_time_seconds_total{gantry_benchmark="true"}[30s])`}, @@ -77,7 +70,9 @@ func (b *benchmark) capturePhasePerformanceTelemetry( {name: "node_cpu_busy_ratio", query: `1 - avg by(pod) (rate(node_cpu_seconds_total{gantry_benchmark="true",mode="idle"}[30s]))`}, {name: "node_memory_available_bytes", query: `node_memory_MemAvailable_bytes{gantry_benchmark="true"}`}, {name: "containerd_process", query: `{__name__=~"process_(cpu_seconds_total|resident_memory_bytes|virtual_memory_bytes)",gantry_benchmark="true",endpoint="ctr-metrics"}`}, - {name: "containerd_metrics", query: `{__name__=~"containerd_.*|grpc_server_.*",gantry_benchmark="true"}`}, + {name: "containerd_image_pulls", query: `{__name__=~"containerd_cri_sandboxed_(image_pulls_total|in_progress_image_pulls_total|image_pulling_throughput_(sum|count))",gantry_benchmark="true"}`}, + {name: "containerd_grpc_started", query: `sum by (pod, grpc_service, grpc_method) (rate(grpc_server_started_total{gantry_benchmark="true"}[30s])) > 0`}, + {name: "containerd_grpc_handled", query: `sum by (pod, grpc_code) (rate(grpc_server_handled_total{gantry_benchmark="true"}[5m])) > 0`, step: grpcHandledTelemetryStep}, {name: "gantry_peer_outcomes", query: `p2p_peer_fetch_total{gantry_benchmark="true"}`}, {name: "gantry_peer_busy_stall_timestamps", query: `gantry_peer_fetch_last_timestamp_seconds{outcome=~"busy|stall",gantry_benchmark="true"}`}, {name: "gantry_peer_duration", query: `{__name__=~"p2p_peer_fetch_duration_seconds_(bucket|sum|count)",outcome=~"busy|stall",gantry_benchmark="true"}`}, @@ -87,10 +82,32 @@ func (b *benchmark) capturePhasePerformanceTelemetry( {name: "gantry_response_completed", query: `gantry_mirror_response_completed_timestamp_seconds{kind="layer",gantry_benchmark="true"}`}, {name: "gantry_commit_observation", query: `{__name__=~"gantry_containerd_commit_(observed_total|observed_timestamp_seconds|observation_duration_seconds_(sum|count)|latest_observation_duration_seconds|missing_after_stream_total)",gantry_benchmark="true"}`}, } +} + +func (b *benchmark) capturePhasePerformanceTelemetry( + ctx context.Context, + phase proxyPhase, + job jobObservation, +) (phasePerformanceTelemetry, error) { + window := telemetryWindow{ + StartedAt: job.PhaseStartedAt, + FinishedAt: job.PhaseFinishedAt, + } + + observerPods, err := b.observerPodNodes(ctx) + if err != nil { + return phasePerformanceTelemetry{}, err + } + + queries := performanceTelemetryQueries() captures := make([]prometheusRangeCapture, 0, len(queries)) for _, item := range queries { - response, err := b.queryPrometheusRange(ctx, item.query, window, performanceTelemetryStep) + step := item.step + if step == 0 { + step = performanceTelemetryStep + } + response, err := b.queryPrometheusRange(ctx, item.query, window, step) if err != nil { return phasePerformanceTelemetry{}, fmt.Errorf("capture %s: %w", item.name, err) } @@ -100,7 +117,7 @@ func (b *benchmark) capturePhasePerformanceTelemetry( captures = append(captures, prometheusRangeCapture{ Name: item.name, Query: item.query, - StepSeconds: int(performanceTelemetryStep.Seconds()), + StepSeconds: int(step.Seconds()), Response: response, }) } @@ -274,6 +291,9 @@ func (b *benchmark) queryPrometheusRange( if err != nil { return nil, err } + if err := validatePrometheusRangeResponseSize(output, maxPrometheusRangeResponseBytes); err != nil { + return nil, err + } var envelope struct { Status string `json:"status"` @@ -288,6 +308,18 @@ func (b *benchmark) queryPrometheusRange( return json.RawMessage(output), nil } +func validatePrometheusRangeResponseSize(output []byte, limit int) error { + if len(output) > limit { + return fmt.Errorf( + "Prometheus range response is %d bytes, exceeds %d-byte capture limit", + len(output), + limit, + ) + } + + return nil +} + func (b *benchmark) observerPodNodes(ctx context.Context) (map[string]string, error) { output, err := b.commands.Run( ctx, diff --git a/hack/cmd/gantry-benchmark/performance_telemetry_test.go b/hack/cmd/gantry-benchmark/performance_telemetry_test.go index b7ac1e230..eca974eaf 100644 --- a/hack/cmd/gantry-benchmark/performance_telemetry_test.go +++ b/hack/cmd/gantry-benchmark/performance_telemetry_test.go @@ -60,6 +60,38 @@ func TestQueryPrometheusRange(t *testing.T) { } } +func TestPerformanceTelemetryQueriesBoundContainerdCardinality(t *testing.T) { + queries := performanceTelemetryQueries() + byName := make(map[string]performanceTelemetryQuery, len(queries)) + for _, query := range queries { + byName[query.name] = query + if strings.Contains(query.query, `containerd_.*|grpc_server_.*`) { + t.Fatalf("query %q uses unbounded containerd and gRPC selector", query.name) + } + } + + for _, name := range []string{"containerd_image_pulls", "containerd_grpc_started", "containerd_grpc_handled"} { + if _, ok := byName[name]; !ok { + t.Fatalf("missing bounded telemetry query %q", name) + } + } + if byName["containerd_image_pulls"].step != 0 || byName["containerd_grpc_started"].step != 0 { + t.Fatal("containerd pull and gRPC started queries must use the default 10-second step") + } + if byName["containerd_grpc_handled"].step != 5*time.Minute { + t.Fatalf("gRPC handled step = %s, want 5m", byName["containerd_grpc_handled"].step) + } +} + +func TestValidatePrometheusRangeResponseSize(t *testing.T) { + if err := validatePrometheusRangeResponseSize([]byte("1234"), 4); err != nil { + t.Fatalf("response at limit: %v", err) + } + if err := validatePrometheusRangeResponseSize([]byte("12345"), 4); err == nil || !strings.Contains(err.Error(), "5 bytes") { + t.Fatalf("oversized response error = %v", err) + } +} + func TestParseContainerdJournal(t *testing.T) { window := telemetryWindow{ StartedAt: time.Date(2026, time.August, 4, 1, 2, 3, 0, time.UTC), From a454ffcb0c273a06ad45b01885283e5e8126d7f8 Mon Sep 17 00:00:00 2001 From: Vaibhav Patel Date: Wed, 5 Aug 2026 19:20:23 -0400 Subject: [PATCH 26/60] feat(gantry): adopt retained benchmark images --- hack/gantry-benchmark/deploy.env.example | 5 +++ hack/gantry-benchmark/deploy.sh | 40 +++++++++++++++++ .../gantry-benchmark/operator-vm-bootstrap.sh | 44 ++++++++++++++++++- .../gantry-benchmark/operator-vm-provision.sh | 6 +++ 4 files changed, 93 insertions(+), 2 deletions(-) diff --git a/hack/gantry-benchmark/deploy.env.example b/hack/gantry-benchmark/deploy.env.example index bca95e3a8..ed23c1d8f 100644 --- a/hack/gantry-benchmark/deploy.env.example +++ b/hack/gantry-benchmark/deploy.env.example @@ -31,6 +31,11 @@ BENCHMARK_IMAGE_LAYERS="40" BENCHMARK_MINIMUM_BYTE_REDUCTION="0.90" BENCHMARK_MAXIMUM_LATENCY_RATIO="1.0" +# Optional all-or-none set to reuse an identical-payload image pair from a retained run. +ADOPT_BASELINE_IMAGE="" +ADOPT_GANTRY_IMAGE="" +ADOPT_PAYLOAD_SHA256="" + OPERATOR_VM_SIZE="Standard_D32ds_v5" OPERATOR_VM_ZONE="1" OPERATOR_BUILD_DISK_GB="512" diff --git a/hack/gantry-benchmark/deploy.sh b/hack/gantry-benchmark/deploy.sh index b6d2f3e66..0048c6492 100755 --- a/hack/gantry-benchmark/deploy.sh +++ b/hack/gantry-benchmark/deploy.sh @@ -81,6 +81,9 @@ BENCHMARK_IMAGE_SIZE_MIB=${BENCHMARK_IMAGE_SIZE_MIB:-40960} BENCHMARK_IMAGE_LAYERS=${BENCHMARK_IMAGE_LAYERS:-40} BENCHMARK_MINIMUM_BYTE_REDUCTION=${BENCHMARK_MINIMUM_BYTE_REDUCTION:-0.90} BENCHMARK_MAXIMUM_LATENCY_RATIO=${BENCHMARK_MAXIMUM_LATENCY_RATIO:-1.0} +ADOPT_BASELINE_IMAGE=${ADOPT_BASELINE_IMAGE:-} +ADOPT_GANTRY_IMAGE=${ADOPT_GANTRY_IMAGE:-} +ADOPT_PAYLOAD_SHA256=${ADOPT_PAYLOAD_SHA256:-} BASELINE_PULL_MAX_NODE_REPLACEMENTS=${BASELINE_PULL_MAX_NODE_REPLACEMENTS:-5} GANTRY_NAMESPACE=${GANTRY_NAMESPACE:-gantry-system} @@ -139,6 +142,34 @@ done echo "START_BENCHMARK must be true or false" >&2 exit 2 } +valid_adopted_image() { + local image=$1 + local login_server=$2 + local prefix="$login_server/gantry-benchmark-pull@" + [[ "$image" == "$prefix"* && "${image#"$prefix"}" =~ ^sha256:[0-9a-f]{64}$ ]] +} +adoption_values=0 +for value in "$ADOPT_BASELINE_IMAGE" "$ADOPT_GANTRY_IMAGE" "$ADOPT_PAYLOAD_SHA256"; do + [[ -z "$value" ]] || adoption_values=$((adoption_values + 1)) +done +if ((adoption_values != 0 && adoption_values != 3)); then + echo "ADOPT_BASELINE_IMAGE, ADOPT_GANTRY_IMAGE, and ADOPT_PAYLOAD_SHA256 must be set together" >&2 + exit 2 +fi +if ((adoption_values == 3)); then + valid_adopted_image "$ADOPT_BASELINE_IMAGE" "$BASELINE_ACR_LOGIN_SERVER" || { + echo "ADOPT_BASELINE_IMAGE must be an immutable gantry-benchmark-pull image in $BASELINE_ACR_LOGIN_SERVER" >&2 + exit 2 + } + valid_adopted_image "$ADOPT_GANTRY_IMAGE" "$GANTRY_ACR_LOGIN_SERVER" || { + echo "ADOPT_GANTRY_IMAGE must be an immutable gantry-benchmark-pull image in $GANTRY_ACR_LOGIN_SERVER" >&2 + exit 2 + } + [[ "$ADOPT_PAYLOAD_SHA256" =~ ^sha256:[0-9a-f]{64}$ ]] || { + echo "ADOPT_PAYLOAD_SHA256 must be a sha256 digest" >&2 + exit 2 + } +fi assert_default() { local name=$1 local actual=$2 @@ -178,6 +209,10 @@ retry_command() { } print_plan() { + local image_preparation="build and push fresh workload images" + if ((adoption_values == 3)); then + image_preparation="adopt existing immutable workload images" + fi cat < \ \ \ \ - + \ + USAGE } -[[ $# -eq 20 ]] || { usage >&2; exit 2; } +[[ $# -eq 23 ]] || { usage >&2; exit 2; } subscription_id=$1 resource_group=$2 @@ -36,6 +37,21 @@ build_disk_lun=${17} build_mount=${18} source_image=${19} source_revision=${20} +adopt_baseline_image=${21} +adopt_gantry_image=${22} +adopt_payload_sha256=${23} +[[ "$adopt_baseline_image" != - ]] || adopt_baseline_image="" +[[ "$adopt_gantry_image" != - ]] || adopt_gantry_image="" +[[ "$adopt_payload_sha256" != - ]] || adopt_payload_sha256="" + +adoption_values=0 +for value in "$adopt_baseline_image" "$adopt_gantry_image" "$adopt_payload_sha256"; do + [[ -z "$value" ]] || adoption_values=$((adoption_values + 1)) +done +if ((adoption_values != 0 && adoption_values != 3)); then + echo "adopted baseline image, Gantry image, and payload digest must be set together" >&2 + exit 2 +fi retry() { local attempts=0 @@ -226,6 +242,27 @@ aks_id=$(az aks show -g "$resource_group" -n "$aks_cluster" --query id -o tsv) baseline_login_server=$(az acr show -g "$resource_group" -n "$baseline_acr_name" --query loginServer -o tsv) gantry_login_server=$(az acr show -g "$resource_group" -n "$gantry_acr_name" --query loginServer -o tsv) +valid_adopted_image() { + local image=$1 + local login_server=$2 + local prefix="$login_server/gantry-benchmark-pull@" + [[ "$image" == "$prefix"* && "${image#"$prefix"}" =~ ^sha256:[0-9a-f]{64}$ ]] +} +if ((adoption_values == 3)); then + valid_adopted_image "$adopt_baseline_image" "$baseline_login_server" || { + echo "adopted baseline image is not an immutable gantry-benchmark-pull image in $baseline_login_server" >&2 + exit 2 + } + valid_adopted_image "$adopt_gantry_image" "$gantry_login_server" || { + echo "adopted Gantry image is not an immutable gantry-benchmark-pull image in $gantry_login_server" >&2 + exit 2 + } + [[ "$adopt_payload_sha256" =~ ^sha256:[0-9a-f]{64}$ ]] || { + echo "adopted payload fingerprint must be a sha256 digest" >&2 + exit 2 + } +fi + cat >/etc/gantry-benchmark/env < Date: Wed, 5 Aug 2026 20:09:38 -0400 Subject: [PATCH 27/60] fix(gantry): wait for monitoring scrape coverage --- hack/cmd/gantry-benchmark/preflight.go | 68 +++++++++-- .../preflight_monitoring_test.go | 109 ++++++++++++++++++ 2 files changed, 166 insertions(+), 11 deletions(-) create mode 100644 hack/cmd/gantry-benchmark/preflight_monitoring_test.go diff --git a/hack/cmd/gantry-benchmark/preflight.go b/hack/cmd/gantry-benchmark/preflight.go index 9af45d1c4..7ac81a7f0 100644 --- a/hack/cmd/gantry-benchmark/preflight.go +++ b/hack/cmd/gantry-benchmark/preflight.go @@ -443,17 +443,8 @@ func (b *benchmark) checkMonitoring(ctx context.Context, state benchmarkState) e } for _, check := range monitoringChecks { - count, err := b.queryPrometheus(ctx, check.query) - if err != nil { - return fmt.Errorf("query %s metric count: %w", check.description, err) - } - if int(count) != b.config.NodeCount { - return fmt.Errorf( - "prometheus reports %s metrics for %.0f/%d observer pods", - check.description, - count, - b.config.NodeCount, - ) + if err := b.waitForPrometheusMetricCoverage(ctx, check.description, check.query); err != nil { + return err } } @@ -489,6 +480,61 @@ func (b *benchmark) checkMonitoring(ctx context.Context, state benchmarkState) e } } +func (b *benchmark) waitForPrometheusMetricCoverage(ctx context.Context, description, query string) error { + pollContext, cancel := context.WithTimeout(ctx, b.config.TelemetryTimeout) + defer cancel() + + var count float64 + var queryErr error + + for { + count, queryErr = b.queryPrometheus(pollContext, query) + if queryErr == nil && int(count) == b.config.NodeCount { + return nil + } + + if queryErr == nil { + writeAll(b.stdout, fmt.Sprintf( + "waiting for Prometheus %s coverage: %.0f/%d observer pods\n", + description, + count, + b.config.NodeCount, + )) + } + + timer := time.NewTimer(b.config.TelemetryPollInterval) + select { + case <-pollContext.Done(): + if !timer.Stop() { + select { + case <-timer.C: + default: + } + } + if ctx.Err() != nil { + return ctx.Err() + } + if queryErr != nil { + return fmt.Errorf( + "prometheus %s metrics were not queryable before %s: %w", + description, + b.config.TelemetryTimeout, + queryErr, + ) + } + + return fmt.Errorf( + "prometheus reports %s metrics for %.0f/%d observer pods after waiting %s", + description, + count, + b.config.NodeCount, + b.config.TelemetryTimeout, + ) + case <-timer.C: + } + } +} + func (b *benchmark) checkAzureTelemetry(ctx context.Context) error { if _, err := b.commands.Run(ctx, nil, "az", "account", "show", "--output", "none"); err != nil { return fmt.Errorf("validate Azure CLI authentication: %w", err) diff --git a/hack/cmd/gantry-benchmark/preflight_monitoring_test.go b/hack/cmd/gantry-benchmark/preflight_monitoring_test.go new file mode 100644 index 000000000..e54f59247 --- /dev/null +++ b/hack/cmd/gantry-benchmark/preflight_monitoring_test.go @@ -0,0 +1,109 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package main + +import ( + "bytes" + "context" + "errors" + "fmt" + "strings" + "testing" + "time" +) + +type monitoringCoverageResult struct { + count float64 + err error +} + +type monitoringCoverageRunner struct { + results []monitoringCoverageResult + calls int +} + +func (r *monitoringCoverageRunner) Run(_ context.Context, _ []byte, _ string, _ ...string) ([]byte, error) { + index := r.calls + r.calls++ + if index >= len(r.results) { + index = len(r.results) - 1 + } + result := r.results[index] + if result.err != nil { + return nil, result.err + } + + return []byte(fmt.Sprintf( + `{"status":"success","data":{"result":[{"value":[0,%q]}]}}`, + fmt.Sprintf("%.0f", result.count), + )), nil +} + +func TestWaitForPrometheusMetricCoverageRetriesPartialScrape(t *testing.T) { + runner := &monitoringCoverageRunner{results: []monitoringCoverageResult{{count: 9}, {count: 1000}}} + var stdout bytes.Buffer + benchmark := benchmark{ + config: benchmarkConfig{ + MonitoringNamespace: "monitoring", + PrometheusService: "prometheus", + NodeCount: 1000, + TelemetryTimeout: time.Second, + TelemetryPollInterval: time.Millisecond, + }, + commands: runner, + stdout: &stdout, + } + + if err := benchmark.waitForPrometheusMetricCoverage(context.Background(), "containerd build", "count(metric)"); err != nil { + t.Fatal(err) + } + if runner.calls != 2 { + t.Fatalf("query calls = %d, want 2", runner.calls) + } + if !strings.Contains(stdout.String(), "9/1000 observer pods") { + t.Fatalf("progress output = %q", stdout.String()) + } +} + +func TestWaitForPrometheusMetricCoverageRetriesQueryError(t *testing.T) { + runner := &monitoringCoverageRunner{results: []monitoringCoverageResult{{err: errors.New("not ready")}, {count: 1000}}} + benchmark := benchmark{ + config: benchmarkConfig{ + MonitoringNamespace: "monitoring", + PrometheusService: "prometheus", + NodeCount: 1000, + TelemetryTimeout: time.Second, + TelemetryPollInterval: time.Millisecond, + }, + commands: runner, + stdout: &bytes.Buffer{}, + } + + if err := benchmark.waitForPrometheusMetricCoverage(context.Background(), "containerd build", "count(metric)"); err != nil { + t.Fatal(err) + } + if runner.calls != 2 { + t.Fatalf("query calls = %d, want 2", runner.calls) + } +} + +func TestWaitForPrometheusMetricCoverageTimesOut(t *testing.T) { + runner := &monitoringCoverageRunner{results: []monitoringCoverageResult{{count: 9}}} + benchmark := benchmark{ + config: benchmarkConfig{ + MonitoringNamespace: "monitoring", + PrometheusService: "prometheus", + NodeCount: 1000, + TelemetryTimeout: time.Nanosecond, + TelemetryPollInterval: time.Hour, + }, + commands: runner, + stdout: &bytes.Buffer{}, + } + + err := benchmark.waitForPrometheusMetricCoverage(context.Background(), "containerd build", "count(metric)") + if err == nil || !strings.Contains(err.Error(), "9/1000 observer pods after waiting 1ns") { + t.Fatalf("timeout error = %v", err) + } +} From 8d841ca614a7f195471fcafe0d2573eeb0e39e74 Mon Sep 17 00:00:00 2001 From: Vaibhav Patel Date: Thu, 6 Aug 2026 13:45:57 -0400 Subject: [PATCH 28/60] style(gantry-benchmark): satisfy lint gates on benchmark and metrics code Resolve 105 golangci-lint findings across the packages touched by the benchmark observability work: - govet shadow: rename the local `benchmark` values in the monitoring coverage tests, which shadowed the `benchmark` type declaration. - staticcheck ST1005: lowercase the Prometheus range-response size error string to match the surrounding error text. - wsl_v5/gofumpt: apply the repository formatters. No behavior change. Verified by diffing with whitespace ignored: the only semantic edits are the two lint fixes above plus a gofumpt var-block consolidation in preflight.go. --- cmd/gantry/agent_metrics.go | 3 +++ cmd/gantry/main.go | 1 + cmd/gantry/stream_commit_tracker.go | 2 ++ cmd/gantry/stream_commit_tracker_test.go | 2 ++ .../gantry-benchmark/azure_preflight_test.go | 1 + hack/cmd/gantry-benchmark/enable_test.go | 5 +++++ hack/cmd/gantry-benchmark/gantry_only.go | 5 +++++ hack/cmd/gantry-benchmark/image.go | 7 +++++++ hack/cmd/gantry-benchmark/image_test.go | 2 ++ hack/cmd/gantry-benchmark/job_test.go | 1 + hack/cmd/gantry-benchmark/peer_telemetry.go | 16 +++++++++++++++ .../gantry-benchmark/peer_telemetry_test.go | 6 ++++++ .../gantry-benchmark/performance_telemetry.go | 20 ++++++++++++++++++- .../performance_telemetry_test.go | 10 ++++++++++ hack/cmd/gantry-benchmark/preflight.go | 8 ++++++-- .../preflight_monitoring_test.go | 19 ++++++++++++------ hack/cmd/gantry-benchmark/run.go | 9 +++++++++ internal/gantry/mirror/byte_metrics_test.go | 3 +++ 18 files changed, 111 insertions(+), 9 deletions(-) diff --git a/cmd/gantry/agent_metrics.go b/cmd/gantry/agent_metrics.go index 0d33cb22b..980b1f7c5 100644 --- a/cmd/gantry/agent_metrics.go +++ b/cmd/gantry/agent_metrics.go @@ -183,6 +183,7 @@ func newPhase2Metrics(reg *metrics.Registry) *phase2Metrics { p.mirrorCompletedAt.WithLabelValues(kind, source).Set(0) } } + for _, outcome := range []string{ "hit", "notfound", @@ -198,9 +199,11 @@ func newPhase2Metrics(reg *metrics.Registry) *phase2Metrics { p.peerFetch.WithLabelValues(outcome).Add(0) p.peerFetchDur.WithLabelValues(outcome) } + for _, outcome := range []string{"busy", "stall"} { p.peerFetchLastAt.WithLabelValues(outcome).Set(0) } + for _, outcome := range []string{"hit", "miss", "error", "timeout"} { p.dhtLookup.WithLabelValues(outcome).Add(0) p.dhtLookupDur.WithLabelValues(outcome) diff --git a/cmd/gantry/main.go b/cmd/gantry/main.go index 2769f94ae..8b028afde 100644 --- a/cmd/gantry/main.go +++ b/cmd/gantry/main.go @@ -619,6 +619,7 @@ func runAgent(args []string) error { mirror.WithPeerMetrics( func(outcome string) { p2.peerFetch.WithLabelValues(outcome).Inc() + if outcome == "busy" || outcome == "stall" { p2.peerFetchLastAt.WithLabelValues(outcome).SetToCurrentTime() } diff --git a/cmd/gantry/stream_commit_tracker.go b/cmd/gantry/stream_commit_tracker.go index 2e22c2160..88dc9c40b 100644 --- a/cmd/gantry/stream_commit_tracker.go +++ b/cmd/gantry/stream_commit_tracker.go @@ -186,9 +186,11 @@ func (t *streamCommitTracker) probe(parent context.Context) { if observed > 0 && t.onObserved != nil { t.onObserved(observed) } + sort.Slice(observedCommits, func(i, j int) bool { return observedCommits[i].completedAt.Before(observedCommits[j].completedAt) }) + if t.onObservedDuration != nil { for _, commit := range observedCommits { t.onObservedDuration(commit.duration) diff --git a/cmd/gantry/stream_commit_tracker_test.go b/cmd/gantry/stream_commit_tracker_test.go index 98ee74860..97952ca85 100644 --- a/cmd/gantry/stream_commit_tracker_test.go +++ b/cmd/gantry/stream_commit_tracker_test.go @@ -99,6 +99,7 @@ func TestStreamCommitTracker_ObservedAfterInventoryAppears(t *testing.T) { if duration <= 0 { t.Errorf("observed duration = %s, want positive", duration) } + atomic.AddInt32(&durations, 1) }, func(n int) { atomic.AddInt32(&missing, int32(n)) }, @@ -190,6 +191,7 @@ func TestStreamCommitTracker_ReportsLatestCompletedStreamLast(t *testing.T) { inv := &fakeInventorySource{current: []digest.Digest{earlier, later}} var durations []time.Duration + tracker := newStreamCommitTracker(inv, nil, nil, func(duration time.Duration) { durations = append(durations, duration) }, nil) diff --git a/hack/cmd/gantry-benchmark/azure_preflight_test.go b/hack/cmd/gantry-benchmark/azure_preflight_test.go index a881c24d3..f2e1420dc 100644 --- a/hack/cmd/gantry-benchmark/azure_preflight_test.go +++ b/hack/cmd/gantry-benchmark/azure_preflight_test.go @@ -210,6 +210,7 @@ func TestDecodeAzureDiagnosticSettingsSupportsCLIAndARMShapes(t *testing.T) { if err != nil { t.Fatalf("decodeAzureDiagnosticSettings: %v", err) } + if len(settings) != 1 || settings[0].WorkspaceID != "/subscriptions/s/workspaces/law" || len(settings[0].Logs) != 1 || !settings[0].Logs[0].Enabled { t.Fatalf("settings = %+v, want one dedicated audit setting", settings) diff --git a/hack/cmd/gantry-benchmark/enable_test.go b/hack/cmd/gantry-benchmark/enable_test.go index 9730a21f9..800bebc2a 100644 --- a/hack/cmd/gantry-benchmark/enable_test.go +++ b/hack/cmd/gantry-benchmark/enable_test.go @@ -87,6 +87,7 @@ func TestRenderMonitoringManifest(t *testing.T) { !strings.Contains(string(rendered), `- controller-revision-hash`) { t.Fatalf("rendered manifest is missing benchmark scrape or Gantry revision labels") } + if strings.Count(string(rendered), `gantry_benchmark: "true"`) != 2 { t.Fatalf("rendered manifest does not label both benchmark PodMonitors for discovery") } @@ -94,15 +95,19 @@ func TestRenderMonitoringManifest(t *testing.T) { if !strings.Contains(string(rendered), `action: keep`) { t.Fatalf("rendered manifest does not limit Gantry metric cardinality") } + if !strings.Contains(string(rendered), `systemctl show --property MainPID --value containerd`) { t.Fatalf("rendered manifest does not validate the running containerd debug configuration") } + if !strings.Contains(string(rendered), `- port: ctr-metrics`) || strings.Contains(string(rendered), `- port: containerd-metrics`) { t.Fatalf("rendered manifest does not use the Kubernetes-valid containerd metrics port name") } + if !strings.Contains(string(rendered), `--web.listen-address=:29100`) { t.Fatalf("rendered manifest does not use the benchmark node-exporter port") } + for _, metric := range []string{ "p2p_peer_fetch_duration_seconds_(bucket|sum|count)", "p2p_dht_lookup_duration_seconds_(bucket|sum|count)", diff --git a/hack/cmd/gantry-benchmark/gantry_only.go b/hack/cmd/gantry-benchmark/gantry_only.go index 33a42cf31..9ebe6798d 100644 --- a/hack/cmd/gantry-benchmark/gantry_only.go +++ b/hack/cmd/gantry-benchmark/gantry_only.go @@ -682,6 +682,7 @@ func (b *benchmark) runGantryOnly(ctx context.Context) (returnErr error) { if err != nil { return err } + diagnosticTimestamps, err := b.fetchGantryDiagnosticTimestamps(ctx, revision, telemetryWindow{ StartedAt: job.PhaseStartedAt, FinishedAt: job.PhaseFinishedAt, @@ -689,19 +690,23 @@ func (b *benchmark) runGantryOnly(ctx context.Context) (returnErr error) { if err != nil { return err } + if err := requireFinalLayerResponseTimestamps(diagnosticTimestamps, diagnosticsAfter.PodNodes); err != nil { return err } + diagnostics, err := subtractGantryDiagnosticSnapshots(diagnosticsBefore, diagnosticsAfter, diagnosticTimestamps) if err != nil { return err } bytes, bytesSource := deriveOriginBytes(b.config, proxyPhaseGantryCold, proxyPhaseTotals{}, metrics, job) + performance, err := b.capturePhasePerformanceTelemetry(ctx, proxyPhaseGantryCold, job) if err != nil { return err } + if err := b.writePerformanceTelemetryArtifact(state.RunID, proxyPhaseGantryCold, performance); err != nil { return err } diff --git a/hack/cmd/gantry-benchmark/image.go b/hack/cmd/gantry-benchmark/image.go index 58906daeb..d131a691e 100644 --- a/hack/cmd/gantry-benchmark/image.go +++ b/hack/cmd/gantry-benchmark/image.go @@ -135,15 +135,19 @@ func (b *benchmark) prepareAdoptedImages(ctx context.Context, baselineImage, gan if err != nil { return err } + if state.Status != "enabled" { return fmt.Errorf("benchmark state is %q, run enable before prepare-adopt", state.Status) } + if state.usesProxy() { return fmt.Errorf("prepare-adopt requires direct dual-ACR mode") } + if err := b.requireLock(ctx, state.RunID); err != nil { return err } + if err := b.validateContext(ctx); err != nil { return err } @@ -152,6 +156,7 @@ func (b *benchmark) prepareAdoptedImages(ctx context.Context, baselineImage, gan if err != nil { return err } + if err := b.saveState(ctx, state); err != nil { return err } @@ -170,10 +175,12 @@ func adoptPreparedImages(state benchmarkState, baselineImage, gantryImage, paylo state.BaselineImage = baselineImage state.GantryColdImage = gantryImage state.WorkloadPayloadSHA256 = payloadSHA + state.WorkloadComparisonMode = workloadComparisonIdenticalPayload if _, _, err := state.preparedImages(); err != nil { return benchmarkState{}, fmt.Errorf("validate adopted images: %w", err) } + state.Status = "images-prepared" return state, nil diff --git a/hack/cmd/gantry-benchmark/image_test.go b/hack/cmd/gantry-benchmark/image_test.go index 44eff4bcb..3bb60deab 100644 --- a/hack/cmd/gantry-benchmark/image_test.go +++ b/hack/cmd/gantry-benchmark/image_test.go @@ -140,6 +140,7 @@ func TestAdoptPreparedImages(t *testing.T) { if err != nil { t.Fatalf("adoptPreparedImages: %v", err) } + if adopted.Status != "images-prepared" || adopted.BaselineImage != baseline || adopted.GantryColdImage != gantry || adopted.WorkloadPayloadSHA256 != payload || adopted.WorkloadComparisonMode != workloadComparisonIdenticalPayload { @@ -161,6 +162,7 @@ func TestAdoptPreparedImagesRejectsInvalidInputs(t *testing.T) { if _, err := adoptPreparedImages(state, baseline, gantry, "not-a-digest"); err == nil { t.Fatal("expected invalid payload digest rejection") } + if _, err := adoptPreparedImages(state, baseline, gantry, "sha256:"+strings.Repeat("c", 64)); err == nil || !strings.Contains(err.Error(), "would reuse") { t.Fatalf("error = %v, want identical image digest rejection", err) diff --git a/hack/cmd/gantry-benchmark/job_test.go b/hack/cmd/gantry-benchmark/job_test.go index 62dbdff11..de4e90246 100644 --- a/hack/cmd/gantry-benchmark/job_test.go +++ b/hack/cmd/gantry-benchmark/job_test.go @@ -40,6 +40,7 @@ func TestParseJobObservation(t *testing.T) { if len(observation.Pods) != 4 || len(observation.PodNodes) != 4 || len(observation.PodTimings) != 4 { t.Fatalf("pod identities = %v nodes=%v timings=%v, want four", observation.Pods, observation.PodNodes, observation.PodTimings) } + podB := observation.PodTimings["pod-b"] if podB.NodeName != "node-b" || podB.StartLatencySeconds != 20 || podB.FinishLatencySeconds != 21 || !podB.ContainerStartedAt.Equal(phaseStartedAt.Add(20*time.Second)) { diff --git a/hack/cmd/gantry-benchmark/peer_telemetry.go b/hack/cmd/gantry-benchmark/peer_telemetry.go index 0a6c3a3c4..d4b4b945a 100644 --- a/hack/cmd/gantry-benchmark/peer_telemetry.go +++ b/hack/cmd/gantry-benchmark/peer_telemetry.go @@ -181,6 +181,7 @@ func (b *benchmark) fetchGantryDiagnosticSnapshot(ctx context.Context, revision if _, ok := podNodes[pod]; !ok { return gantryDiagnosticSnapshot{}, fmt.Errorf("diagnostic sample belongs to unexpected pod %q", pod) } + if sample.Value < 0 { return gantryDiagnosticSnapshot{}, fmt.Errorf("diagnostic sample for pod %s is negative: %v", pod, sample.Value) } @@ -189,6 +190,7 @@ func (b *benchmark) fetchGantryDiagnosticSnapshot(ctx context.Context, revision if err != nil { return gantryDiagnosticSnapshot{}, err } + counters[pod][key] = sample.Value } @@ -202,11 +204,13 @@ func diagnosticMetricKey(labels map[string]string) (string, error) { } parts := make([]string, 0, 3) + for _, label := range []string{"kind", "outcome", "source"} { if value := labels[label]; value != "" { parts = append(parts, label+"="+value) } } + if len(parts) == 0 { return name, nil } @@ -233,9 +237,11 @@ func (b *benchmark) fetchGantryDiagnosticTimestamps( if err != nil { return nil, err } + if err := validatePrometheusRangePodCoverage("gantry diagnostic timestamps", raw, b.config.NodeCount); err != nil { return nil, err } + var response struct { Data struct { Result []struct { @@ -249,28 +255,34 @@ func (b *benchmark) fetchGantryDiagnosticTimestamps( } result := map[string]map[string]float64{} + for _, series := range response.Data.Result { key, err := diagnosticMetricKey(series.Metric) if err != nil { return nil, err } + pod := series.Metric["pod"] for _, pair := range series.Values { rawValue, ok := pair[1].(string) if !ok { return nil, fmt.Errorf("diagnostic timestamp sample has non-string value") } + value, err := strconv.ParseFloat(rawValue, 64) if err != nil { return nil, fmt.Errorf("parse diagnostic timestamp sample %q: %w", rawValue, err) } + observedAt := time.Unix(0, int64(value*float64(time.Second))) if observedAt.Before(window.StartedAt) || observedAt.After(window.FinishedAt) { continue } + if result[pod] == nil { result[pod] = map[string]float64{} } + if value > result[pod][key] { result[pod][key] = value } @@ -300,6 +312,7 @@ func subtractGantryDiagnosticSnapshots( } deltas := map[string]float64{} + for key, afterValue := range after.Counters[pod] { beforeValue := before.Counters[pod][key] if afterValue < beforeValue { @@ -311,6 +324,7 @@ func subtractGantryDiagnosticSnapshots( afterValue, ) } + if delta := afterValue - beforeValue; delta != 0 { deltas[key] = delta } @@ -348,11 +362,13 @@ func requireFinalLayerResponseTimestamps( podNodes map[string]string, ) error { missing := make([]string, 0) + for pod := range podNodes { if finalLayerResponseCompletedTimestamp(timestamps[pod]) == 0 { missing = append(missing, pod) } } + if len(missing) == 0 { return nil } diff --git a/hack/cmd/gantry-benchmark/peer_telemetry_test.go b/hack/cmd/gantry-benchmark/peer_telemetry_test.go index c361a9a2d..167af404a 100644 --- a/hack/cmd/gantry-benchmark/peer_telemetry_test.go +++ b/hack/cmd/gantry-benchmark/peer_telemetry_test.go @@ -50,13 +50,16 @@ func TestFetchGantryDiagnosticTimestampsUsesExactJobWindow(t *testing.T) { if err != nil { t.Fatalf("fetchGantryDiagnosticTimestamps: %v", err) } + if len(timestamps["gantry-a"]) != 2 || timestamps["gantry-b"] != nil { t.Fatalf("timestamps = %v, want only two in-window gantry-a values", timestamps) } + if timestamps["gantry-a"]["gantry_mirror_response_completed_timestamp_seconds{kind=layer,source=peer}"] != float64(time.Date(2026, time.August, 4, 1, 3, 0, 0, time.UTC).Unix()) { t.Fatalf("timestamps = %v, want latest in-window layer completion", timestamps) } + if !strings.Contains(runner.queryPath, `kind%3D%22layer%22`) { t.Fatalf("query path %q does not restrict completion timestamps to layers", runner.queryPath) } @@ -171,6 +174,7 @@ func TestDiagnosticMetricKey(t *testing.T) { if err != nil { t.Fatalf("diagnosticMetricKey: %v", err) } + if key != "p2p_peer_fetch_total{outcome=busy}" { t.Fatalf("key = %q, want p2p_peer_fetch_total{outcome=busy}", key) } @@ -200,9 +204,11 @@ func TestSubtractGantryDiagnosticSnapshots(t *testing.T) { if err != nil { t.Fatalf("subtractGantryDiagnosticSnapshots: %v", err) } + if !measurement.Complete || len(measurement.Pods) != 1 { t.Fatalf("measurement = %+v, want one complete pod", measurement) } + pod := measurement.Pods[0] if pod.NodeName != "node-a" || pod.CounterDeltas["p2p_peer_fetch_total{outcome=busy}"] != 3 || pod.TimestampSeconds["gantry_mirror_response_completed_timestamp_seconds{kind=layer,source=peer}"] != 1234 || diff --git a/hack/cmd/gantry-benchmark/performance_telemetry.go b/hack/cmd/gantry-benchmark/performance_telemetry.go index 74bbaea02..253bc549b 100644 --- a/hack/cmd/gantry-benchmark/performance_telemetry.go +++ b/hack/cmd/gantry-benchmark/performance_telemetry.go @@ -107,13 +107,16 @@ func (b *benchmark) capturePhasePerformanceTelemetry( if step == 0 { step = performanceTelemetryStep } + response, err := b.queryPrometheusRange(ctx, item.query, window, step) if err != nil { return phasePerformanceTelemetry{}, fmt.Errorf("capture %s: %w", item.name, err) } + if err := validatePrometheusRangePodCoverage(item.name, response, b.config.NodeCount); err != nil { return phasePerformanceTelemetry{}, err } + captures = append(captures, prometheusRangeCapture{ Name: item.name, Query: item.query, @@ -126,6 +129,7 @@ func (b *benchmark) capturePhasePerformanceTelemetry( if err != nil { return phasePerformanceTelemetry{}, err } + journalEvents, err := parseContainerdJournal(journal, observerPods, window) if err != nil { return phasePerformanceTelemetry{}, err @@ -155,15 +159,18 @@ func validatePrometheusRangePodCoverage(name string, raw json.RawMessage, expect } pods := map[string]struct{}{} + for _, series := range response.Data.Result { pod := series.Metric["pod"] if pod == "" { return fmt.Errorf("%s range series has no pod label", name) } + if len(series.Values) > 0 { pods[pod] = struct{}{} } } + if len(pods) != expectedPods { return fmt.Errorf("%s range capture has samples from %d/%d pods", name, len(pods), expectedPods) } @@ -189,31 +196,37 @@ func parseContainerdJournal( if !strings.HasPrefix(line, "[pod/") || prefixEnd < 0 { return nil, fmt.Errorf("parse containerd journal prefix: %q", line) } + prefixParts := strings.Split(strings.TrimSuffix(strings.TrimPrefix(line[:prefixEnd], "["), "]"), "/") if len(prefixParts) != 3 || prefixParts[0] != "pod" || prefixParts[2] != "containerd-journal" { return nil, fmt.Errorf("parse containerd journal source: %q", line[:prefixEnd+1]) } observerPod := prefixParts[1] + nodeName, ok := observerPodNodes[observerPod] if !ok { return nil, fmt.Errorf("containerd journal belongs to unexpected observer pod %q", observerPod) } remainder := line[prefixEnd+2:] + timestampEnd := strings.IndexByte(remainder, ' ') if timestampEnd < 0 { return nil, fmt.Errorf("parse containerd journal timestamp: %q", line) } + timestamp, err := time.Parse(time.RFC3339Nano, remainder[:timestampEnd]) if err != nil { return nil, fmt.Errorf("parse containerd journal timestamp %q: %w", remainder[:timestampEnd], err) } + if timestamp.Before(window.StartedAt) || timestamp.After(window.FinishedAt) { continue } message := remainder[timestampEnd+1:] + eventType := classifyContainerdJournalEvent(message) if eventType == "" { return nil, fmt.Errorf("classify filtered containerd journal message: %q", message) @@ -233,6 +246,7 @@ func parseContainerdJournal( if err != nil { return nil, fmt.Errorf("parse containerd journal duration %q: %w", match[2], err) } + event.DurationSeconds = duration.Seconds() case "layer": event.LayerDigest = match[2] @@ -291,6 +305,7 @@ func (b *benchmark) queryPrometheusRange( if err != nil { return nil, err } + if err := validatePrometheusRangeResponseSize(output, maxPrometheusRangeResponseBytes); err != nil { return nil, err } @@ -301,6 +316,7 @@ func (b *benchmark) queryPrometheusRange( if err := json.Unmarshal(output, &envelope); err != nil { return nil, fmt.Errorf("decode Prometheus range response: %w", err) } + if envelope.Status != "success" { return nil, fmt.Errorf("prometheus range query status is %q", envelope.Status) } @@ -311,7 +327,7 @@ func (b *benchmark) queryPrometheusRange( func validatePrometheusRangeResponseSize(output []byte, limit int) error { if len(output) > limit { return fmt.Errorf( - "Prometheus range response is %d bytes, exceeds %d-byte capture limit", + "prometheus range response is %d bytes, exceeds %d-byte capture limit", len(output), limit, ) @@ -351,8 +367,10 @@ func (b *benchmark) observerPodNodes(ctx context.Context) (map[string]string, er if pod.Metadata.Name == "" || pod.Spec.NodeName == "" { return nil, fmt.Errorf("observer pod has empty name or nodeName") } + result[pod.Metadata.Name] = pod.Spec.NodeName } + if len(result) != b.config.NodeCount { return nil, fmt.Errorf("observer pod/node map has %d pods, want %d", len(result), b.config.NodeCount) } diff --git a/hack/cmd/gantry-benchmark/performance_telemetry_test.go b/hack/cmd/gantry-benchmark/performance_telemetry_test.go index eca974eaf..28536c7d3 100644 --- a/hack/cmd/gantry-benchmark/performance_telemetry_test.go +++ b/hack/cmd/gantry-benchmark/performance_telemetry_test.go @@ -45,9 +45,11 @@ func TestQueryPrometheusRange(t *testing.T) { if err != nil { t.Fatalf("queryPrometheusRange: %v", err) } + if !strings.Contains(string(response), `"status":"success"`) { t.Fatalf("response = %s, want successful raw envelope", response) } + if len(runner.commands) != 1 { t.Fatalf("commands = %v, want one command", runner.commands) } @@ -62,6 +64,7 @@ func TestQueryPrometheusRange(t *testing.T) { func TestPerformanceTelemetryQueriesBoundContainerdCardinality(t *testing.T) { queries := performanceTelemetryQueries() + byName := make(map[string]performanceTelemetryQuery, len(queries)) for _, query := range queries { byName[query.name] = query @@ -75,9 +78,11 @@ func TestPerformanceTelemetryQueriesBoundContainerdCardinality(t *testing.T) { t.Fatalf("missing bounded telemetry query %q", name) } } + if byName["containerd_image_pulls"].step != 0 || byName["containerd_grpc_started"].step != 0 { t.Fatal("containerd pull and gRPC started queries must use the default 10-second step") } + if byName["containerd_grpc_handled"].step != 5*time.Minute { t.Fatalf("gRPC handled step = %s, want 5m", byName["containerd_grpc_handled"].step) } @@ -87,6 +92,7 @@ func TestValidatePrometheusRangeResponseSize(t *testing.T) { if err := validatePrometheusRangeResponseSize([]byte("1234"), 4); err != nil { t.Fatalf("response at limit: %v", err) } + if err := validatePrometheusRangeResponseSize([]byte("12345"), 4); err == nil || !strings.Contains(err.Error(), "5 bytes") { t.Fatalf("oversized response error = %v", err) } @@ -111,13 +117,16 @@ func TestParseContainerdJournal(t *testing.T) { if err != nil { t.Fatalf("parseContainerdJournal: %v", err) } + if len(events) != 2 { t.Fatalf("events = %v, want two phase-bounded events", events) } + if events[0].NodeName != "node-a" || events[0].Type != "layer_unpacked" || events[0].LayerDigest != "sha256:abc" || events[0].DurationSeconds != 2.5 { t.Fatalf("first event = %+v, want parsed layer event", events[0]) } + if events[1].NodeName != "node-b" || events[1].Type != "pull_completed" { t.Fatalf("second event = %+v, want correlated pull completion", events[1]) } @@ -148,6 +157,7 @@ func TestValidatePrometheusRangePodCoverage(t *testing.T) { if err := validatePrometheusRangePodCoverage("disk", raw, 2); err != nil { t.Fatalf("validatePrometheusRangePodCoverage: %v", err) } + if err := validatePrometheusRangePodCoverage("disk", raw, 3); err == nil || !strings.Contains(err.Error(), "2/3 pods") { t.Fatalf("error = %v, want partial pod coverage", err) } diff --git a/hack/cmd/gantry-benchmark/preflight.go b/hack/cmd/gantry-benchmark/preflight.go index 7ac81a7f0..a6ce5f1f9 100644 --- a/hack/cmd/gantry-benchmark/preflight.go +++ b/hack/cmd/gantry-benchmark/preflight.go @@ -484,8 +484,10 @@ func (b *benchmark) waitForPrometheusMetricCoverage(ctx context.Context, descrip pollContext, cancel := context.WithTimeout(ctx, b.config.TelemetryTimeout) defer cancel() - var count float64 - var queryErr error + var ( + count float64 + queryErr error + ) for { count, queryErr = b.queryPrometheus(pollContext, query) @@ -511,9 +513,11 @@ func (b *benchmark) waitForPrometheusMetricCoverage(ctx context.Context, descrip default: } } + if ctx.Err() != nil { return ctx.Err() } + if queryErr != nil { return fmt.Errorf( "prometheus %s metrics were not queryable before %s: %w", diff --git a/hack/cmd/gantry-benchmark/preflight_monitoring_test.go b/hack/cmd/gantry-benchmark/preflight_monitoring_test.go index e54f59247..6ea226c26 100644 --- a/hack/cmd/gantry-benchmark/preflight_monitoring_test.go +++ b/hack/cmd/gantry-benchmark/preflight_monitoring_test.go @@ -25,10 +25,12 @@ type monitoringCoverageRunner struct { func (r *monitoringCoverageRunner) Run(_ context.Context, _ []byte, _ string, _ ...string) ([]byte, error) { index := r.calls + r.calls++ if index >= len(r.results) { index = len(r.results) - 1 } + result := r.results[index] if result.err != nil { return nil, result.err @@ -42,8 +44,10 @@ func (r *monitoringCoverageRunner) Run(_ context.Context, _ []byte, _ string, _ func TestWaitForPrometheusMetricCoverageRetriesPartialScrape(t *testing.T) { runner := &monitoringCoverageRunner{results: []monitoringCoverageResult{{count: 9}, {count: 1000}}} + var stdout bytes.Buffer - benchmark := benchmark{ + + bench := benchmark{ config: benchmarkConfig{ MonitoringNamespace: "monitoring", PrometheusService: "prometheus", @@ -55,12 +59,14 @@ func TestWaitForPrometheusMetricCoverageRetriesPartialScrape(t *testing.T) { stdout: &stdout, } - if err := benchmark.waitForPrometheusMetricCoverage(context.Background(), "containerd build", "count(metric)"); err != nil { + if err := bench.waitForPrometheusMetricCoverage(context.Background(), "containerd build", "count(metric)"); err != nil { t.Fatal(err) } + if runner.calls != 2 { t.Fatalf("query calls = %d, want 2", runner.calls) } + if !strings.Contains(stdout.String(), "9/1000 observer pods") { t.Fatalf("progress output = %q", stdout.String()) } @@ -68,7 +74,7 @@ func TestWaitForPrometheusMetricCoverageRetriesPartialScrape(t *testing.T) { func TestWaitForPrometheusMetricCoverageRetriesQueryError(t *testing.T) { runner := &monitoringCoverageRunner{results: []monitoringCoverageResult{{err: errors.New("not ready")}, {count: 1000}}} - benchmark := benchmark{ + bench := benchmark{ config: benchmarkConfig{ MonitoringNamespace: "monitoring", PrometheusService: "prometheus", @@ -80,9 +86,10 @@ func TestWaitForPrometheusMetricCoverageRetriesQueryError(t *testing.T) { stdout: &bytes.Buffer{}, } - if err := benchmark.waitForPrometheusMetricCoverage(context.Background(), "containerd build", "count(metric)"); err != nil { + if err := bench.waitForPrometheusMetricCoverage(context.Background(), "containerd build", "count(metric)"); err != nil { t.Fatal(err) } + if runner.calls != 2 { t.Fatalf("query calls = %d, want 2", runner.calls) } @@ -90,7 +97,7 @@ func TestWaitForPrometheusMetricCoverageRetriesQueryError(t *testing.T) { func TestWaitForPrometheusMetricCoverageTimesOut(t *testing.T) { runner := &monitoringCoverageRunner{results: []monitoringCoverageResult{{count: 9}}} - benchmark := benchmark{ + bench := benchmark{ config: benchmarkConfig{ MonitoringNamespace: "monitoring", PrometheusService: "prometheus", @@ -102,7 +109,7 @@ func TestWaitForPrometheusMetricCoverageTimesOut(t *testing.T) { stdout: &bytes.Buffer{}, } - err := benchmark.waitForPrometheusMetricCoverage(context.Background(), "containerd build", "count(metric)") + err := bench.waitForPrometheusMetricCoverage(context.Background(), "containerd build", "count(metric)") if err == nil || !strings.Contains(err.Error(), "9/1000 observer pods after waiting 1ns") { t.Fatalf("timeout error = %v", err) } diff --git a/hack/cmd/gantry-benchmark/run.go b/hack/cmd/gantry-benchmark/run.go index 1642085e3..24d4152a3 100644 --- a/hack/cmd/gantry-benchmark/run.go +++ b/hack/cmd/gantry-benchmark/run.go @@ -179,6 +179,7 @@ func (b *benchmark) runBenchmark(ctx context.Context) (returnErr error) { if err != nil { return err } + baselineDiagnosticTimestamps, err := b.fetchGantryDiagnosticTimestamps(ctx, revision, telemetryWindow{ StartedAt: baselineJob.PhaseStartedAt, FinishedAt: baselineJob.PhaseFinishedAt, @@ -186,6 +187,7 @@ func (b *benchmark) runBenchmark(ctx context.Context) (returnErr error) { if err != nil { return err } + baselineDiagnostics, err := subtractGantryDiagnosticSnapshots( baselineDiagnosticsBefore, baselineDiagnosticsAfter, @@ -196,10 +198,12 @@ func (b *benchmark) runBenchmark(ctx context.Context) (returnErr error) { } baselineBytes, baselineBytesSource := deriveOriginBytes(b.config, proxyPhaseBaseline, baselineProxy, baselineGantry, baselineJob) + baselinePerformance, err := b.capturePhasePerformanceTelemetry(ctx, proxyPhaseBaseline, baselineJob) if err != nil { return err } + if err := b.writePerformanceTelemetryArtifact(state.RunID, proxyPhaseBaseline, baselinePerformance); err != nil { return err } @@ -301,6 +305,7 @@ func (b *benchmark) runBenchmark(ctx context.Context) (returnErr error) { if err != nil { return err } + gantryDiagnosticTimestamps, err := b.fetchGantryDiagnosticTimestamps(ctx, revision, telemetryWindow{ StartedAt: gantryJob.PhaseStartedAt, FinishedAt: gantryJob.PhaseFinishedAt, @@ -308,9 +313,11 @@ func (b *benchmark) runBenchmark(ctx context.Context) (returnErr error) { if err != nil { return err } + if err := requireFinalLayerResponseTimestamps(gantryDiagnosticTimestamps, gantryDiagnosticsAfter.PodNodes); err != nil { return err } + gantryDiagnostics, err := subtractGantryDiagnosticSnapshots( gantryDiagnosticsBefore, gantryDiagnosticsAfter, @@ -330,10 +337,12 @@ func (b *benchmark) runBenchmark(ctx context.Context) (returnErr error) { } gantryBytes, gantryBytesSource := deriveOriginBytes(b.config, proxyPhaseGantryCold, gantryProxy, phaseMetrics, gantryJob) + gantryPerformance, err := b.capturePhasePerformanceTelemetry(ctx, proxyPhaseGantryCold, gantryJob) if err != nil { return err } + if err := b.writePerformanceTelemetryArtifact(state.RunID, proxyPhaseGantryCold, gantryPerformance); err != nil { return err } diff --git a/internal/gantry/mirror/byte_metrics_test.go b/internal/gantry/mirror/byte_metrics_test.go index 272ae3c68..269d1bf8e 100644 --- a/internal/gantry/mirror/byte_metrics_test.go +++ b/internal/gantry/mirror/byte_metrics_test.go @@ -78,6 +78,7 @@ func TestMirrorByteMetricsCacheSource(t *testing.T) { if len(served) != 1 || served[0] != want { t.Fatalf("served observations = %+v, want [%+v]", served, want) } + if len(completed) != 1 || completed[0].kind != want.kind || completed[0].source != want.source { t.Fatalf("completed observations = %+v, want kind=%s source=%s", completed, want.kind, want.source) } @@ -128,6 +129,7 @@ func TestMirrorByteMetricsPeerSource(t *testing.T) { if len(served) != 1 || served[0] != wantServed { t.Fatalf("served observations = %+v, want [%+v]", served, wantServed) } + if len(completed) != 1 || completed[0].kind != wantServed.kind || completed[0].source != wantServed.source { t.Fatalf("completed observations = %+v, want kind=%s source=%s", completed, wantServed.kind, wantServed.source) } @@ -187,6 +189,7 @@ func TestMirrorByteMetricsOriginSource(t *testing.T) { if len(served) != 1 || served[0] != wantServed { t.Fatalf("served observations = %+v, want [%+v]", served, wantServed) } + if len(completed) != 1 || completed[0].kind != wantServed.kind || completed[0].source != wantServed.source { t.Fatalf("completed observations = %+v, want kind=%s source=%s", completed, wantServed.kind, wantServed.source) } From 7492b16b87da5ab2367b618fbe1fcd44d11e7b3b Mon Sep 17 00:00:00 2001 From: Vaibhav Patel Date: Thu, 6 Aug 2026 13:46:04 -0400 Subject: [PATCH 29/60] docs(gantry-benchmark): record Canada Central 1000-node results Add the run-20260806-142719-660ecfb3 sample to the byte reduction, pod startup latency, and audit-filtered aggregate tables. Result was PASS: 99.479% byte reduction, 99.800% pull reduction, 59.759% P95 improvement, and zero fallbacks. --- hack/gantry-benchmark/RESULTS.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/hack/gantry-benchmark/RESULTS.md b/hack/gantry-benchmark/RESULTS.md index 3e40a8ecb..64422fd1e 100644 --- a/hack/gantry-benchmark/RESULTS.md +++ b/hack/gantry-benchmark/RESULTS.md @@ -23,6 +23,7 @@ logs, and measured peer traffic from Gantry metrics. | --- | ---: | ---: | ---: | ---: | ---: | ---: | | 1000 nodes - sample 1 | 40 GiB | 47.296 TB | 174.773 GB | 99.630% | 1002 / 4 | 99.601% | | 1000 nodes - sample 2 | 40 GiB | 47.566 TB | 174.781 GB | 99.633% | 1008 / 5 | 99.504% | +| **1000 nodes - Canada Central ACR** | **40 GiB** | **47.178 TB** | **245.878 GB** | **99.479%** | **1000 / 2** | **99.800%** | | **1000 nodes - UK South ACR** | **40 GiB** | **53.369 TB** | **219.262 GB** | **99.589%** | **1254 / 5** | **99.601%** | | **1000 nodes - East US ACR** | **40 GiB** | **47.562 TB** | **182.317 GB** | **99.617%** | **1004 / 4** | **99.602%** | | **1000 nodes - Central India ACR** 1 | **40 GiB** | **97.115 TB** | **803.184 GB** | **99.173%** | **2287 / 6** | **99.738%** | @@ -46,6 +47,7 @@ aggregates below. | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | | 1000 nodes - sample 1 | 40 GiB | 682.725s | 1099.629s | 821.209s | 1171.863s | 1422.461s | 1885.385s | 42.700% slower | | 1000 nodes - sample 2 | 40 GiB | 894.597s | 1087.179s | 1065.724s | 1169.448s | 1652.704s | 1885.011s | 9.733% slower | +| **1000 nodes - Canada Central ACR** | **40 GiB** | **1862.580s** | **774.028s** | **2030.636s** | **817.139s** | **2149.535s** | **891.766s** | **59.759% faster** | | **1000 nodes - UK South ACR** | **40 GiB** | **3561.000s** | **1064.557s** | **3953.000s** | **1146.557s** | **5399.000s** | **1815.557s** | **70.995% faster** | | **1000 nodes - East US ACR** | **40 GiB** | **1401.026s** | **1065.950s** | **1655.894s** | **1144.771s** | **2351.081s** | **1831.832s** | **30.867% faster** | | **1000 nodes - Central India ACR** 2 | **40 GiB** | **3184.649s** | **1065.570s** | **4851.649s** | **1155.570s** | **5351.649s** | **1865.570s** | **76.182% faster** | @@ -89,6 +91,7 @@ The unfiltered table remains the primary end-to-end result because | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | | 1000 nodes - sample 1 | 40 GiB | 43.566 TB | 158.928 GB | 157 | 153 | 42,597 | 0 | | 1000 nodes - sample 2 | 40 GiB | 43.438 TB | 160.002 GB | 159 | 154 | 42,472 | 0 | +| **1000 nodes - Canada Central ACR** | **40 GiB** | **42.735 TB** | **223.358 GB** | **214** | **212** | **41,791** | **0** | | **1000 nodes - East US ACR** | **40 GiB** | **43.137 TB** | **161.075 GB** | **159** | **155** | **42,179** | **0** | | **1000 nodes - Central India ACR** | **40 GiB** | **43.070 TB** | **709.766 GB** | **682** | **662** | **42,129** | **0** | | 2000 nodes - sample 1 | 40 GiB | 86.971 TB | 221.210 GB | 213 | 210 | 85,044 | 0 | From 631c6b8ca98d12cefc98c8a758e3a1c9dc577af0 Mon Sep 17 00:00:00 2001 From: Vaibhav Patel Date: Thu, 6 Aug 2026 14:12:59 -0400 Subject: [PATCH 30/60] fix(gantry-benchmark): drop max_concurrent_downloads override The transfer plugin override caused problems during benchmark runs, so return to the containerd default of 3 concurrent downloads. Remove the setting from the conf.d drop-in, its readiness assertion, and the manifest test guard. The debug log level and 15m image pull progress timeout are unchanged. Redeploy propagates the removal: the drop-in is replaced wholesale and its content hash changes, so the DaemonSet restarts containerd. --- hack/cmd/gantry-benchmark/enable_test.go | 1 - hack/gantry-benchmark/manifests/containerd.yaml | 4 ---- 2 files changed, 5 deletions(-) diff --git a/hack/cmd/gantry-benchmark/enable_test.go b/hack/cmd/gantry-benchmark/enable_test.go index 800bebc2a..66c32b21f 100644 --- a/hack/cmd/gantry-benchmark/enable_test.go +++ b/hack/cmd/gantry-benchmark/enable_test.go @@ -158,7 +158,6 @@ func TestContainerdBenchmarkManifest(t *testing.T) { for _, setting := range []string{ `level = "debug"`, `image_pull_progress_timeout = "15m"`, - `max_concurrent_downloads = 6`, `systemd-run`, } { if !bytes.Contains(manifest, []byte(setting)) { diff --git a/hack/gantry-benchmark/manifests/containerd.yaml b/hack/gantry-benchmark/manifests/containerd.yaml index 2299172a1..d9e0cbce9 100644 --- a/hack/gantry-benchmark/manifests/containerd.yaml +++ b/hack/gantry-benchmark/manifests/containerd.yaml @@ -12,9 +12,6 @@ data: [plugins."io.containerd.cri.v1.images"] image_pull_progress_timeout = "15m" - - [plugins."io.containerd.transfer.v1.local"] - max_concurrent_downloads = 6 --- apiVersion: apps/v1 kind: DaemonSet @@ -106,7 +103,6 @@ spec: ')" printf '%s\n' "$config_dump" | grep -Eq "^[[:space:]]*level = ['\"]debug['\"]$" printf '%s\n' "$config_dump" | grep -Eq "^[[:space:]]*image_pull_progress_timeout = ['\"]15m(0s)?['\"]$" - printf '%s\n' "$config_dump" | grep -Eq '^[[:space:]]*max_concurrent_downloads = 6$' initialDelaySeconds: 10 periodSeconds: 10 failureThreshold: 30 From 08a40d77448dd5cbbb8fddaf418d2687e6079f06 Mon Sep 17 00:00:00 2001 From: Vaibhav Patel Date: Thu, 6 Aug 2026 14:16:38 -0400 Subject: [PATCH 31/60] docs(gantry-benchmark): drop removed download concurrency note --- hack/gantry-benchmark/RUNBOOK.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hack/gantry-benchmark/RUNBOOK.md b/hack/gantry-benchmark/RUNBOOK.md index fe9b728cc..b9113bfae 100644 --- a/hack/gantry-benchmark/RUNBOOK.md +++ b/hack/gantry-benchmark/RUNBOOK.md @@ -91,9 +91,9 @@ The graphroot must be `/opt/gantry-benchmark/containers`. Before provisioning the operator VM or starting the lifecycle on AKS, apply the benchmark containerd configuration and require it to be Ready on every -target node. This enables debug unpack logs, sets the no-progress timeout to -15 minutes, and raises transfer-service layer downloads to six. The DaemonSet -performs one detached containerd restart per configuration hash. +target node. This enables debug unpack logs and sets the no-progress timeout +to 15 minutes. The DaemonSet performs one detached containerd restart per +configuration hash. ```bash kubectl create namespace gantry-system --dry-run=client -o yaml | kubectl apply -f - From 97cbbdd0624c64ef0672049258d42bbc683890c8 Mon Sep 17 00:00:00 2001 From: Vaibhav Patel Date: Thu, 6 Aug 2026 15:06:27 -0400 Subject: [PATCH 32/60] feat(gantry-benchmark): stream live image build and push progress Image preparation ran for many minutes behind a single "building one shared payload" line. execCommandRunner.Run buffers with CombinedOutput, so podman build and push progress was withheld until the process exited and then discarded on success. Add an optional RunStreaming capability that tees child output to the progress writer while the command runs, and report each step: - per-layer payload generation with size, elapsed time, and percentage - payload hashing and the resulting shared fingerprint - which of the two images is building, and its target registry and tag - podman/docker build and push output streamed live, line prefixed - elapsed time for every build and push RunStreaming is a separate interface so the nine existing command runner fakes are unaffected. prefixWriter treats carriage returns as line ends so redrawn progress bars stream instead of accumulating. Tests cover streaming before process exit (the child blocks until the parent consumes its first line, so it deadlocks if output is buffered), stderr capture on failure, carriage-return splitting, and the emitted preparation steps. --- hack/cmd/gantry-benchmark/command.go | 86 ++++++++++ .../command_streaming_test.go | 157 ++++++++++++++++++ hack/cmd/gantry-benchmark/image.go | 94 ++++++++++- hack/cmd/gantry-benchmark/image_test.go | 22 +++ 4 files changed, 354 insertions(+), 5 deletions(-) create mode 100644 hack/cmd/gantry-benchmark/command_streaming_test.go diff --git a/hack/cmd/gantry-benchmark/command.go b/hack/cmd/gantry-benchmark/command.go index 10c48046e..8f3b8bc93 100644 --- a/hack/cmd/gantry-benchmark/command.go +++ b/hack/cmd/gantry-benchmark/command.go @@ -16,6 +16,13 @@ type commandRunner interface { Run(ctx context.Context, stdin []byte, name string, args ...string) ([]byte, error) } +// streamingCommandRunner is an optional capability. Long image builds and +// pushes emit progress for many minutes, so callers stream it live instead of +// buffering until the process exits. +type streamingCommandRunner interface { + RunStreaming(ctx context.Context, stdin []byte, progress io.Writer, name string, args ...string) ([]byte, error) +} + type execCommandRunner struct { directory string } @@ -36,6 +43,85 @@ func (r execCommandRunner) Run(ctx context.Context, stdin []byte, name string, a return output, nil } +func (r execCommandRunner) RunStreaming( + ctx context.Context, + stdin []byte, + progress io.Writer, + name string, + args ...string, +) ([]byte, error) { + command := exec.CommandContext(ctx, name, args...) + + command.Dir = r.directory + if stdin != nil { + command.Stdin = bytes.NewReader(stdin) + } + + var output bytes.Buffer + + // os/exec serializes writes when Stdout and Stderr are the same value. + sink := io.Writer(io.MultiWriter(&output, progress)) + command.Stdout = sink + command.Stderr = sink + + if err := command.Run(); err != nil { + return output.Bytes(), fmt.Errorf("%s %s: %w\n%s", name, strings.Join(args, " "), err, strings.TrimSpace(output.String())) + } + + return output.Bytes(), nil +} + +// prefixWriter reproduces child-process output one line at a time with a +// prefix. Carriage returns terminate a line so redrawn progress bars stream +// rather than accumulating into a single unbounded line. +type prefixWriter struct { + target io.Writer + prefix string + pending []byte +} + +func (w *prefixWriter) Write(p []byte) (int, error) { + w.pending = append(w.pending, p...) + + for { + index := bytes.IndexAny(w.pending, "\n\r") + if index < 0 { + break + } + + line := w.pending[:index] + w.pending = w.pending[index+1:] + + w.emit(line) + } + + return len(p), nil +} + +func (w *prefixWriter) Flush() { + if len(w.pending) == 0 { + return + } + + line := w.pending + w.pending = nil + + w.emit(line) +} + +func (w *prefixWriter) emit(line []byte) { + trimmed := strings.TrimRight(string(line), " \t") + if strings.TrimSpace(trimmed) == "" { + return + } + + writeAll(w.target, w.prefix+trimmed+"\n") +} + func writeAll(writer io.Writer, value string) { + if writer == nil { + return + } + _, _ = io.WriteString(writer, value) //nolint:errcheck // CLI progress output is best effort. } diff --git a/hack/cmd/gantry-benchmark/command_streaming_test.go b/hack/cmd/gantry-benchmark/command_streaming_test.go new file mode 100644 index 000000000..c5e28e543 --- /dev/null +++ b/hack/cmd/gantry-benchmark/command_streaming_test.go @@ -0,0 +1,157 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package main + +import ( + "bytes" + "context" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" +) + +// signalOnFirstWrite creates syncPath as soon as any output arrives, which lets +// a child process block until it observes that the parent already received it. +type signalOnFirstWrite struct { + syncPath string + mu sync.Mutex + buffer bytes.Buffer + signaled bool +} + +func (w *signalOnFirstWrite) Write(p []byte) (int, error) { + w.mu.Lock() + defer w.mu.Unlock() + + w.buffer.Write(p) + + if !w.signaled { + w.signaled = true + if err := os.WriteFile(w.syncPath, []byte("go\n"), 0o600); err != nil { + return 0, err + } + } + + return len(p), nil +} + +func (w *signalOnFirstWrite) String() string { + w.mu.Lock() + defer w.mu.Unlock() + + return w.buffer.String() +} + +// The child only emits its second line after the parent has already consumed +// the first, so this deadlocks and times out if output is buffered until exit. +func TestRunStreamingDeliversOutputBeforeProcessExits(t *testing.T) { + directory := t.TempDir() + syncPath := filepath.Join(directory, "received.flag") + progress := &signalOnFirstWrite{syncPath: syncPath} + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + script := fmt.Sprintf( + "echo first; while [ ! -f %q ]; do sleep 0.01; done; echo second", + syncPath, + ) + + output, err := execCommandRunner{directory: directory}.RunStreaming(ctx, nil, progress, "sh", "-c", script) + if err != nil { + t.Fatalf("RunStreaming: %v", err) + } + + for _, want := range []string{"first", "second"} { + if !strings.Contains(string(output), want) { + t.Fatalf("returned output %q missing %q", string(output), want) + } + + if !strings.Contains(progress.String(), want) { + t.Fatalf("streamed output %q missing %q", progress.String(), want) + } + } +} + +func TestRunStreamingCapturesStderrAndReportsFailure(t *testing.T) { + var progress bytes.Buffer + + output, err := execCommandRunner{directory: t.TempDir()}.RunStreaming( + context.Background(), nil, &progress, "sh", "-c", "echo to-stderr >&2; exit 3", + ) + if err == nil { + t.Fatal("RunStreaming succeeded, want failure") + } + + if !strings.Contains(string(output), "to-stderr") { + t.Fatalf("returned output = %q, want stderr content", string(output)) + } + + if !strings.Contains(progress.String(), "to-stderr") { + t.Fatalf("streamed output = %q, want stderr content", progress.String()) + } +} + +func TestPrefixWriterSplitsProgressRedrawsAndFlushesRemainder(t *testing.T) { + var target bytes.Buffer + + writer := &prefixWriter{target: &target, prefix: " [push] "} + + // Carriage returns are how push progress bars redraw in place. + if _, err := writer.Write([]byte("Copying blob 10%\rCopying blob 60%\rCopying blob 100%\n")); err != nil { + t.Fatalf("Write: %v", err) + } + + if _, err := writer.Write([]byte("trailing without newline")); err != nil { + t.Fatalf("Write: %v", err) + } + + if strings.Contains(target.String(), "trailing") { + t.Fatalf("partial line emitted before Flush: %q", target.String()) + } + + writer.Flush() + + want := []string{ + " [push] Copying blob 10%", + " [push] Copying blob 60%", + " [push] Copying blob 100%", + " [push] trailing without newline", + } + if got := strings.Split(strings.TrimRight(target.String(), "\n"), "\n"); !slicesEqual(got, want) { + t.Fatalf("lines = %q, want %q", got, want) + } +} + +func TestPrefixWriterSkipsBlankLines(t *testing.T) { + var target bytes.Buffer + + writer := &prefixWriter{target: &target, prefix: "> "} + + if _, err := writer.Write([]byte("\n\r\n \nreal\n")); err != nil { + t.Fatalf("Write: %v", err) + } + + if target.String() != "> real\n" { + t.Fatalf("output = %q, want %q", target.String(), "> real\n") + } +} + +func slicesEqual(a, b []string) bool { + if len(a) != len(b) { + return false + } + + for i := range a { + if a[i] != b[i] { + return false + } + } + + return true +} diff --git a/hack/cmd/gantry-benchmark/image.go b/hack/cmd/gantry-benchmark/image.go index d131a691e..22ba3a562 100644 --- a/hack/cmd/gantry-benchmark/image.go +++ b/hack/cmd/gantry-benchmark/image.go @@ -14,10 +14,37 @@ import ( "os" "path/filepath" "strings" + "time" "github.com/opencontainers/go-digest" ) +// runStreaming reports child-process output live. Image builds and pushes run +// for many minutes, so buffering until exit hides all progress. +func (b *benchmark) runStreaming(ctx context.Context, prefix, name string, args ...string) ([]byte, error) { + streamer, ok := b.commands.(streamingCommandRunner) + if !ok { + return b.commands.Run(ctx, nil, name, args...) + } + + writer := &prefixWriter{target: b.stdout, prefix: prefix} + defer writer.Flush() + + return streamer.RunStreaming(ctx, nil, writer, name, args...) +} + +func formatMiB(sizeMiB int) string { + if sizeMiB >= 1024 { + return fmt.Sprintf("%.2f GiB", float64(sizeMiB)/1024) + } + + return fmt.Sprintf("%d MiB", sizeMiB) +} + +func formatElapsed(since time.Time) string { + return time.Since(since).Round(time.Second).String() +} + func (b *benchmark) loginRegistry(ctx context.Context, loginServer, username, password string) error { _, err := b.commands.Run( ctx, @@ -203,11 +230,17 @@ func (b *benchmark) buildDualACRImages(ctx context.Context, state benchmarkState defer removePayloads(payloadPaths) + hashStarted := time.Now() + + writeAll(b.stdout, fmt.Sprintf("hashing %s shared payload\n", formatMiB(b.config.ImageSizeMiB))) + payloadSHA, err := payloadSHA256(payloadPaths) if err != nil { return "", "", "", err } + writeAll(b.stdout, fmt.Sprintf("shared payload fingerprint %s in %s\n", payloadSHA, formatElapsed(hashStarted))) + for _, phase := range []proxyPhase{proxyPhaseBaseline, proxyPhaseGantryCold} { dockerfile := dualACRDockerfile(phase, payloadPaths, payloadSHA) if err := os.WriteFile(filepath.Join(buildDirectory, "Dockerfile."+string(phase)), []byte(dockerfile), 0o640); err != nil { @@ -219,11 +252,15 @@ func (b *benchmark) buildDualACRImages(ctx context.Context, state benchmarkState baselineTagged := fmt.Sprintf("%s/%s:%s", state.BaselineACRLoginServer, b.config.WorkloadRepository, tag) gantryTagged := fmt.Sprintf("%s/%s:%s", state.GantryACRLoginServer, b.config.WorkloadRepository, tag) + writeAll(b.stdout, fmt.Sprintf("image 1 of 2: baseline -> %s\n", baselineTagged)) + baselineDigest, err := b.buildAndPushPreparedImage(ctx, buildDirectory, proxyPhaseBaseline, baselineTagged) if err != nil { return "", "", "", fmt.Errorf("build and push baseline ACR image: %w", err) } + writeAll(b.stdout, fmt.Sprintf("image 2 of 2: Gantry -> %s\n", gantryTagged)) + gantryDigest, err := b.buildAndPushPreparedImage(ctx, buildDirectory, proxyPhaseGantryCold, gantryTagged) if err != nil { return "", "", "", fmt.Errorf("build and push Gantry ACR image: %w", err) @@ -245,6 +282,14 @@ func (b *benchmark) writeImagePayloads(buildDirectory string) ([]string, error) remainderMiB := b.config.ImageSizeMiB % layers payloadPaths := make([]string, 0, layers) + started := time.Now() + writtenMiB := 0 + + writeAll(b.stdout, fmt.Sprintf( + "generating %s of random payload across %d layers in %s\n", + formatMiB(b.config.ImageSizeMiB), layers, buildDirectory, + )) + for index := range layers { sizeMiB := perLayerMiB if index == layers-1 { @@ -252,6 +297,9 @@ func (b *benchmark) writeImagePayloads(buildDirectory string) ([]string, error) } path := filepath.Join(buildDirectory, fmt.Sprintf("payload%d.bin", index)) + + layerStarted := time.Now() + if err := writeRandomPayload(path, int64(sizeMiB)*mibibyte); err != nil { removePayloads(payloadPaths) @@ -259,8 +307,21 @@ func (b *benchmark) writeImagePayloads(buildDirectory string) ([]string, error) } payloadPaths = append(payloadPaths, path) + writtenMiB += sizeMiB + + writeAll(b.stdout, fmt.Sprintf( + " payload layer %d/%d: %s in %s (%s of %s, %.0f%%)\n", + index+1, layers, formatMiB(sizeMiB), formatElapsed(layerStarted), + formatMiB(writtenMiB), formatMiB(b.config.ImageSizeMiB), + float64(writtenMiB)/float64(b.config.ImageSizeMiB)*100, + )) } + writeAll(b.stdout, fmt.Sprintf( + "payload generation complete: %s in %s\n", + formatMiB(b.config.ImageSizeMiB), formatElapsed(started), + )) + return payloadPaths, nil } @@ -315,11 +376,16 @@ func (b *benchmark) buildAndPushPreparedImage(ctx context.Context, buildDirector switch b.config.ContainerEngine { case "docker": - if _, err := b.commands.Run( + buildStarted := time.Now() + + writeAll(b.stdout, fmt.Sprintf(" [%s] docker buildx build and push starting\n", phase)) + + if _, err := b.runStreaming( ctx, - nil, + " ["+string(phase)+"] ", "docker", "buildx", "build", "--platform", b.config.ImagePlatform, + "--progress", "plain", "--file", dockerfilePath, "--tag", taggedImage, "--output", "type=image,push=true,oci-mediatypes=true", @@ -331,6 +397,8 @@ func (b *benchmark) buildAndPushPreparedImage(ctx context.Context, buildDirector return "", err } + writeAll(b.stdout, fmt.Sprintf(" [%s] build and push complete in %s\n", phase, formatElapsed(buildStarted))) + metadata, err := os.ReadFile(metadataPath) if err != nil { return "", fmt.Errorf("read Buildx metadata: %w", err) @@ -345,9 +413,13 @@ func (b *benchmark) buildAndPushPreparedImage(ctx context.Context, buildDirector imageDigest = parsed.Digest case "podman": - if _, err := b.commands.Run( + buildStarted := time.Now() + + writeAll(b.stdout, fmt.Sprintf(" [%s] podman build starting\n", phase)) + + if _, err := b.runStreaming( ctx, - nil, + " ["+string(phase)+" build] ", "podman", "build", "--isolation", "chroot", "--platform", b.config.ImagePlatform, @@ -359,10 +431,22 @@ func (b *benchmark) buildAndPushPreparedImage(ctx context.Context, buildDirector return "", err } - if _, err := b.commands.Run(ctx, nil, "podman", "push", "--digestfile", digestPath, taggedImage); err != nil { + writeAll(b.stdout, fmt.Sprintf(" [%s] build complete in %s\n", phase, formatElapsed(buildStarted))) + + pushStarted := time.Now() + + writeAll(b.stdout, fmt.Sprintf(" [%s] pushing %s to %s\n", phase, formatMiB(b.config.ImageSizeMiB), taggedImage)) + + if _, err := b.runStreaming( + ctx, + " ["+string(phase)+" push] ", + "podman", "push", "--digestfile", digestPath, taggedImage, + ); err != nil { return "", err } + writeAll(b.stdout, fmt.Sprintf(" [%s] push complete in %s\n", phase, formatElapsed(pushStarted))) + pushedDigest, err := os.ReadFile(digestPath) if err != nil { return "", fmt.Errorf("read Podman push digest: %w", err) diff --git a/hack/cmd/gantry-benchmark/image_test.go b/hack/cmd/gantry-benchmark/image_test.go index 3bb60deab..77f802361 100644 --- a/hack/cmd/gantry-benchmark/image_test.go +++ b/hack/cmd/gantry-benchmark/image_test.go @@ -4,6 +4,7 @@ package main import ( + "bytes" "context" "os" "path/filepath" @@ -58,6 +59,9 @@ func (r *dualACRImageRunner) Run(_ context.Context, _ []byte, name string, args func TestBuildDualACRImagesUsesSharedPayloadAndSameImageName(t *testing.T) { runner := &dualACRImageRunner{} + + var progress bytes.Buffer + benchmark := &benchmark{ config: benchmarkConfig{ StateRoot: t.TempDir(), @@ -68,6 +72,7 @@ func TestBuildDualACRImagesUsesSharedPayloadAndSameImageName(t *testing.T) { WorkloadRepository: "benchmark-pull", }, commands: runner, + stdout: &progress, } state := benchmarkState{ RunID: "run-1", @@ -122,6 +127,23 @@ func TestBuildDualACRImagesUsesSharedPayloadAndSameImageName(t *testing.T) { string(baselineDockerfile) == string(gantryDockerfile) { t.Fatalf("phase Dockerfiles do not isolate content cache:\nbaseline:\n%s\nGantry:\n%s", baselineDockerfile, gantryDockerfile) } + + for _, want := range []string{ + "payload layer 1/2", + "payload layer 2/2", + "payload generation complete", + "shared payload fingerprint sha256:", + "image 1 of 2: baseline -> baseline.azurecr.io/benchmark-pull:run-1", + "image 2 of 2: Gantry -> gantry.azurecr.io/benchmark-pull:run-1", + "[baseline] build complete", + "[baseline] push complete", + "[gantry_cold] build complete", + "[gantry_cold] push complete", + } { + if !strings.Contains(progress.String(), want) { + t.Fatalf("progress output is missing %q:\n%s", want, progress.String()) + } + } } func TestAdoptPreparedImages(t *testing.T) { From b6a22e6bd4d8532806d2ecce56d17364b37a5957 Mon Sep 17 00:00:00 2001 From: Vaibhav Patel Date: Thu, 6 Aug 2026 15:26:30 -0400 Subject: [PATCH 33/60] feat(gantry-benchmark): stream pull job pod progress The pull phase blocked on `kubectl wait --for=condition=complete` for its entire duration, so "running baseline pull on 1000 nodes" was the last output until the phase ended or the Job timed out. A stalled or backing-off pull was invisible for hours. Poll the Job's pods on BENCHMARK_JOB_PROGRESS_INTERVAL (default 15s) while the wait runs, and report a live breakdown: [pull-baseline] 612/1000 succeeded, 38 running, 141 pulling, 203 creating, 6 image-pull-backoff (ErrImagePull=2,ImagePullBackOff=4) (elapsed 7m30s) Empty categories are omitted, so a healthy phase stays terse. Image pull failures are broken out by reason because they indicate a real fault rather than slowness. kubectl wait remains the authoritative completion signal. The reporter stops on context cancellation and the caller waits for it to finish so its output cannot interleave with the phase result. It is disabled when the interval is non-positive, keeping existing tests single-threaded. --- hack/cmd/gantry-benchmark/config.go | 7 + hack/cmd/gantry-benchmark/job.go | 193 ++++++++++++++- .../cmd/gantry-benchmark/job_progress_test.go | 224 ++++++++++++++++++ 3 files changed, 421 insertions(+), 3 deletions(-) create mode 100644 hack/cmd/gantry-benchmark/job_progress_test.go diff --git a/hack/cmd/gantry-benchmark/config.go b/hack/cmd/gantry-benchmark/config.go index 05420db7e..f2ccd8e94 100644 --- a/hack/cmd/gantry-benchmark/config.go +++ b/hack/cmd/gantry-benchmark/config.go @@ -70,6 +70,7 @@ type benchmarkConfig struct { ACRPrivateEndpointResourceID string TelemetryTimeout time.Duration TelemetryPollInterval time.Duration + JobProgressInterval time.Duration StateRoot string } @@ -130,6 +131,11 @@ func loadBenchmarkConfig(getenv func(string) string) (benchmarkConfig, error) { return benchmarkConfig{}, err } + jobProgressInterval, err := envDuration(getenv, "BENCHMARK_JOB_PROGRESS_INTERVAL", 15*time.Second) + if err != nil { + return benchmarkConfig{}, err + } + mode := benchmarkMode(envDefault(getenv, "BENCHMARK_MODE", string(benchmarkModeProxy))) if mode != benchmarkModeProxy && mode != benchmarkModeDirect { return benchmarkConfig{}, fmt.Errorf( @@ -182,6 +188,7 @@ func loadBenchmarkConfig(getenv func(string) string) (benchmarkConfig, error) { ACRPrivateEndpointResourceID: getenv("AZURE_ACR_PRIVATE_ENDPOINT_RESOURCE_ID"), TelemetryTimeout: telemetryTimeout, TelemetryPollInterval: telemetryPollInterval, + JobProgressInterval: jobProgressInterval, StateRoot: filepath.Join(repoRoot, "tmp", "gantry-benchmark"), } diff --git a/hack/cmd/gantry-benchmark/job.go b/hack/cmd/gantry-benchmark/job.go index 32ec6cdcc..ed091d3ac 100644 --- a/hack/cmd/gantry-benchmark/job.go +++ b/hack/cmd/gantry-benchmark/job.go @@ -136,14 +136,21 @@ func (b *benchmark) runPullJob(ctx context.Context, state benchmarkState, phase waitContext, cancel := context.WithTimeout(ctx, b.config.JobTimeout) defer cancel() - if _, err := b.commands.Run( + progressStopped := b.startPullJobProgress(waitContext, jobName, phaseStartedAt) + + _, waitErr := b.commands.Run( waitContext, nil, "kubectl", "-n", b.config.Namespace, "wait", "--for=condition=complete", "job/"+jobName, "--timeout", b.config.JobTimeout.String(), - ); err != nil { - return jobObservation{}, err + ) + + cancel() + <-progressStopped + + if waitErr != nil { + return jobObservation{}, waitErr } phaseFinishedAt := time.Now().UTC() @@ -182,8 +189,188 @@ func pullContainer(image string) map[string]any { } } +// pullPodList is the reduced projection used for live progress. It reads the +// waiting reason that podList intentionally omits. +type pullPodList struct { + Items []struct { + Spec struct { + NodeName string `json:"nodeName"` + } `json:"spec"` + Status struct { + Phase string `json:"phase"` + ContainerStatuses []struct { + Name string `json:"name"` + State struct { + Waiting *struct { + Reason string `json:"reason"` + } `json:"waiting"` + } `json:"state"` + } `json:"containerStatuses"` + } `json:"status"` + } `json:"items"` +} + +// pullProgress is a point-in-time breakdown of the pull Job's pods, reported +// while the phase runs so a stalled pull is visible before the Job times out. +type pullProgress struct { + Total int + Succeeded int + Running int + ContainerCreating int + PullingImage int + ImagePullBackOff int + Unscheduled int + Failed int + Other int + BackOffReasons map[string]int +} + +func (p pullProgress) String() string { + parts := []string{fmt.Sprintf("%d/%d succeeded", p.Succeeded, p.Total)} + + for _, entry := range []struct { + label string + count int + }{ + {"running", p.Running}, + {"pulling", p.PullingImage}, + {"creating", p.ContainerCreating}, + {"unscheduled", p.Unscheduled}, + {"failed", p.Failed}, + {"other", p.Other}, + } { + if entry.count > 0 { + parts = append(parts, fmt.Sprintf("%d %s", entry.count, entry.label)) + } + } + + if p.ImagePullBackOff > 0 { + reasons := make([]string, 0, len(p.BackOffReasons)) + for reason, count := range p.BackOffReasons { + reasons = append(reasons, fmt.Sprintf("%s=%d", reason, count)) + } + + sort.Strings(reasons) + parts = append(parts, fmt.Sprintf("%d image-pull-backoff (%s)", p.ImagePullBackOff, strings.Join(reasons, ","))) + } + + return strings.Join(parts, ", ") +} + +func summarizePullProgress(raw []byte, expectedPods int) (pullProgress, error) { + var pods pullPodList + if err := json.Unmarshal(raw, &pods); err != nil { + return pullProgress{}, fmt.Errorf("decode pull Job pod progress: %w", err) + } + + progress := pullProgress{Total: expectedPods, BackOffReasons: map[string]int{}} + + for _, pod := range pods.Items { + switch pod.Status.Phase { + case "Succeeded": + progress.Succeeded++ + + continue + case "Failed": + progress.Failed++ + + continue + case "Running": + progress.Running++ + + continue + } + + waitingReason := "" + + for _, status := range pod.Status.ContainerStatuses { + if status.Name == "pull" && status.State.Waiting != nil { + waitingReason = status.State.Waiting.Reason + } + } + + switch waitingReason { + case "ImagePullBackOff", "ErrImagePull", "RegistryUnavailable": + progress.ImagePullBackOff++ + progress.BackOffReasons[waitingReason]++ + case "ContainerCreating", "PodInitializing": + progress.ContainerCreating++ + case "": + if pod.Spec.NodeName == "" { + progress.Unscheduled++ + } else { + progress.PullingImage++ + } + default: + progress.Other++ + } + } + + return progress, nil +} + +// startPullJobProgress streams pod-state counts until ctx is cancelled. The +// returned channel closes once the reporter has stopped, so the caller can +// avoid interleaving output with whatever it prints next. +func (b *benchmark) startPullJobProgress(ctx context.Context, jobName string, phaseStartedAt time.Time) <-chan struct{} { + stopped := make(chan struct{}) + + if b.config.JobProgressInterval <= 0 { + close(stopped) + + return stopped + } + + go func() { + defer close(stopped) + + ticker := time.NewTicker(b.config.JobProgressInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + + output, err := b.commands.Run( + ctx, + nil, + "kubectl", "-n", b.config.Namespace, + "get", "pods", "-l", "job-name="+jobName, + "-o", "json", + ) + if err != nil { + if ctx.Err() != nil { + return + } + + writeAll(b.stdout, fmt.Sprintf(" [%s] pod progress unavailable: %v\n", jobName, err)) + + continue + } + + progress, err := summarizePullProgress(output, b.config.NodeCount) + if err != nil { + writeAll(b.stdout, fmt.Sprintf(" [%s] pod progress unreadable: %v\n", jobName, err)) + + continue + } + + writeAll(b.stdout, fmt.Sprintf( + " [%s] %s (elapsed %s)\n", + jobName, progress, time.Since(phaseStartedAt).Round(time.Second), + )) + } + }() + + return stopped +} + func parseJobObservation(raw []byte, expectedPods int, phaseStartedAt time.Time) (jobObservation, error) { var pods podList + if err := json.Unmarshal(raw, &pods); err != nil { return jobObservation{}, fmt.Errorf("decode pull Job pods: %w", err) } diff --git a/hack/cmd/gantry-benchmark/job_progress_test.go b/hack/cmd/gantry-benchmark/job_progress_test.go new file mode 100644 index 000000000..f8a38e857 --- /dev/null +++ b/hack/cmd/gantry-benchmark/job_progress_test.go @@ -0,0 +1,224 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package main + +import ( + "bytes" + "context" + "encoding/json" + "strings" + "sync" + "testing" + "time" +) + +func pullProgressFixture(t *testing.T, pods []map[string]any) []byte { + t.Helper() + + raw, err := json.Marshal(map[string]any{"items": pods}) + if err != nil { + t.Fatalf("marshal fixture: %v", err) + } + + return raw +} + +func waitingPod(node, reason string) map[string]any { + return map[string]any{ + "spec": map[string]any{"nodeName": node}, + "status": map[string]any{ + "phase": "Pending", + "containerStatuses": []any{map[string]any{ + "name": "pull", + "state": map[string]any{"waiting": map[string]any{"reason": reason}}, + }}, + }, + } +} + +func phasePod(node, phase string) map[string]any { + return map[string]any{ + "spec": map[string]any{"nodeName": node}, + "status": map[string]any{"phase": phase}, + } +} + +func TestSummarizePullProgressClassifiesEveryPodState(t *testing.T) { + raw := pullProgressFixture(t, []map[string]any{ + phasePod("node-a", "Succeeded"), + phasePod("node-b", "Succeeded"), + phasePod("node-c", "Running"), + phasePod("node-d", "Failed"), + waitingPod("node-e", "ContainerCreating"), + waitingPod("node-f", "PodInitializing"), + waitingPod("node-g", "ImagePullBackOff"), + waitingPod("node-h", "ErrImagePull"), + waitingPod("node-i", "CreateContainerConfigError"), + // Scheduled with no waiting state: kubelet is pulling the image. + phasePod("node-j", "Pending"), + // No node assigned yet. + phasePod("", "Pending"), + }) + + progress, err := summarizePullProgress(raw, 11) + if err != nil { + t.Fatalf("summarizePullProgress: %v", err) + } + + for _, check := range []struct { + name string + got int + want int + }{ + {"total", progress.Total, 11}, + {"succeeded", progress.Succeeded, 2}, + {"running", progress.Running, 1}, + {"failed", progress.Failed, 1}, + {"containerCreating", progress.ContainerCreating, 2}, + {"imagePullBackOff", progress.ImagePullBackOff, 2}, + {"other", progress.Other, 1}, + {"pullingImage", progress.PullingImage, 1}, + {"unscheduled", progress.Unscheduled, 1}, + } { + if check.got != check.want { + t.Errorf("%s = %d, want %d", check.name, check.got, check.want) + } + } + + rendered := progress.String() + for _, want := range []string{ + "2/11 succeeded", + "1 running", + "2 creating", + "1 unscheduled", + "1 failed", + "2 image-pull-backoff (ErrImagePull=1,ImagePullBackOff=1)", + } { + if !strings.Contains(rendered, want) { + t.Errorf("rendered progress %q is missing %q", rendered, want) + } + } +} + +func TestSummarizePullProgressOmitsEmptyCategories(t *testing.T) { + raw := pullProgressFixture(t, []map[string]any{ + phasePod("node-a", "Succeeded"), + phasePod("node-b", "Succeeded"), + }) + + progress, err := summarizePullProgress(raw, 2) + if err != nil { + t.Fatalf("summarizePullProgress: %v", err) + } + + if got := progress.String(); got != "2/2 succeeded" { + t.Fatalf("rendered progress = %q, want %q", got, "2/2 succeeded") + } +} + +type progressPollRunner struct { + mu sync.Mutex + calls int + output []byte +} + +func (r *progressPollRunner) Run(_ context.Context, _ []byte, _ string, _ ...string) ([]byte, error) { + r.mu.Lock() + defer r.mu.Unlock() + + r.calls++ + + return r.output, nil +} + +func (r *progressPollRunner) callCount() int { + r.mu.Lock() + defer r.mu.Unlock() + + return r.calls +} + +func TestStartPullJobProgressReportsUntilCancelled(t *testing.T) { + runner := &progressPollRunner{ + output: pullProgressFixture(t, []map[string]any{ + phasePod("node-a", "Succeeded"), + waitingPod("node-b", "ImagePullBackOff"), + }), + } + + var progress lockedBuffer + + bench := &benchmark{ + config: benchmarkConfig{ + Namespace: "gantry-benchmark", + NodeCount: 2, + JobProgressInterval: time.Millisecond, + }, + commands: runner, + stdout: &progress, + } + + ctx, cancel := context.WithCancel(context.Background()) + stopped := bench.startPullJobProgress(ctx, "pull-job", time.Now()) + + deadline := time.After(10 * time.Second) + + for runner.callCount() == 0 { + select { + case <-deadline: + cancel() + <-stopped + t.Fatal("progress reporter never polled") + default: + } + } + + cancel() + <-stopped + + if !strings.Contains(progress.String(), "1/2 succeeded") || + !strings.Contains(progress.String(), "image-pull-backoff") { + t.Fatalf("progress output = %q", progress.String()) + } +} + +func TestStartPullJobProgressDisabledWhenIntervalUnset(t *testing.T) { + runner := &progressPollRunner{} + bench := &benchmark{ + config: benchmarkConfig{Namespace: "gantry-benchmark", NodeCount: 2}, + commands: runner, + stdout: &bytes.Buffer{}, + } + + stopped := bench.startPullJobProgress(context.Background(), "pull-job", time.Now()) + + select { + case <-stopped: + case <-time.After(5 * time.Second): + t.Fatal("disabled reporter did not close its channel") + } + + if got := runner.callCount(); got != 0 { + t.Fatalf("poll calls = %d, want 0", got) + } +} + +type lockedBuffer struct { + mu sync.Mutex + buffer bytes.Buffer +} + +func (b *lockedBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + + return b.buffer.Write(p) +} + +func (b *lockedBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + + return b.buffer.String() +} From dbe89481a8a92c1cadb1c93f5996ade2da3e7b84 Mon Sep 17 00:00:00 2001 From: Vaibhav Patel Date: Thu, 6 Aug 2026 15:32:49 -0400 Subject: [PATCH 34/60] feat(gantry-benchmark): timestamp all progress output Long phases emitted untimestamped lines, so a log could not answer how long a step took or when it stalled. Go tool: wrap stdout and stderr in a timestampWriter at the process boundary, so every line, including streamed podman output and the pull progress reporter, is prefixed with a UTC timestamp. Wrapping once avoids touching every call site. Blank separator lines stay blank rather than becoming a bare timestamp, and the fatal error now goes through the same writer instead of bypassing it. The writer is line buffered, so fragments are stamped once per line, and Flush emits a trailing line without a newline. Its lock also serializes the concurrent pull-progress reporter with the main goroutine. Shell: deploy.sh log() and a new operator-vm-run.sh log() stamp their messages, and operator-vm-watch.sh prints a snapshot timestamp on every refresh so a stale screen is obvious. --- hack/cmd/gantry-benchmark/command.go | 69 ++++++++++++++ .../command_streaming_test.go | 94 +++++++++++++++++++ hack/cmd/gantry-benchmark/main.go | 14 ++- hack/gantry-benchmark/deploy.sh | 2 +- hack/gantry-benchmark/operator-vm-run.sh | 10 +- hack/gantry-benchmark/operator-vm-watch.sh | 2 + 6 files changed, 184 insertions(+), 7 deletions(-) diff --git a/hack/cmd/gantry-benchmark/command.go b/hack/cmd/gantry-benchmark/command.go index 8f3b8bc93..c2cd75add 100644 --- a/hack/cmd/gantry-benchmark/command.go +++ b/hack/cmd/gantry-benchmark/command.go @@ -10,6 +10,8 @@ import ( "io" "os/exec" "strings" + "sync" + "time" ) type commandRunner interface { @@ -118,6 +120,73 @@ func (w *prefixWriter) emit(line []byte) { writeAll(w.target, w.prefix+trimmed+"\n") } +// timestampWriter prefixes each complete line with a UTC timestamp. It is +// applied once at the process boundary so every progress line is stamped, and +// its lock also serializes the concurrent pull-progress reporter. +type timestampWriter struct { + target io.Writer + now func() time.Time + mu sync.Mutex + pending []byte +} + +func (w *timestampWriter) Write(p []byte) (int, error) { + w.mu.Lock() + defer w.mu.Unlock() + + w.pending = append(w.pending, p...) + + for { + index := bytes.IndexByte(w.pending, '\n') + if index < 0 { + break + } + + line := w.pending[:index] + w.pending = w.pending[index+1:] + + if err := w.emit(line); err != nil { + return len(p), err + } + } + + return len(p), nil +} + +// Flush writes any line not terminated by a newline, so a final partial line +// is not lost when the process exits. +func (w *timestampWriter) Flush() { + w.mu.Lock() + defer w.mu.Unlock() + + if len(w.pending) == 0 { + return + } + + line := w.pending + w.pending = nil + + _ = w.emit(line) //nolint:errcheck // Progress output is best effort. +} + +func (w *timestampWriter) emit(line []byte) error { + // Blank separator lines stay blank rather than becoming a bare timestamp. + if len(bytes.TrimSpace(line)) == 0 { + _, err := io.WriteString(w.target, "\n") + + return err + } + + clock := w.now + if clock == nil { + clock = time.Now + } + + _, err := io.WriteString(w.target, clock().UTC().Format("2006-01-02T15:04:05Z")+" "+string(line)+"\n") + + return err +} + func writeAll(writer io.Writer, value string) { if writer == nil { return diff --git a/hack/cmd/gantry-benchmark/command_streaming_test.go b/hack/cmd/gantry-benchmark/command_streaming_test.go index c5e28e543..e521eb9ec 100644 --- a/hack/cmd/gantry-benchmark/command_streaming_test.go +++ b/hack/cmd/gantry-benchmark/command_streaming_test.go @@ -155,3 +155,97 @@ func slicesEqual(a, b []string) bool { return true } + +func TestTimestampWriterStampsEveryLine(t *testing.T) { + var target bytes.Buffer + + fixed := time.Date(2026, 8, 6, 19, 30, 5, 0, time.UTC) + writer := ×tampWriter{target: &target, now: func() time.Time { return fixed }} + + if _, err := writer.Write([]byte("first\nsecond\n")); err != nil { + t.Fatalf("Write: %v", err) + } + + want := "2026-08-06T19:30:05Z first\n2026-08-06T19:30:05Z second\n" + if target.String() != want { + t.Fatalf("output = %q, want %q", target.String(), want) + } +} + +// Progress arriving in fragments must still produce exactly one stamp per line. +func TestTimestampWriterStampsOncePerLineAcrossPartialWrites(t *testing.T) { + var target bytes.Buffer + + fixed := time.Date(2026, 8, 6, 19, 30, 5, 0, time.UTC) + writer := ×tampWriter{target: &target, now: func() time.Time { return fixed }} + + for _, fragment := range []string{"pay", "load ", "layer 1/40"} { + if _, err := writer.Write([]byte(fragment)); err != nil { + t.Fatalf("Write: %v", err) + } + } + + if target.String() != "" { + t.Fatalf("partial line emitted before newline: %q", target.String()) + } + + if _, err := writer.Write([]byte("\n")); err != nil { + t.Fatalf("Write: %v", err) + } + + want := "2026-08-06T19:30:05Z payload layer 1/40\n" + if target.String() != want { + t.Fatalf("output = %q, want %q", target.String(), want) + } +} + +func TestTimestampWriterFlushEmitsTrailingLine(t *testing.T) { + var target bytes.Buffer + + fixed := time.Date(2026, 8, 6, 19, 30, 5, 0, time.UTC) + writer := ×tampWriter{target: &target, now: func() time.Time { return fixed }} + + if _, err := writer.Write([]byte("no trailing newline")); err != nil { + t.Fatalf("Write: %v", err) + } + + writer.Flush() + + want := "2026-08-06T19:30:05Z no trailing newline\n" + if target.String() != want { + t.Fatalf("output = %q, want %q", target.String(), want) + } +} + +func TestTimestampWriterIsConcurrencySafe(t *testing.T) { + var target bytes.Buffer + + writer := ×tampWriter{target: &target} + + var group sync.WaitGroup + + for i := range 8 { + group.Add(1) + + go func() { + defer group.Done() + + for j := range 25 { + writeAll(writer, fmt.Sprintf("worker %d line %d\n", i, j)) + } + }() + } + + group.Wait() + + lines := strings.Split(strings.TrimRight(target.String(), "\n"), "\n") + if len(lines) != 200 { + t.Fatalf("lines = %d, want 200", len(lines)) + } + + for _, line := range lines { + if !strings.Contains(line, "Z worker ") { + t.Fatalf("line is missing its timestamp: %q", line) + } + } +} diff --git a/hack/cmd/gantry-benchmark/main.go b/hack/cmd/gantry-benchmark/main.go index 674617636..e8dcc38fe 100644 --- a/hack/cmd/gantry-benchmark/main.go +++ b/hack/cmd/gantry-benchmark/main.go @@ -23,8 +23,18 @@ func main() { ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() - if err := runCLI(ctx, os.Args[1:], os.Stdout, os.Stderr); err != nil { - fmt.Fprintln(os.Stderr, err) + stdout := ×tampWriter{target: os.Stdout} + stderr := ×tampWriter{target: os.Stderr} + + defer func() { + stdout.Flush() + stderr.Flush() + }() + + if err := runCLI(ctx, os.Args[1:], stdout, stderr); err != nil { + writeAll(stderr, err.Error()+"\n") + stdout.Flush() + stderr.Flush() os.Exit(1) } } diff --git a/hack/gantry-benchmark/deploy.sh b/hack/gantry-benchmark/deploy.sh index 0048c6492..3a34c0301 100755 --- a/hack/gantry-benchmark/deploy.sh +++ b/hack/gantry-benchmark/deploy.sh @@ -185,7 +185,7 @@ assert_default MONITORING_NAMESPACE "$MONITORING_NAMESPACE" monitoring assert_default KPS_RELEASE "$KPS_RELEASE" kps assert_default PROMETHEUS_SERVICE "$PROMETHEUS_SERVICE" kps-kube-prometheus-stack-prometheus -log() { printf '[deploy] %s\n' "$*"; } +log() { printf '%s [deploy] %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*"; } require_command() { command -v "$1" >/dev/null 2>&1 || { echo "required command not found: $1" >&2; exit 1; } diff --git a/hack/gantry-benchmark/operator-vm-run.sh b/hack/gantry-benchmark/operator-vm-run.sh index 853507111..8f4100cee 100755 --- a/hack/gantry-benchmark/operator-vm-run.sh +++ b/hack/gantry-benchmark/operator-vm-run.sh @@ -32,6 +32,8 @@ chmod 0700 "$HOME" LOG_FILE="$BENCHMARK_ARTIFACT_ROOT/operator.log" exec > >(tee -a "$LOG_FILE") 2>&1 +log() { printf '%s [operator] %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*"; } + run_id="" run_status=0 cleanup_started=false @@ -66,7 +68,7 @@ cleanup() { if [[ -f "$KUBECONFIG" ]]; then if kubectl -n "${BENCHMARK_NAMESPACE:-gantry-benchmark}" get configmap gantry-benchmark-state >/dev/null 2>&1 || \ kubectl -n "${GANTRY_NAMESPACE:-gantry-system}" get configmap gantry-benchmark-lock >/dev/null 2>&1; then - echo "restoring benchmark cluster state" + log "restoring benchmark cluster state" make -C "$BENCHMARK_REPO_ROOT/hack/gantry-benchmark" disable || original_status=$? fi fi @@ -104,7 +106,7 @@ trap 'exit 143' TERM cd "$BENCHMARK_REPO_ROOT" write_progress "authenticate" "authenticating managed identity and loading kubeconfig" -echo "authenticating operator VM managed identity" +log "authenticating operator VM managed identity" az login --identity --allow-no-subscriptions --output none az account set --subscription "$AZURE_SUBSCRIPTION_ID" @@ -119,7 +121,7 @@ chmod 0600 "$KUBECONFIG" kubectl auth can-i '*' '*' --all-namespaces | grep -qx yes export BENCHMARK_CONFIRM_CONTEXT="$(kubectl config current-context)" -echo "using Kubernetes context $BENCHMARK_CONFIRM_CONTEXT" +log "using Kubernetes context $BENCHMARK_CONFIRM_CONTEXT" if kubectl -n "${BENCHMARK_NAMESPACE:-gantry-benchmark}" get configmap gantry-benchmark-state >/dev/null 2>&1; then echo "an active benchmark state already exists; run disable before starting a new VM lifecycle" >&2 @@ -129,7 +131,7 @@ fi write_progress "enable" "installing benchmark state, lock, and monitoring" make -C hack/gantry-benchmark enable run_id=$(kubectl -n "${BENCHMARK_NAMESPACE:-gantry-benchmark}" get configmap gantry-benchmark-state -o jsonpath='{.data.state\.json}' | jq -er '.run_id') -echo "enabled benchmark $run_id" +log "enabled benchmark $run_id" if [[ -n "${ADOPT_BASELINE_IMAGE:-}" || -n "${ADOPT_GANTRY_IMAGE:-}" || -n "${ADOPT_PAYLOAD_SHA256:-}" ]]; then : "${ADOPT_BASELINE_IMAGE:?Set ADOPT_BASELINE_IMAGE with the full adoption set}" : "${ADOPT_GANTRY_IMAGE:?Set ADOPT_GANTRY_IMAGE with the full adoption set}" diff --git a/hack/gantry-benchmark/operator-vm-watch.sh b/hack/gantry-benchmark/operator-vm-watch.sh index ff724ac23..d37861d9a 100755 --- a/hack/gantry-benchmark/operator-vm-watch.sh +++ b/hack/gantry-benchmark/operator-vm-watch.sh @@ -88,6 +88,7 @@ status_once() { } if [[ "$follow" == false ]]; then + printf '%s snapshot\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" status_once exit 0 fi @@ -95,6 +96,7 @@ fi while true; do printf '\033[2J\033[H' status=$(status_once) + printf '%s snapshot (refreshing every %ss)\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$WATCH_INTERVAL_SECONDS" printf '%s\n' "$status" service=$(awk -F': ' '/^service: /{print $2; exit}' <<<"$status") From 29ef0d061bdbe8df9e371cf93059d11b9bf788c0 Mon Sep 17 00:00:00 2001 From: Vaibhav Patel Date: Thu, 6 Aug 2026 16:26:08 -0400 Subject: [PATCH 35/60] docs(gantry-benchmark): record Canada Central rerun results Add run-20260806-185139-9995e167 to the ACR traffic, pod startup latency, and Gantry internals tables. The run reported FAIL. Byte reduction of 99.656% and pull reduction of 99.800% are the strongest Canada Central figures recorded, but the baseline pulled unusually fast at a P95 of 1091.173s, roughly half the earlier Canada Central sample. Gantry's 1180.970s is in line with its usual range, so the P95 ratio of 1.0823 exceeded the 1.0 gate this run was configured with. Correct the surrounding text, which claimed every run passed its gate. --- hack/gantry-benchmark/RESULTS.md | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/hack/gantry-benchmark/RESULTS.md b/hack/gantry-benchmark/RESULTS.md index 64422fd1e..6c37241a1 100644 --- a/hack/gantry-benchmark/RESULTS.md +++ b/hack/gantry-benchmark/RESULTS.md @@ -24,6 +24,7 @@ logs, and measured peer traffic from Gantry metrics. | 1000 nodes - sample 1 | 40 GiB | 47.296 TB | 174.773 GB | 99.630% | 1002 / 4 | 99.601% | | 1000 nodes - sample 2 | 40 GiB | 47.566 TB | 174.781 GB | 99.633% | 1008 / 5 | 99.504% | | **1000 nodes - Canada Central ACR** | **40 GiB** | **47.178 TB** | **245.878 GB** | **99.479%** | **1000 / 2** | **99.800%** | +| **1000 nodes - Canada Central ACR rerun** 3 | **40 GiB** | **45.008 TB** | **154.787 GB** | **99.656%** | **1000 / 2** | **99.800%** | | **1000 nodes - UK South ACR** | **40 GiB** | **53.369 TB** | **219.262 GB** | **99.589%** | **1254 / 5** | **99.601%** | | **1000 nodes - East US ACR** | **40 GiB** | **47.562 TB** | **182.317 GB** | **99.617%** | **1004 / 4** | **99.602%** | | **1000 nodes - Central India ACR** 1 | **40 GiB** | **97.115 TB** | **803.184 GB** | **99.173%** | **2287 / 6** | **99.738%** | @@ -48,6 +49,7 @@ aggregates below. | 1000 nodes - sample 1 | 40 GiB | 682.725s | 1099.629s | 821.209s | 1171.863s | 1422.461s | 1885.385s | 42.700% slower | | 1000 nodes - sample 2 | 40 GiB | 894.597s | 1087.179s | 1065.724s | 1169.448s | 1652.704s | 1885.011s | 9.733% slower | | **1000 nodes - Canada Central ACR** | **40 GiB** | **1862.580s** | **774.028s** | **2030.636s** | **817.139s** | **2149.535s** | **891.766s** | **59.759% faster** | +| **1000 nodes - Canada Central ACR rerun** 3 | **40 GiB** | **939.368s** | **1105.766s** | **1091.173s** | **1180.970s** | **1180.111s** | **1239.429s** | **8.229% slower** | | **1000 nodes - UK South ACR** | **40 GiB** | **3561.000s** | **1064.557s** | **3953.000s** | **1146.557s** | **5399.000s** | **1815.557s** | **70.995% faster** | | **1000 nodes - East US ACR** | **40 GiB** | **1401.026s** | **1065.950s** | **1655.894s** | **1144.771s** | **2351.081s** | **1831.832s** | **30.867% faster** | | **1000 nodes - Central India ACR** 2 | **40 GiB** | **3184.649s** | **1065.570s** | **4851.649s** | **1155.570s** | **5351.649s** | **1865.570s** | **76.182% faster** | @@ -55,16 +57,28 @@ aggregates below. | 2000 nodes - sample 2 | 40 GiB | 939.834s | 1093.091s | 1141.022s | 1177.821s | 1724.219s | 1856.380s | 3.225% slower | | 2000 nodes - sample 3 | 40 GiB | 1241.331s | 1096.589s | 1472.041s | 1184.248s | 2131.053s | 1821.000s | 19.551% faster | -Positive improvement means Gantry started pods faster. The configured gate was -a maximum Gantry-to-baseline P95 ratio of 3.0, so all six audit-complete runs -and the cross-region performance-only samples passed even when an unusually -fast baseline made Gantry slower. The UK South and Central India rows use -retained Kubernetes pod status timestamps; every other row, including East US, -uses AKS audit timestamps. +Positive improvement means Gantry started pods faster. Most rows used a maximum +Gantry-to-baseline P95 ratio of 3.0, so all six audit-complete runs and the +cross-region performance-only samples passed even when an unusually fast +baseline made Gantry slower. The Canada Central rerun is the exception: it ran +with the gate tightened to 1.0 and did not meet it. The UK South and Central +India rows use retained Kubernetes pod status timestamps; every other row, +including East US, uses AKS audit timestamps. 2 Central India latency uses retained Kubernetes pod status because the telemetry timeout occurred before the runner wrote its audit measurement. +3 Run `run-20260806-185139-9995e167`, reported **FAIL**. Byte and +pull reduction were the strongest Canada Central results recorded, but the +baseline was unusually fast: its P95 of 1091.173s is roughly half the 2030.636s +of the earlier Canada Central sample, while Gantry landed at 1180.970s, close to +the 1146.557s to 1184.248s that Gantry produces on nearly every run. The +resulting P95 ratio of 1.0823 exceeded the 1.0 gate configured for this run, +which the 3.0 gate used elsewhere would have passed. All 2000 pods across both +phases succeeded with no image-pull backoff and no origin fallbacks. This +sample also ran with the containerd transfer-service download concurrency +override removed, so both phases used the containerd default of 3. + #### Latency excluding image-pull backoff AKS audit logs retained the pod status patches containing `ErrImagePull` and @@ -92,6 +106,7 @@ The unfiltered table remains the primary end-to-end result because | 1000 nodes - sample 1 | 40 GiB | 43.566 TB | 158.928 GB | 157 | 153 | 42,597 | 0 | | 1000 nodes - sample 2 | 40 GiB | 43.438 TB | 160.002 GB | 159 | 154 | 42,472 | 0 | | **1000 nodes - Canada Central ACR** | **40 GiB** | **42.735 TB** | **223.358 GB** | **214** | **212** | **41,791** | **0** | +| **1000 nodes - Canada Central ACR rerun** 3 | **40 GiB** | **42.817 TB** | **140.672 GB** | **134** | **132** | **41,870** | **0** | | **1000 nodes - East US ACR** | **40 GiB** | **43.137 TB** | **161.075 GB** | **159** | **155** | **42,179** | **0** | | **1000 nodes - Central India ACR** | **40 GiB** | **43.070 TB** | **709.766 GB** | **682** | **662** | **42,129** | **0** | | 2000 nodes - sample 1 | 40 GiB | 86.971 TB | 221.210 GB | 213 | 210 | 85,044 | 0 | From 66ad0ed78ace11ec76c50747a82b294ec2ee0517 Mon Sep 17 00:00:00 2001 From: Vaibhav Patel Date: Thu, 6 Aug 2026 16:43:01 -0400 Subject: [PATCH 36/60] feat(gantry-benchmark): restore max_concurrent_downloads override Reinstate the transfer-service override at 6 concurrent layer downloads for a controlled comparison against run-20260806-185139-9995e167, which ran at the containerd default of 3 and produced an unusually fast baseline. Restores the conf.d drop-in setting, its readiness assertion, and the manifest test guard, and the runbook note. --- hack/cmd/gantry-benchmark/enable_test.go | 1 + hack/gantry-benchmark/RUNBOOK.md | 6 +++--- hack/gantry-benchmark/manifests/containerd.yaml | 4 ++++ 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/hack/cmd/gantry-benchmark/enable_test.go b/hack/cmd/gantry-benchmark/enable_test.go index 66c32b21f..800bebc2a 100644 --- a/hack/cmd/gantry-benchmark/enable_test.go +++ b/hack/cmd/gantry-benchmark/enable_test.go @@ -158,6 +158,7 @@ func TestContainerdBenchmarkManifest(t *testing.T) { for _, setting := range []string{ `level = "debug"`, `image_pull_progress_timeout = "15m"`, + `max_concurrent_downloads = 6`, `systemd-run`, } { if !bytes.Contains(manifest, []byte(setting)) { diff --git a/hack/gantry-benchmark/RUNBOOK.md b/hack/gantry-benchmark/RUNBOOK.md index b9113bfae..fe9b728cc 100644 --- a/hack/gantry-benchmark/RUNBOOK.md +++ b/hack/gantry-benchmark/RUNBOOK.md @@ -91,9 +91,9 @@ The graphroot must be `/opt/gantry-benchmark/containers`. Before provisioning the operator VM or starting the lifecycle on AKS, apply the benchmark containerd configuration and require it to be Ready on every -target node. This enables debug unpack logs and sets the no-progress timeout -to 15 minutes. The DaemonSet performs one detached containerd restart per -configuration hash. +target node. This enables debug unpack logs, sets the no-progress timeout to +15 minutes, and raises transfer-service layer downloads to six. The DaemonSet +performs one detached containerd restart per configuration hash. ```bash kubectl create namespace gantry-system --dry-run=client -o yaml | kubectl apply -f - diff --git a/hack/gantry-benchmark/manifests/containerd.yaml b/hack/gantry-benchmark/manifests/containerd.yaml index d9e0cbce9..2299172a1 100644 --- a/hack/gantry-benchmark/manifests/containerd.yaml +++ b/hack/gantry-benchmark/manifests/containerd.yaml @@ -12,6 +12,9 @@ data: [plugins."io.containerd.cri.v1.images"] image_pull_progress_timeout = "15m" + + [plugins."io.containerd.transfer.v1.local"] + max_concurrent_downloads = 6 --- apiVersion: apps/v1 kind: DaemonSet @@ -103,6 +106,7 @@ spec: ')" printf '%s\n' "$config_dump" | grep -Eq "^[[:space:]]*level = ['\"]debug['\"]$" printf '%s\n' "$config_dump" | grep -Eq "^[[:space:]]*image_pull_progress_timeout = ['\"]15m(0s)?['\"]$" + printf '%s\n' "$config_dump" | grep -Eq '^[[:space:]]*max_concurrent_downloads = 6$' initialDelaySeconds: 10 periodSeconds: 10 failureThreshold: 30 From 53c478eb27e30757a20eedecc288720fe954c1ff Mon Sep 17 00:00:00 2001 From: Vaibhav Patel Date: Thu, 6 Aug 2026 18:21:38 -0400 Subject: [PATCH 37/60] docs(gantry-benchmark): analyze pod startup latency breakdown Add PULL-LATENCY-ANALYSIS.md decomposing where per-pod startup time goes on a 1000-node pull of a 40 GiB, 40-layer image, using two runs that differ only in containerd max_concurrent_downloads (3 vs 6). Findings: - Scheduling is 0.3% of startup. Effectively all wall time is image pull. - Per-layer unpack costs a stable 8.33s to 8.40s across both runs and both phases, so 40 layers cost roughly 333s per node regardless of where the bytes come from. - max_concurrent_unpacks defaults to 1, which leaves that 333s strictly serialized. All 2000 image unpacks in each run report parallel=false, and no rebase-capability warning appears, so the setting alone disabled it. Overlayfs does advertise rebase outside a user namespace. - Raising downloads to 6 cut P95 by 20.5% for baseline and 15.8% for Gantry, entirely out of byte-waiting time. - The NIC stays roughly 98% idle throughout, so these pulls are concurrency-limited rather than bandwidth-limited. - Gantry nodes reach 90.2% P95 CPU while serving peers, which is why extra download concurrency helps baseline more and widens the ratio. Record run-20260806-205719-51c38730 in the three RESULTS.md tables and cross-link the analysis. --- .../gantry-benchmark/PULL-LATENCY-ANALYSIS.md | 167 ++++++++++++++++++ hack/gantry-benchmark/RESULTS.md | 17 ++ 2 files changed, 184 insertions(+) create mode 100644 hack/gantry-benchmark/PULL-LATENCY-ANALYSIS.md diff --git a/hack/gantry-benchmark/PULL-LATENCY-ANALYSIS.md b/hack/gantry-benchmark/PULL-LATENCY-ANALYSIS.md new file mode 100644 index 000000000..092bd2fdc --- /dev/null +++ b/hack/gantry-benchmark/PULL-LATENCY-ANALYSIS.md @@ -0,0 +1,167 @@ +# Pod Startup Latency Analysis + +Where the ~750-1100 seconds of per-pod startup time actually goes on a +1000-node AKS cluster pulling a 40 GiB, 40-layer image, and which containerd +settings move it. + +Source runs, both Canada Central, `Standard_D8s_v3`, Kubernetes 1.35, Azure CNI +Overlay, containerd 2.x, images pulled over ACR Private Endpoints: + +| Run | containerd `max_concurrent_downloads` | +| --- | --- | +| `run-20260806-185139-9995e167` | 3 (containerd default) | +| `run-20260806-205719-51c38730` | 6 (benchmark drop-in) | + +`max_concurrent_unpacks` was 1 (the containerd default) in both. + +Measurements come from AKS audit timestamps, containerd debug journals captured +by the node-observer DaemonSet on all 1000 nodes, and Prometheus node metrics. +Percentiles are nearest-rank, matching the benchmark runner. + +## Scheduling is not the story + +| Component | Baseline | Gantry cold | +| --- | ---: | ---: | +| Scheduling, create to bind (P50) | 2.162s | 1.836s | +| Post-bind, bind to container started (P50) | 757.726s | 921.710s | +| Total pod startup (P50) | 759.746s | 923.473s | + +Scheduling is 0.3% of startup. Effectively all wall time is the image pull, so +the rest of this document decomposes post-bind. + +## Post-bind splits into byte-waiting and unpacking + +containerd logs one `layer unpacked` event per layer and one `image unpacked` +event per image, both with durations. The image event covers the whole pull, so +the difference between it and the sum of its layer events is time containerd +spent waiting for bytes rather than unpacking them. + +Per-layer unpack cost is stable across every phase and both runs: mean 8.331s +to 8.404s over 14,000 samples per phase. Forty layers therefore cost roughly +333s of unpack work per node, and that figure does not depend on where the bytes +came from. + +| Run / phase | Image unpack (P50) | Unpack work | Waiting for bytes | +| --- | ---: | ---: | ---: | +| downloads=3, baseline | 931.1s | 333.5s (35.8%) | 597.6s (64.2%) | +| downloads=3, Gantry cold | 1098.0s | 336.2s (30.6%) | 761.8s (69.4%) | +| downloads=6, baseline | 753.3s | 333.2s (44.2%) | 420.0s (55.8%) | +| downloads=6, Gantry cold | 917.3s | 334.0s (36.4%) | 583.3s (63.6%) | + +## Raising download concurrency to 6 is a 15-20% win + +| Metric | downloads=3 | downloads=6 | Change | +| --- | ---: | ---: | ---: | +| Baseline P50 | 939.368s | 759.746s | 19.1% faster | +| Baseline P95 | 1091.173s | 867.850s | 20.5% faster | +| Gantry P50 | 1105.766s | 923.473s | 16.5% faster | +| Gantry P95 | 1180.970s | 994.468s | 15.8% faster | + +The gain comes entirely out of byte-waiting. Baseline waiting fell from 597.6s +to 420.0s while unpack work stayed at 333s. + +## Unpacking is serialized, and that floor is now dominant + +containerd's transfer plugin defaults to `max_concurrent_unpacks = 1` +(`plugins/transfer/plugin.go`), and `core/transfer/local/transfer.go` only +builds the unpack semaphore when the value exceeds 1. Without that semaphore, +`Unpacker.supportParallel` returns false at its first branch and every layer is +unpacked sequentially. + +Both runs confirm this empirically: all 2000 `image unpacked` events in each run +carry `parallel=false`, and the journals contain no "snapshotter does not +support rebase capability" message. The rebase check was never reached, so the +only thing disabling parallel unpack was the concurrency setting. + +This matters because overlayfs does advertise the `rebase` capability required +for parallel unpack whenever containerd is not running inside a user namespace +(`plugins/snapshots/overlay/plugin/plugin.go`), which is the case on AKS. The +capability is available and unused. + +As delivery gets faster the serialized floor becomes proportionally larger. On +baseline it moved from 35.8% of the pull at 3 concurrent downloads to 44.2% at +6. Raising download concurrency further without also raising unpack concurrency +has diminishing returns. + +## Nothing is resource-saturated except Gantry's CPU + +| Resource | downloads=3 baseline | downloads=6 baseline | downloads=3 Gantry | downloads=6 Gantry | +| --- | ---: | ---: | ---: | ---: | +| NIC utilization (P95) | 1.26% | 1.71% | 8.89% | 11.67% | +| CPU busy (mean) | 23.5% | 28.1% | 50.2% | 53.7% | +| CPU busy (P95) | 49.4% | 53.4% | 85.2% | 90.2% | +| Disk busy (P95) | 69.3% | 73.7% | 64.9% | 64.7% | + +The NIC is roughly 98% idle in every configuration, so these pulls are +concurrency-limited rather than bandwidth-limited. Disk is the second-most +loaded resource and is the one to watch when raising unpack concurrency, +because unpacking is write-heavy. + +Gantry CPU is the exception. Gantry nodes pull layers, unpack them, and serve +roughly 42 TB to peers, which puts them at 90.2% P95 CPU on 8 vCPU at 6 +concurrent downloads. Baseline nodes, doing only the first two, sit at 53.4%. + +That headroom difference explains why raising concurrency helped baseline more +than Gantry (20.5% versus 15.8% at P95) and why the Gantry-to-baseline P95 ratio +worsened from 1.0823 to 1.1459. Feeding more concurrent streams to a node with +no spare CPU does not help it. + +## Byte reduction is unaffected and remains the headline + +| Run | ACR bytes | Byte reduction | Pulls B/G | Peer bytes served | Fallbacks | +| --- | ---: | ---: | ---: | ---: | ---: | +| downloads=3 | 45.008 TB to 154.787 GB | 99.656% | 1000 / 2 | 42.817 TB | 0 | +| downloads=6 | 47.165 TB to 256.512 GB | 99.456% | 1000 / 4 | 42.734 TB | 0 | + +Both runs reported `FAIL` only because they were configured with a maximum +Gantry-to-baseline P95 ratio of 1.0. Every other sample in `RESULTS.md` used +3.0, which both runs pass. No pod in either run reported `ErrImagePull` or +`ImagePullBackOff`, and neither run recorded an origin fallback. + +## Conclusions + +1. Pod startup on this workload is almost entirely image pull. Scheduling and + container creation are negligible. +2. `max_concurrent_downloads = 6` is worth taking. It cut P95 by 15-20% for + both phases at negligible resource cost. +3. `max_concurrent_unpacks = 1` leaves roughly 333s of strictly serialized work + per node, now 36-44% of the pull. Overlayfs supports the `rebase` capability + needed to parallelize it, so this is the largest untested lever. +4. Gantry's constraint is node CPU, not network. At 90.2% P95 CPU it cannot + convert extra download concurrency into speed, so raising concurrency alone + widens the gap against baseline rather than closing it. +5. Gantry's value on this workload is the 99.5% reduction in registry egress + and origin pulls, not pod startup latency, which stays 15-22% above baseline. + +## Suggested next experiments + +- `max_concurrent_unpacks = 4` at `max_concurrent_downloads = 6`, changing one + variable from the run above. Watch disk busy, which is already at 73.7% P95. +- A larger node SKU or a smaller peer-serving fan-out for the Gantry phase, to + test whether Gantry latency is CPU-bound as the data suggests. +- Do not raise `max_concurrent_downloads` past 6 until unpack concurrency is + addressed, since byte-waiting is no longer the majority of baseline pull time. + +## Reproducing + +Per-layer and per-image unpack durations, with the parallel flag: + +```bash +r=/var/lib/gantry-benchmark/artifacts/ +grep -ao 'msg=\\"layer unpacked\\" duration=[0-9.]*[a-z]*' "$r/baseline-performance.json" +grep -ao 'msg=\\"image unpacked\\"[^|]\{0,400\}' "$r/baseline-performance.json" +grep -ao 'parallel=[a-z]*' "$r/baseline-performance.json" | sort | uniq -c +``` + +Audit-derived latency decomposition: + +```bash +jq -c '{startup: .baseline.azure.audit.pod_startup_latency, + scheduling: .baseline.azure.audit.scheduling_latency, + post_bind: .baseline.azure.audit.post_bind_startup_latency}' "$r/comparison.json" +``` + +Node resource utilization is under `.prometheus[] | select(.name == "") +| .response.data.result` in the phase performance artifacts, using capture names +`node_cpu_busy_ratio`, `node_disk_busy_ratio`, and +`node_network_receive_utilization_ratio`. diff --git a/hack/gantry-benchmark/RESULTS.md b/hack/gantry-benchmark/RESULTS.md index 6c37241a1..0ed3be80d 100644 --- a/hack/gantry-benchmark/RESULTS.md +++ b/hack/gantry-benchmark/RESULTS.md @@ -16,6 +16,10 @@ We removed prior benchmark images before repeat samples, measured registry traffic at each Azure Private Endpoint, measured pod startup from AKS audit logs, and measured peer traffic from Gantry metrics. +For a breakdown of where pod startup time is actually spent, and the effect of +containerd download and unpack concurrency, see +[PULL-LATENCY-ANALYSIS.md](PULL-LATENCY-ANALYSIS.md). + ### ACR traffic @@ -25,6 +29,7 @@ logs, and measured peer traffic from Gantry metrics. | 1000 nodes - sample 2 | 40 GiB | 47.566 TB | 174.781 GB | 99.633% | 1008 / 5 | 99.504% | | **1000 nodes - Canada Central ACR** | **40 GiB** | **47.178 TB** | **245.878 GB** | **99.479%** | **1000 / 2** | **99.800%** | | **1000 nodes - Canada Central ACR rerun** 3 | **40 GiB** | **45.008 TB** | **154.787 GB** | **99.656%** | **1000 / 2** | **99.800%** | +| **1000 nodes - Canada Central ACR, 6 downloads** 4 | **40 GiB** | **47.165 TB** | **256.512 GB** | **99.456%** | **1000 / 4** | **99.600%** | | **1000 nodes - UK South ACR** | **40 GiB** | **53.369 TB** | **219.262 GB** | **99.589%** | **1254 / 5** | **99.601%** | | **1000 nodes - East US ACR** | **40 GiB** | **47.562 TB** | **182.317 GB** | **99.617%** | **1004 / 4** | **99.602%** | | **1000 nodes - Central India ACR** 1 | **40 GiB** | **97.115 TB** | **803.184 GB** | **99.173%** | **2287 / 6** | **99.738%** | @@ -50,6 +55,7 @@ aggregates below. | 1000 nodes - sample 2 | 40 GiB | 894.597s | 1087.179s | 1065.724s | 1169.448s | 1652.704s | 1885.011s | 9.733% slower | | **1000 nodes - Canada Central ACR** | **40 GiB** | **1862.580s** | **774.028s** | **2030.636s** | **817.139s** | **2149.535s** | **891.766s** | **59.759% faster** | | **1000 nodes - Canada Central ACR rerun** 3 | **40 GiB** | **939.368s** | **1105.766s** | **1091.173s** | **1180.970s** | **1180.111s** | **1239.429s** | **8.229% slower** | +| **1000 nodes - Canada Central ACR, 6 downloads** 4 | **40 GiB** | **759.746s** | **923.473s** | **867.850s** | **994.468s** | **956.492s** | **1058.555s** | **14.590% slower** | | **1000 nodes - UK South ACR** | **40 GiB** | **3561.000s** | **1064.557s** | **3953.000s** | **1146.557s** | **5399.000s** | **1815.557s** | **70.995% faster** | | **1000 nodes - East US ACR** | **40 GiB** | **1401.026s** | **1065.950s** | **1655.894s** | **1144.771s** | **2351.081s** | **1831.832s** | **30.867% faster** | | **1000 nodes - Central India ACR** 2 | **40 GiB** | **3184.649s** | **1065.570s** | **4851.649s** | **1155.570s** | **5351.649s** | **1865.570s** | **76.182% faster** | @@ -79,6 +85,16 @@ phases succeeded with no image-pull backoff and no origin fallbacks. This sample also ran with the containerd transfer-service download concurrency override removed, so both phases used the containerd default of 3. +4 Run `run-20260806-205719-51c38730`, reported **FAIL**. Same cluster +as footnote 3 with the containerd transfer-service override restored to 6 +concurrent downloads, isolating that one variable. Both phases got faster: +baseline P95 improved 20.5% and Gantry P95 improved 15.8%. Baseline gained more +because Gantry nodes were already near their CPU limit at 90.2% P95 while +serving peers, so the P95 ratio worsened from 1.0823 to 1.1459 and again +exceeded the 1.0 gate this run used. All 2000 pods succeeded with no image-pull +backoff and no origin fallbacks. See +[PULL-LATENCY-ANALYSIS.md](PULL-LATENCY-ANALYSIS.md). + #### Latency excluding image-pull backoff AKS audit logs retained the pod status patches containing `ErrImagePull` and @@ -107,6 +123,7 @@ The unfiltered table remains the primary end-to-end result because | 1000 nodes - sample 2 | 40 GiB | 43.438 TB | 160.002 GB | 159 | 154 | 42,472 | 0 | | **1000 nodes - Canada Central ACR** | **40 GiB** | **42.735 TB** | **223.358 GB** | **214** | **212** | **41,791** | **0** | | **1000 nodes - Canada Central ACR rerun** 3 | **40 GiB** | **42.817 TB** | **140.672 GB** | **134** | **132** | **41,870** | **0** | +| **1000 nodes - Canada Central ACR, 6 downloads** 4 | **40 GiB** | **42.734 TB** | **233.022 GB** | **223** | **219** | **41,789** | **0** | | **1000 nodes - East US ACR** | **40 GiB** | **43.137 TB** | **161.075 GB** | **159** | **155** | **42,179** | **0** | | **1000 nodes - Central India ACR** | **40 GiB** | **43.070 TB** | **709.766 GB** | **682** | **662** | **42,129** | **0** | | 2000 nodes - sample 1 | 40 GiB | 86.971 TB | 221.210 GB | 213 | 210 | 85,044 | 0 | From 9af85cd491c0ad2b39a31887cada5ff320176db0 Mon Sep 17 00:00:00 2001 From: Vaibhav Patel Date: Thu, 6 Aug 2026 18:46:03 -0400 Subject: [PATCH 38/60] docs(gantry-benchmark): attribute the Gantry gap to cold start Add the peer-fetch outcome breakdown and the layer-byte delivery timeline, and correct the earlier attribution. The previous revision blamed the Gantry latency gap on node CPU at 90.2% P95. The delivery timeline contradicts that: once warm, Gantry peaks near 350 MB/s per node against about 182 MB/s for baseline, so it is not CPU-starved for throughput. The gap is the cold start. Layer delivery needs four minutes to reach full rate because at the start of a cold run almost no node holds the image, while ACR is at capacity immediately. That ramp costs about 2.6 minutes, more than the 1.7 minute penalty actually observed, because Gantry's steady-state throughput recovers part of it. Two findings that look alarming are ruled out as costs. The 1.16M HTTP 429s are a startup transient, 99.6% inside the first six minutes at 1.5ms each. The 31,339 60s stalls preserve the delivered prefix through resume-from-offset and occur while delivery runs at peak rate. Also note that PeerFetchTimeout is a total request deadline rather than a no-progress one, so the rate a stream must sustain to survive it scales with layer size. That did not bind these 1 GiB layers but does not scale to larger ones. --- .../gantry-benchmark/PULL-LATENCY-ANALYSIS.md | 98 +++++++++++++++++-- hack/gantry-benchmark/RESULTS.md | 11 ++- 2 files changed, 94 insertions(+), 15 deletions(-) diff --git a/hack/gantry-benchmark/PULL-LATENCY-ANALYSIS.md b/hack/gantry-benchmark/PULL-LATENCY-ANALYSIS.md index 092bd2fdc..12a17a130 100644 --- a/hack/gantry-benchmark/PULL-LATENCY-ANALYSIS.md +++ b/hack/gantry-benchmark/PULL-LATENCY-ANALYSIS.md @@ -100,11 +100,73 @@ because unpacking is write-heavy. Gantry CPU is the exception. Gantry nodes pull layers, unpack them, and serve roughly 42 TB to peers, which puts them at 90.2% P95 CPU on 8 vCPU at 6 concurrent downloads. Baseline nodes, doing only the first two, sit at 53.4%. +That is worth watching as a headroom limit, but the delivery timeline below +shows it is not what makes the Gantry phase slower. -That headroom difference explains why raising concurrency helped baseline more -than Gantry (20.5% versus 15.8% at P95) and why the Gantry-to-baseline P95 ratio -worsened from 1.0823 to 1.1459. Feeding more concurrent streams to a node with -no spare CPU does not help it. +## The Gantry phase is slower because of its cold start, not contention + +Peer fetch outcomes over the whole Gantry-cold phase, summed across 1000 nodes: + +| Outcome | Count | Mean duration | +| --- | ---: | ---: | +| busy (HTTP 429) | 1,162,435 | 0.0015s | +| hit | 41,789 | n/a | +| stall | 31,339 | 60.0007s | +| notfound | 10,164 | n/a | +| unavailable | 4,668 | n/a | +| digest_mismatch, auth, protocol, server, local error | 0 | n/a | + +Only 3.3% of attempts succeed, and there are 27.8 rejections per success. Those +headline numbers invite two wrong conclusions, so both are worth ruling out. + +First, the rejections are almost entirely a startup transient. 99.6% of them +occur in the first six minutes and 56% in the second minute alone, falling to +zero by minute eleven. At 1.5ms each they cost about 1.7 seconds per node in +total. + +Second, the stalls are not lost work. The 60.0007s mean is `PeerFetchTimeout` +firing, but `livePeerStream` streams through to the containerd-facing response +and records the verified byte offset, and re-selection resumes from that offset. +A stall costs a DHT lookup and a redial, not the delivered prefix. Stalls also +hold steady at roughly 3,700 per minute from minute four to minute ten, which is +exactly when delivery runs at peak rate. + +What does explain the gap is the rate at which layer bytes reach nodes: + +| Minute | Layer GB served, all nodes | MB/s per node | Cumulative | +| ---: | ---: | ---: | ---: | +| 0 | 0 | 0.0 | 0.0% | +| 1 | 936 | 15.6 | 2.2% | +| 2 | 2,220 | 37.0 | 7.3% | +| 3 | 4,306 | 71.8 | 17.4% | +| 4 | 4,953 | 82.6 | 28.9% | +| 5 to 9 | about 5,100 each | about 85 | 88.1% | +| 10 | 4,617 | 77.0 | 98.9% | +| 11 | 487 | 8.1 | 100.0% | + +Baseline reaches its full network rate inside the first minute because ACR is +already at capacity. Gantry needs four minutes, because at the start of a cold +run almost no node holds the image and there is nothing to serve. Supply has to +be built before it can be consumed, and the 429 storm is the visible signature +of that shortage rather than a cost in its own right. + +Once the swarm is warm, Gantry is the faster of the two. Node network receive +peaks near 350 MB/s during the Gantry phase against about 182 MB/s for baseline. + +At the observed steady rate of about 5,087 GB per minute, all 42.95 TB would +move in 8.4 minutes. It took roughly 11, so the cascade ramp costs about 2.6 +minutes. Byte delivery finishes at minute 11 while the phase runs to minute 18; +that closing stretch is the serialized unpack draining, which matches the 334s +of measured unpack work per node. + +| Component | Time | Attribution | +| --- | ---: | --- | +| Cascade cold-start ramp | about 2.6 min | Gantry only | +| Steady-state delivery | about 8.4 min | Gantry faster than baseline | +| Serialized unpack tail | about 5.6 min | both phases | + +Gantry's total penalty against baseline in this run was 1.7 minutes, less than +the ramp alone, because its steady-state throughput recovers part of the deficit. ## Byte reduction is unaffected and remains the headline @@ -127,18 +189,28 @@ Gantry-to-baseline P95 ratio of 1.0. Every other sample in `RESULTS.md` used 3. `max_concurrent_unpacks = 1` leaves roughly 333s of strictly serialized work per node, now 36-44% of the pull. Overlayfs supports the `rebase` capability needed to parallelize it, so this is the largest untested lever. -4. Gantry's constraint is node CPU, not network. At 90.2% P95 CPU it cannot - convert extra download concurrency into speed, so raising concurrency alone - widens the gap against baseline rather than closing it. -5. Gantry's value on this workload is the 99.5% reduction in registry egress +4. The Gantry phase is slower than baseline because of its cold start. Delivery + takes four minutes to reach full rate while baseline is there in one, costing + about 2.6 minutes. Neither the 429 storm nor the 60s stalls are the cost: + the first is a startup transient at 1.5ms each, and the second preserves the + delivered prefix and occurs while delivery is at peak rate. +5. Once warm, Gantry delivers faster than pulling from the registry, peaking + near 350 MB/s per node against about 182 MB/s for baseline. +6. `PeerFetchTimeout` is a total request deadline rather than a no-progress + deadline, so the throughput a stream must sustain to survive it scales with + layer size: 17.9 MB/s for a 1 GiB layer, 716 MB/s for a 40 GiB one. This did + not dominate these runs, but it does not scale to larger layers. containerd's + own `image_pull_progress_timeout` uses no-progress semantics by contrast. +7. Gantry's value on this workload is the 99.5% reduction in registry egress and origin pulls, not pod startup latency, which stays 15-22% above baseline. ## Suggested next experiments - `max_concurrent_unpacks = 4` at `max_concurrent_downloads = 6`, changing one variable from the run above. Watch disk busy, which is already at 73.7% P95. -- A larger node SKU or a smaller peer-serving fan-out for the Gantry phase, to - test whether Gantry latency is CPU-bound as the data suggests. +- Anything that shortens the cascade ramp, since that is the whole Gantry + penalty. Seeding more than the observed 223 origin pulls before the fan-out + begins is the obvious direction to test. - Do not raise `max_concurrent_downloads` past 6 until unpack concurrency is addressed, since byte-waiting is no longer the majority of baseline pull time. @@ -165,3 +237,9 @@ Node resource utilization is under `.prometheus[] | select(.name == "") | .response.data.result` in the phase performance artifacts, using capture names `node_cpu_busy_ratio`, `node_disk_busy_ratio`, and `node_network_receive_utilization_ratio`. + +Peer fetch outcomes, DHT results, and layer bytes served use the same path with +capture names `gantry_peer_outcomes`, `gantry_peer_duration`, +`gantry_dht_outcomes`, `gantry_dht_duration`, and `gantry_mirror_bytes`. These +are counters, so a phase total is the last sample minus the first, summed across +pods; bin those per-sample increases by minute to recover the timelines above. diff --git a/hack/gantry-benchmark/RESULTS.md b/hack/gantry-benchmark/RESULTS.md index 0ed3be80d..e715fd95b 100644 --- a/hack/gantry-benchmark/RESULTS.md +++ b/hack/gantry-benchmark/RESULTS.md @@ -88,11 +88,12 @@ override removed, so both phases used the containerd default of 3. 4 Run `run-20260806-205719-51c38730`, reported **FAIL**. Same cluster as footnote 3 with the containerd transfer-service override restored to 6 concurrent downloads, isolating that one variable. Both phases got faster: -baseline P95 improved 20.5% and Gantry P95 improved 15.8%. Baseline gained more -because Gantry nodes were already near their CPU limit at 90.2% P95 while -serving peers, so the P95 ratio worsened from 1.0823 to 1.1459 and again -exceeded the 1.0 gate this run used. All 2000 pods succeeded with no image-pull -backoff and no origin fallbacks. See +baseline P95 improved 20.5% and Gantry P95 improved 15.8%. The remaining gap is +Gantry's cold start: layer delivery takes four minutes to reach full rate while +baseline is at full rate within one, costing about 2.6 minutes. Once warm, +Gantry delivers faster than the registry. The P95 ratio was 1.1459 and exceeded +the 1.0 gate this run used. All 2000 pods succeeded with no image-pull backoff +and no origin fallbacks. See [PULL-LATENCY-ANALYSIS.md](PULL-LATENCY-ANALYSIS.md). #### Latency excluding image-pull backoff From 011a9716b5c5d39329196c049c57a3a03cc32d6f Mon Sep 17 00:00:00 2001 From: Vaibhav Patel Date: Thu, 6 Aug 2026 18:57:17 -0400 Subject: [PATCH 39/60] docs(gantry-benchmark): explain the ramp as uniform layer ordering Record why the cascade needs four minutes to reach full rate. containerd walks the manifest in order, so all 1000 nodes request layer positions in the same sequence and the swarm seeds one position at a time instead of all 40 at once. Per-layer unpack timestamps show strict waves: each layer's first completion trails the previous by about 7 seconds, matching the serialized unpack cost. Extrapolated across 40 layers that puts the first seed for the final layer near 280 seconds, the same scale as the observed ramp. Until a node works through the preceding layers, the later positions have no seeder anywhere and demand for them cannot be served. Baseline is the control: same manifest order, same 7 second stagger, no ramp, because ACR already holds every layer. Ordering costs nothing when supply exists and only bites when supply must be built. This also re-explains the 429 storm as concentrated demand on a narrow wavefront rather than diffuse contention. Note the measurement limit: the journal capture holds 7 distinct layers per phase, so the ordering and the 7 second step are measured while the 40 layer figure is arithmetic. --- .../gantry-benchmark/PULL-LATENCY-ANALYSIS.md | 72 +++++++++++++++++-- 1 file changed, 66 insertions(+), 6 deletions(-) diff --git a/hack/gantry-benchmark/PULL-LATENCY-ANALYSIS.md b/hack/gantry-benchmark/PULL-LATENCY-ANALYSIS.md index 12a17a130..774eb94bb 100644 --- a/hack/gantry-benchmark/PULL-LATENCY-ANALYSIS.md +++ b/hack/gantry-benchmark/PULL-LATENCY-ANALYSIS.md @@ -168,6 +168,52 @@ of measured unpack work per node. Gantry's total penalty against baseline in this run was 1.7 minutes, less than the ramp alone, because its steady-state throughput recovers part of the deficit. +### The ramp exists because every node pulls layers in the same order + +containerd walks the manifest in order, so all 1000 nodes request layer +positions in the same sequence. The containerd journal records each +`layer unpacked` event with its digest, and the completions fall into strict +waves. Seconds are measured from the first event in the phase. + +| Gantry-cold layer | events | first | median | last | +| --- | ---: | ---: | ---: | ---: | +| `6a98e6e4b146` | 2000 | 0 | 114 | 232 | +| `b407264bc620` | 2000 | 8 | 121 | 243 | +| `06e50d442e52` | 2000 | 15 | 129 | 255 | +| `8d171b1dbb8e` | 2000 | 21 | 138 | 266 | +| `c3e575abbc22` | 2000 | 28 | 147 | 275 | +| `5843544c2584` | 2000 | 36 | 155 | 285 | +| `1af94d827f87` | 2000 | 43 | 163 | 297 | + +Each layer's first completion trails the previous one by roughly 7 seconds, and +the medians advance in the same order. Independent per-node layer selection +would instead produce overlapping distributions starting near zero. The 7 second +step also matches the serialized unpack cost, so the wavefront advances at about +the speed of one layer. + +Extrapolating that step across 40 layers puts the first seed for the final layer +near 280 seconds, which is the same scale as the observed four minute ramp. +Until some node has worked through the preceding layers, the later positions +have no seeder anywhere in the swarm, so demand for them cannot be served at any +price. + +Baseline is the control. It shows the same stagger, with first completions at 0, +6, 13, 22, 29, 36 and 44 seconds, because it uses the same manifest order. It +has no ramp at all, because ACR already holds every layer at t=0. Uniform +ordering is therefore harmless when supply exists and expensive only when supply +has to be built. + +This also re-explains the 429 storm. The rejections are not diffuse contention: +1000 nodes want the same layer at the same moment, which saturates whichever few +nodes hold it. Concentrated demand on a narrow wavefront is the cause, and the +serve cap is what makes it visible. + +The journal capture is truncated to 7 distinct layers per phase, and the 2000 +events per layer indicate some line duplication in the capture, so the ordering +and the 7 second step are measured while the 40 layer figure is arithmetic. +Capturing the full journal, or timestamping per-digest mirror advertisements, +would close that gap. + ## Byte reduction is unaffected and remains the headline | Run | ACR bytes | Byte reduction | Pulls B/G | Peer bytes served | Fallbacks | @@ -194,23 +240,30 @@ Gantry-to-baseline P95 ratio of 1.0. Every other sample in `RESULTS.md` used about 2.6 minutes. Neither the 429 storm nor the 60s stalls are the cost: the first is a startup transient at 1.5ms each, and the second preserves the delivered prefix and occurs while delivery is at peak rate. -5. Once warm, Gantry delivers faster than pulling from the registry, peaking +5. The ramp exists because every node walks the manifest in the same order, so + the swarm seeds one layer position at a time instead of all 40 at once. Layer + completions arrive in strict waves about 7 seconds apart. Baseline shows the + same ordering and no ramp, which isolates ordering as costly only when supply + must be built rather than already existing at the origin. +6. Once warm, Gantry delivers faster than pulling from the registry, peaking near 350 MB/s per node against about 182 MB/s for baseline. -6. `PeerFetchTimeout` is a total request deadline rather than a no-progress +7. `PeerFetchTimeout` is a total request deadline rather than a no-progress deadline, so the throughput a stream must sustain to survive it scales with layer size: 17.9 MB/s for a 1 GiB layer, 716 MB/s for a 40 GiB one. This did not dominate these runs, but it does not scale to larger layers. containerd's own `image_pull_progress_timeout` uses no-progress semantics by contrast. -7. Gantry's value on this workload is the 99.5% reduction in registry egress +8. Gantry's value on this workload is the 99.5% reduction in registry egress and origin pulls, not pod startup latency, which stays 15-22% above baseline. ## Suggested next experiments - `max_concurrent_unpacks = 4` at `max_concurrent_downloads = 6`, changing one variable from the run above. Watch disk busy, which is already at 73.7% P95. -- Anything that shortens the cascade ramp, since that is the whole Gantry - penalty. Seeding more than the observed 223 origin pulls before the fan-out - begins is the obvious direction to test. +- Desynchronize layer acquisition order across nodes so the swarm seeds every + layer position at once instead of advancing a single wavefront. With 1000 + nodes and 40 layers, starting nodes at staggered offsets would put a seed on + every position within roughly one layer-time rather than 40. This targets the + ramp, which is the entire Gantry penalty. - Do not raise `max_concurrent_downloads` past 6 until unpack concurrency is addressed, since byte-waiting is no longer the majority of baseline pull time. @@ -225,6 +278,13 @@ grep -ao 'msg=\\"image unpacked\\"[^|]\{0,400\}' "$r/baseline-performance.json" grep -ao 'parallel=[a-z]*' "$r/baseline-performance.json" | sort | uniq -c ``` +Per-layer completion order, which shows the wavefront: + +```bash +grep -ao 'time=\\"[0-9T:.Z-]*\\" level=debug msg=\\"layer unpacked\\" duration=[0-9.a-z]* layer=\\"sha256:[0-9a-f]*' \ + "$r/gantry_cold-performance.json" +``` + Audit-derived latency decomposition: ```bash From 647eb4e3cc61a91467339a70535e02f56816c021 Mon Sep 17 00:00:00 2001 From: Vaibhav Patel Date: Thu, 6 Aug 2026 19:39:16 -0400 Subject: [PATCH 40/60] fix(gantry): run cold-start prefetch under live stream-through Cold-start seeding never executed in any production deployment. The peer-served manifest path gated the prefetch hook behind !liveStreamThrough, and cmd/gantry always enables live stream-through, so the branch was unreachable in practice. Measured on a 1000-node cluster: p2p_prefetch_batches_total and p2p_prefetch_pullers_per_manifest_count were 0 on every node across two full benchmark runs, with no prefetch debug lines despite debug logging emitting 2808 other lines per pod. Manifests were served exclusively from peers (7.4 MB), with the cache and origin manifest paths at zero, so the guarded branch was the only one that could have fired. The orchestrator itself was wired correctly and logged "cold-start orchestrator wired". The consequence is that prefetch_puller_fraction never selected any pullers. Instead of 40 layers seeded 20 ways, the swarm bootstrapped from 219 incidental origin fallbacks, which is the likely cause of the four minute delivery ramp that accounts for Gantry's entire latency gap against baseline. The guard was defensible in isolation: under live stream-through the mirror does not write the body into its own cache, so the prefetcher's manifest read would miss. But the store is the containerd content store in production, so the manifest does arrive there once containerd commits it. Fire the hook in both modes and let the manifest read retry with backoff instead of dropping the prefetch. Also collect the seeding metrics in the benchmark, which filtered out every coord and prefetch series and hid this. --- cmd/gantry/main.go | 39 +++++- cmd/gantry/prefetch_manifest_test.go | 118 ++++++++++++++++++ .../gantry-benchmark/performance_telemetry.go | 4 + .../manifests/monitoring.yaml.tmpl | 2 +- internal/gantry/mirror/mirror.go | 8 +- .../gantry/mirror/mirror_prefetch_test.go | 49 ++++++++ 6 files changed, 215 insertions(+), 5 deletions(-) create mode 100644 cmd/gantry/prefetch_manifest_test.go diff --git a/cmd/gantry/main.go b/cmd/gantry/main.go index 8b028afde..ad59402f6 100644 --- a/cmd/gantry/main.go +++ b/cmd/gantry/main.go @@ -1933,6 +1933,43 @@ func newLayerPrefetcher(r *coldstart.Resolver, cache ifaces.LocalContentStore, l } } +// openManifest reads the manifest body from the shared content store. Under +// live stream-through the mirror proxies the body straight to containerd, so +// it only appears once containerd commits it; retry briefly rather than +// dropping the prefetch and losing cold-start seeding entirely. +func (p *layerPrefetchAdapter) openManifest(ctx context.Context, d digest.Digest) (io.ReadCloser, error) { + const attempts = 8 + + delay := 100 * time.Millisecond + + var lastErr error + + for attempt := range attempts { + rc, _, err := p.cache.Open(ctx, d) + if err == nil { + return rc, nil + } + + lastErr = err + + if attempt == attempts-1 { + break + } + + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(delay): + } + + if delay < time.Second { + delay *= 2 + } + } + + return nil, lastErr +} + func (p *layerPrefetchAdapter) OnManifestServed(ctx context.Context, registry, repository string, manifestDigest digest.Digest) { if p.resolver == nil { return @@ -1943,7 +1980,7 @@ func (p *layerPrefetchAdapter) OnManifestServed(ctx context.Context, registry, r ctx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() - rc, _, err := p.cache.Open(ctx, manifestDigest) + rc, err := p.openManifest(ctx, manifestDigest) if err != nil { p.logger.Debug("prefetch: manifest not in cache", slog.String("digest", manifestDigest.String()), diff --git a/cmd/gantry/prefetch_manifest_test.go b/cmd/gantry/prefetch_manifest_test.go new file mode 100644 index 000000000..00ca033c1 --- /dev/null +++ b/cmd/gantry/prefetch_manifest_test.go @@ -0,0 +1,118 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package main + +import ( + "context" + "errors" + "io" + "log/slog" + "strings" + "sync" + "testing" + "time" + + "github.com/Azure/unbounded/internal/gantry/digest" + "github.com/Azure/unbounded/internal/gantry/ifaces" +) + +// delayedManifestStore returns ErrNotFound until availableAfter opens, which +// models containerd committing the streamed manifest a moment after the mirror +// finishes serving it. +type delayedManifestStore struct { + mu sync.Mutex + opens int + body string + ready bool +} + +func (s *delayedManifestStore) markReady() { + s.mu.Lock() + defer s.mu.Unlock() + + s.ready = true +} + +func (s *delayedManifestStore) Open(_ context.Context, _ digest.Digest) (io.ReadCloser, int64, error) { + s.mu.Lock() + defer s.mu.Unlock() + + s.opens++ + + if !s.ready { + return nil, 0, errors.New("content not found") + } + + return io.NopCloser(strings.NewReader(s.body)), int64(len(s.body)), nil +} + +func (s *delayedManifestStore) openCount() int { + s.mu.Lock() + defer s.mu.Unlock() + + return s.opens +} + +func (s *delayedManifestStore) Has(context.Context, digest.Digest) (bool, error) { + return false, nil +} + +func (s *delayedManifestStore) Writer(context.Context, digest.Digest) (ifaces.ContentWriter, error) { + return nil, errors.New("not implemented") +} + +func (s *delayedManifestStore) Delete(context.Context, digest.Digest) error { + return errors.New("not implemented") +} + +func testDigest(t *testing.T, fill string) digest.Digest { + t.Helper() + + d, err := digest.Parse("sha256:" + strings.Repeat(fill, 64)) + if err != nil { + t.Fatalf("digest.Parse: %v", err) + } + + return d +} + +func TestOpenManifestRetriesUntilContainerdCommits(t *testing.T) { + store := &delayedManifestStore{body: `{"schemaVersion":2}`} + adapter := &layerPrefetchAdapter{cache: store, logger: slog.Default()} + + go func() { + time.Sleep(250 * time.Millisecond) + store.markReady() + }() + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + rc, err := adapter.openManifest(ctx, testDigest(t, "a")) + if err != nil { + t.Fatalf("openManifest: %v", err) + } + + defer func() { _ = rc.Close() }() //nolint:errcheck // best-effort close + + if store.openCount() < 2 { + t.Fatalf("open attempts = %d, want the retry path exercised", store.openCount()) + } +} + +func TestOpenManifestGivesUpWhenNeverCommitted(t *testing.T) { + store := &delayedManifestStore{body: "unused"} + adapter := &layerPrefetchAdapter{cache: store, logger: slog.Default()} + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + + if _, err := adapter.openManifest(ctx, testDigest(t, "b")); err == nil { + t.Fatal("openManifest succeeded, want failure when the manifest never lands") + } + + if store.openCount() < 2 { + t.Fatalf("open attempts = %d, want more than one before giving up", store.openCount()) + } +} diff --git a/hack/cmd/gantry-benchmark/performance_telemetry.go b/hack/cmd/gantry-benchmark/performance_telemetry.go index 253bc549b..8793f6112 100644 --- a/hack/cmd/gantry-benchmark/performance_telemetry.go +++ b/hack/cmd/gantry-benchmark/performance_telemetry.go @@ -81,6 +81,10 @@ func performanceTelemetryQueries() []performanceTelemetryQuery { {name: "gantry_mirror_bytes", query: `gantry_mirror_bytes_served_total{gantry_benchmark="true"}`}, {name: "gantry_response_completed", query: `gantry_mirror_response_completed_timestamp_seconds{kind="layer",gantry_benchmark="true"}`}, {name: "gantry_commit_observation", query: `{__name__=~"gantry_containerd_commit_(observed_total|observed_timestamp_seconds|observation_duration_seconds_(sum|count)|latest_observation_duration_seconds|missing_after_stream_total)",gantry_benchmark="true"}`}, + // Seeding width: how many HRW pullers each layer actually activates, and + // how many please_pull requests convert into origin pulls versus dedup. + {name: "gantry_coord_seeding", query: `{__name__=~"p2p_coord_please_pull_(served|started|declined)_total|p2p_coord_pull_intent_served_total|p2p_prefetch_(batches|digests)_total",gantry_benchmark="true"}`}, + {name: "gantry_prefetch_pullers", query: `{__name__=~"p2p_prefetch_pullers_per_manifest_(bucket|sum|count)",gantry_benchmark="true"}`}, } } diff --git a/hack/gantry-benchmark/manifests/monitoring.yaml.tmpl b/hack/gantry-benchmark/manifests/monitoring.yaml.tmpl index 60c3b09ad..45b824708 100644 --- a/hack/gantry-benchmark/manifests/monitoring.yaml.tmpl +++ b/hack/gantry-benchmark/manifests/monitoring.yaml.tmpl @@ -29,7 +29,7 @@ spec: - action: keep sourceLabels: - __name__ - regex: gantry_storage_mode_info|p2p_dht_health_score|p2p_dht_lookup_total|p2p_dht_lookup_duration_seconds_(bucket|sum|count)|gantry_peer_serve_bytes_total|gantry_origin_bytes_total|p2p_origin_pull_total|p2p_peer_fetch_total|gantry_peer_fetch_last_timestamp_seconds|p2p_peer_fetch_duration_seconds_(bucket|sum|count)|p2p_origin_pull_success_total|p2p_origin_fallback_total|p2p_in_flight_pulls|gantry_peer_fetch_bytes_total|gantry_mirror_bytes_served_total|gantry_mirror_response_completed_timestamp_seconds|gantry_containerd_commit_observed_total|gantry_containerd_commit_observed_timestamp_seconds|gantry_containerd_commit_observation_duration_seconds_(bucket|sum|count)|gantry_containerd_commit_latest_observation_duration_seconds|gantry_containerd_commit_missing_after_stream_total + regex: gantry_storage_mode_info|p2p_dht_health_score|p2p_dht_lookup_total|p2p_dht_lookup_duration_seconds_(bucket|sum|count)|gantry_peer_serve_bytes_total|gantry_origin_bytes_total|p2p_origin_pull_total|p2p_peer_fetch_total|gantry_peer_fetch_last_timestamp_seconds|p2p_peer_fetch_duration_seconds_(bucket|sum|count)|p2p_origin_pull_success_total|p2p_origin_fallback_total|p2p_in_flight_pulls|gantry_peer_fetch_bytes_total|gantry_mirror_bytes_served_total|gantry_mirror_response_completed_timestamp_seconds|gantry_containerd_commit_observed_total|gantry_containerd_commit_observed_timestamp_seconds|gantry_containerd_commit_observation_duration_seconds_(bucket|sum|count)|gantry_containerd_commit_latest_observation_duration_seconds|gantry_containerd_commit_missing_after_stream_total|p2p_coord_please_pull_(served|started|declined)_total|p2p_coord_pull_intent_served_total|p2p_prefetch_(batches|digests)_total|p2p_prefetch_pullers_per_manifest_(bucket|sum|count) - action: replace targetLabel: gantry_benchmark replacement: "true" diff --git a/internal/gantry/mirror/mirror.go b/internal/gantry/mirror/mirror.go index fe790c914..90442d264 100644 --- a/internal/gantry/mirror/mirror.go +++ b/internal/gantry/mirror/mirror.go @@ -875,9 +875,11 @@ func (s *Server) serveDigest(w http.ResponseWriter, r *http.Request, upstream, r case peerFallbackLocalHit: return case peerFallbackServed: - if !s.liveStreamThrough { - s.firePrefetch(ctx, kind, upstream, repo, d) - } + // Live stream-through proxies the body straight to containerd, so a + // served manifest reaches the shared content store on containerd's + // commit rather than ours. The prefetcher waits for it there, so this + // must fire in both modes or cold-start seeding never runs. + s.firePrefetch(ctx, kind, upstream, repo, d) return case peerFallbackPartial: diff --git a/internal/gantry/mirror/mirror_prefetch_test.go b/internal/gantry/mirror/mirror_prefetch_test.go index b4352ef20..aa0eff4fb 100644 --- a/internal/gantry/mirror/mirror_prefetch_test.go +++ b/internal/gantry/mirror/mirror_prefetch_test.go @@ -19,6 +19,7 @@ import ( "github.com/Azure/unbounded/internal/gantry/config" "github.com/Azure/unbounded/internal/gantry/digest" + "github.com/Azure/unbounded/internal/gantry/ifaces" "github.com/Azure/unbounded/internal/gantry/ifaces/fakes" "github.com/Azure/unbounded/internal/gantry/mirror" "github.com/Azure/unbounded/internal/gantry/origin" @@ -373,3 +374,51 @@ func TestMirror_Prefetch_DoesNotFireOnHeadCacheHit(t *testing.T) { t.Fatalf("HEAD cache-hit: got %d prefetch calls, want 1 (HEAD must not trigger prefetch)", n) } } + +// Production wiring always enables live stream-through. Prefetch on the +// peer-served manifest path was previously gated behind !liveStreamThrough, +// which silently disabled cold-start seeding for every real deployment. +func TestMirror_Prefetch_FiresOnPeerServedManifestWithLiveStreamThrough(t *testing.T) { + body := []byte(`{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json"}`) + d := digestOf(body) + + dialer := newCountingPeerDialer() + dialer.Put("10.0.0.1:5001", d, body) + + dht := fakes.NewDHT() + dht.Inject(d, ifaces.Provider{NodeID: "peer-a", Addr: "10.0.0.1:5001"}) + + cfg, originSrc := newMirrorOriginNotFound(t) + spy := newPrefetchSpy() + + m := mirror.New(cfg, &writerSpyCache{}, originSrc, + mirror.WithLiveStreamThrough(), + mirror.WithDiscovery(dht, dialer), + mirror.WithPeerBudgets(time.Second, time.Second, 2), + mirror.WithLayerPrefetcher(spy), + ) + ts := httptest.NewServer(m.Handler()) + + t.Cleanup(ts.Close) + + resp, err := http.Get(ts.URL + "/v2/library/nginx/manifests/" + d.String() + "?ns=reg.example.com") + if err != nil { + t.Fatal(err) + } + + got, _ := io.ReadAll(resp.Body) //nolint:errcheck // body compared below + _ = resp.Body.Close() //nolint:errcheck // best-effort close + + if resp.StatusCode != http.StatusOK || string(got) != string(body) { + t.Fatalf("peer manifest serve: status=%d body=%q", resp.StatusCode, got) + } + + if n := spy.waitForCount(1, 2*time.Second); n != 1 { + t.Fatalf("OnManifestServed calls after peer manifest serve = %d, want 1", n) + } + + call := spy.snapshot()[0] + if call.registry != "reg.example.com" || call.repository != "library/nginx" || call.digest.String() != d.String() { + t.Fatalf("prefetch call = %+v", call) + } +} From 8ac605bb3cd619edd0b9d3465cdfd0191a53c684 Mon Sep 17 00:00:00 2001 From: Vaibhav Patel Date: Thu, 6 Aug 2026 21:03:39 -0400 Subject: [PATCH 41/60] fix(gantry): resolve coordination peers to pod addresses Cold-start prefetch now fires under live stream-through, but 54% of its 549,366 direct please_pull groups failed before reaching their target. The coordinator resolved Kubernetes node names to libp2p peer IDs, then called NewStream without installing the target's membership-announced Pod-IP multiaddrs. DHT bootstrap intentionally stops after a routing table of five peers, so most targets were absent from the peerstore and libp2p fell back to stale loopback addresses: no good addresses /ip4/127.0.0.1/tcp/4001: dial to self attempted The target Pod annotations were correct. For a representative failure, the peer ID matched gantry-76gss and its annotation advertised 10.66.81.213, while the caller tried only 127.0.0.1. When membership resolves a node, parse its full /p2p multiaddrs, verify they carry the same peer ID as the peer-id annotation, and install the transport addresses into the host peerstore before NewStream runs. This keeps direct coord RPCs independent of which five peers happened to populate the DHT routing table first. Add p2p_prefetch_groups_total{target,outcome} so future runs measure RPC dispatch success without scraping 1000 pod logs. Zero-initialize all four label combinations for benchmark coverage validation, and widen the pullers-per-manifest histogram buckets for observed 500+ node fan-out. --- cmd/gantry/agent_metrics.go | 15 ++- cmd/gantry/main.go | 51 +++++++-- cmd/gantry/membership_peer_resolver_test.go | 106 ++++++++++++++++++ .../gantry-benchmark/performance_telemetry.go | 2 +- .../manifests/monitoring.yaml.tmpl | 2 +- internal/gantry/coldstart/coldstart.go | 2 + internal/gantry/coldstart/coldstart_test.go | 3 +- internal/gantry/coldstart/prefetch.go | 14 +++ internal/gantry/coldstart/prefetch_test.go | 61 +++++++++- 9 files changed, 243 insertions(+), 13 deletions(-) create mode 100644 cmd/gantry/membership_peer_resolver_test.go diff --git a/cmd/gantry/agent_metrics.go b/cmd/gantry/agent_metrics.go index 980b1f7c5..44f78668c 100644 --- a/cmd/gantry/agent_metrics.go +++ b/cmd/gantry/agent_metrics.go @@ -230,6 +230,7 @@ type phase3Metrics struct { prefetchBatchesTotal prometheus.Counter prefetchDigestsTotal prometheus.Counter prefetchPullersPerBatch prometheus.Histogram + prefetchGroupsTotal *prometheus.CounterVec } func newPhase3Metrics(reg *metrics.Registry, infl *inflight.Map) *phase3Metrics { @@ -240,6 +241,17 @@ func newPhase3Metrics(reg *metrics.Registry, infl *inflight.Map) *phase3Metrics Help: "Current count of in-flight digest pulls on this node.", }, func() float64 { return float64(infl.Len()) }) + prefetchGroupsTotal := reg.NewCounterVec("coord", prometheus.CounterOpts{ + Name: "p2p_prefetch_groups_total", + Help: "Prefetch dispatch groups by local or remote target and success or error outcome.", + }, []string{"target", "outcome"}) + + for _, target := range []string{"local", "remote"} { + for _, outcome := range []string{"success", "error"} { + prefetchGroupsTotal.WithLabelValues(target, outcome).Add(0) + } + } + return &phase3Metrics{ hrwRankMismatch: reg.NewCounterVec("coord", prometheus.CounterOpts{ Name: "p2p_hrw_rank_mismatch_total", @@ -297,8 +309,9 @@ func newPhase3Metrics(reg *metrics.Registry, infl *inflight.Map) *phase3Metrics prefetchPullersPerBatch: reg.NewHistogram("coord", prometheus.HistogramOpts{ Name: "p2p_prefetch_pullers_per_manifest", Help: "Distribution of distinct HRW rank-0 pullers contacted per manifest pre-fan call.", - Buckets: prometheus.LinearBuckets(1, 1, 10), + Buckets: prometheus.ExponentialBuckets(1, 2, 11), }), + prefetchGroupsTotal: prefetchGroupsTotal, } } diff --git a/cmd/gantry/main.go b/cmd/gantry/main.go index ad59402f6..c886b9b04 100644 --- a/cmd/gantry/main.go +++ b/cmd/gantry/main.go @@ -28,6 +28,7 @@ import ( "time" "github.com/libp2p/go-libp2p/core/peer" + "github.com/libp2p/go-libp2p/core/peerstore" "github.com/multiformats/go-multiaddr" "github.com/Azure/unbounded/internal/gantry/advertise" @@ -384,7 +385,7 @@ func runAgent(args []string) error { // annotation (the design doc) which Members reads in Snapshot. This // lets the cluster use stable K8s node names as NodeIDs // while still dialing libp2p RPCs to the right peer. - coord.WithPeerIDResolver(membershipPeerIDResolver(memberView, logger)), + coord.WithPeerIDResolver(membershipPeerIDResolver(memberView, disco.LibP2P().Peerstore(), logger)), ) // pullerPump bridges inbound please_pull RPCs to the local origin // puller (the step 7). The pump itself MUST NOT block the coord @@ -488,6 +489,9 @@ func runAgent(args []string) error { p3.prefetchDigestsTotal.Add(float64(digests)) p3.prefetchPullersPerBatch.Observe(float64(pullers)) }, + OnPrefetchGroup: func(target, outcome string) { + p3.prefetchGroupsTotal.WithLabelValues(target, outcome).Inc() + }, }, }) coldStartResolver = coldStartAdapter{r: realResolver} @@ -1288,13 +1292,9 @@ func transferPortFromListen(listen string) int { return n } -// membershipPeerIDResolver returns a coord.WithPeerIDResolver callback -// that consults the live members snapshot. NodeID -> Node.PeerID is the -// fast path; on miss the resolver returns (_, false) so coord.Client -// falls through to its static teach-cache and finally to -// peer.Decode(NodeID). The membership view is read on every call (cheap -// in-memory copy) so newly-joined peers are picked up without restart. -func membershipPeerIDResolver(mv ifaces.Members, logger *slog.Logger) func(ifaces.NodeID) (peer.ID, bool) { +// membershipPeerIDResolver also installs the target's Pod-IP addresses before +// direct coordination RPCs dial it; DHT bootstrap does not populate every peer. +func membershipPeerIDResolver(mv ifaces.Members, ps peerstore.Peerstore, logger *slog.Logger) func(ifaces.NodeID) (peer.ID, bool) { return func(id ifaces.NodeID) (peer.ID, bool) { for _, n := range mv.Snapshot() { if n.ID != id || n.PeerID == "" { @@ -1314,6 +1314,41 @@ func membershipPeerIDResolver(mv ifaces.Members, logger *slog.Logger) func(iface return "", false } + var addrs []multiaddr.Multiaddr + + for _, raw := range n.P2PAddrs { + info, err := peer.AddrInfoFromString(raw) + if err != nil { + if logger != nil { + logger.Debug("membership peer address decode failed", + slog.String("node_id", string(id)), + slog.String("address", raw), + slog.Any("err", err), + ) + } + + continue + } + + if info.ID != pid { + if logger != nil { + logger.Warn("membership peer address identity mismatch", + slog.String("node_id", string(id)), + slog.String("peer_id", pid.String()), + slog.String("address_peer_id", info.ID.String()), + ) + } + + continue + } + + addrs = append(addrs, info.Addrs...) + } + + if ps != nil && len(addrs) > 0 { + ps.AddAddrs(pid, addrs, peerstore.AddressTTL) + } + return pid, true } diff --git a/cmd/gantry/membership_peer_resolver_test.go b/cmd/gantry/membership_peer_resolver_test.go new file mode 100644 index 000000000..7486083f8 --- /dev/null +++ b/cmd/gantry/membership_peer_resolver_test.go @@ -0,0 +1,106 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package main + +import ( + "log/slog" + "testing" + + libp2p "github.com/libp2p/go-libp2p" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/multiformats/go-multiaddr" + + "github.com/Azure/unbounded/internal/gantry/ifaces" + "github.com/Azure/unbounded/internal/gantry/ifaces/fakes" +) + +func TestMembershipPeerIDResolverInstallsPodAddresses(t *testing.T) { + t.Parallel() + + target, err := libp2p.New(libp2p.NoListenAddrs) + if err != nil { + t.Fatalf("create target host: %v", err) + } + + defer func() { _ = target.Close() }() //nolint:errcheck // best-effort test cleanup + + caller, err := libp2p.New(libp2p.NoListenAddrs) + if err != nil { + t.Fatalf("create caller host: %v", err) + } + + defer func() { _ = caller.Close() }() //nolint:errcheck // best-effort test cleanup + + announced := "/ip4/10.64.1.23/tcp/4001/p2p/" + target.ID().String() + members := fakes.NewMembers("self", ifaces.Node{ + ID: "target-node", + PeerID: target.ID().String(), + P2PAddrs: []string{announced}, + }) + + resolve := membershipPeerIDResolver(members, caller.Peerstore(), slog.Default()) + + got, ok := resolve("target-node") + if !ok || got != target.ID() { + t.Fatalf("resolved peer = %q, %v; want %q, true", got, ok, target.ID()) + } + + want := multiaddr.StringCast("/ip4/10.64.1.23/tcp/4001") + if addrs := caller.Peerstore().Addrs(target.ID()); len(addrs) != 1 || !addrs[0].Equal(want) { + t.Fatalf("peerstore addresses = %v, want [%s]", addrs, want) + } +} + +func TestMembershipPeerIDResolverRejectsMismatchedAddressIdentity(t *testing.T) { + t.Parallel() + + target, err := libp2p.New(libp2p.NoListenAddrs) + if err != nil { + t.Fatalf("create target host: %v", err) + } + + defer func() { _ = target.Close() }() //nolint:errcheck // best-effort test cleanup + + other, err := libp2p.New(libp2p.NoListenAddrs) + if err != nil { + t.Fatalf("create other host: %v", err) + } + + defer func() { _ = other.Close() }() //nolint:errcheck // best-effort test cleanup + + caller, err := libp2p.New(libp2p.NoListenAddrs) + if err != nil { + t.Fatalf("create caller host: %v", err) + } + + defer func() { _ = caller.Close() }() //nolint:errcheck // best-effort test cleanup + + members := fakes.NewMembers("self", ifaces.Node{ + ID: "target-node", + PeerID: target.ID().String(), + P2PAddrs: []string{"/ip4/10.64.1.23/tcp/4001/p2p/" + other.ID().String()}, + }) + + resolve := membershipPeerIDResolver(members, caller.Peerstore(), slog.Default()) + + got, ok := resolve("target-node") + if !ok || got != target.ID() { + t.Fatalf("resolved peer = %q, %v; want %q, true", got, ok, target.ID()) + } + + if addrs := caller.Peerstore().Addrs(target.ID()); len(addrs) != 0 { + t.Fatalf("peerstore addresses = %v, want none for mismatched identity", addrs) + } +} + +func TestMembershipPeerIDResolverMiss(t *testing.T) { + t.Parallel() + + members := fakes.NewMembers("self") + resolve := membershipPeerIDResolver(members, nil, slog.Default()) + + if got, ok := resolve(ifaces.NodeID(peer.ID("missing"))); ok || got != "" { + t.Fatalf("resolved missing peer = %q, %v; want empty, false", got, ok) + } +} diff --git a/hack/cmd/gantry-benchmark/performance_telemetry.go b/hack/cmd/gantry-benchmark/performance_telemetry.go index 8793f6112..e3b6e65d4 100644 --- a/hack/cmd/gantry-benchmark/performance_telemetry.go +++ b/hack/cmd/gantry-benchmark/performance_telemetry.go @@ -83,7 +83,7 @@ func performanceTelemetryQueries() []performanceTelemetryQuery { {name: "gantry_commit_observation", query: `{__name__=~"gantry_containerd_commit_(observed_total|observed_timestamp_seconds|observation_duration_seconds_(sum|count)|latest_observation_duration_seconds|missing_after_stream_total)",gantry_benchmark="true"}`}, // Seeding width: how many HRW pullers each layer actually activates, and // how many please_pull requests convert into origin pulls versus dedup. - {name: "gantry_coord_seeding", query: `{__name__=~"p2p_coord_please_pull_(served|started|declined)_total|p2p_coord_pull_intent_served_total|p2p_prefetch_(batches|digests)_total",gantry_benchmark="true"}`}, + {name: "gantry_coord_seeding", query: `{__name__=~"p2p_coord_please_pull_(served|started|declined)_total|p2p_coord_pull_intent_served_total|p2p_prefetch_(batches|digests|groups)_total",gantry_benchmark="true"}`}, {name: "gantry_prefetch_pullers", query: `{__name__=~"p2p_prefetch_pullers_per_manifest_(bucket|sum|count)",gantry_benchmark="true"}`}, } } diff --git a/hack/gantry-benchmark/manifests/monitoring.yaml.tmpl b/hack/gantry-benchmark/manifests/monitoring.yaml.tmpl index 45b824708..2df9acd53 100644 --- a/hack/gantry-benchmark/manifests/monitoring.yaml.tmpl +++ b/hack/gantry-benchmark/manifests/monitoring.yaml.tmpl @@ -29,7 +29,7 @@ spec: - action: keep sourceLabels: - __name__ - regex: gantry_storage_mode_info|p2p_dht_health_score|p2p_dht_lookup_total|p2p_dht_lookup_duration_seconds_(bucket|sum|count)|gantry_peer_serve_bytes_total|gantry_origin_bytes_total|p2p_origin_pull_total|p2p_peer_fetch_total|gantry_peer_fetch_last_timestamp_seconds|p2p_peer_fetch_duration_seconds_(bucket|sum|count)|p2p_origin_pull_success_total|p2p_origin_fallback_total|p2p_in_flight_pulls|gantry_peer_fetch_bytes_total|gantry_mirror_bytes_served_total|gantry_mirror_response_completed_timestamp_seconds|gantry_containerd_commit_observed_total|gantry_containerd_commit_observed_timestamp_seconds|gantry_containerd_commit_observation_duration_seconds_(bucket|sum|count)|gantry_containerd_commit_latest_observation_duration_seconds|gantry_containerd_commit_missing_after_stream_total|p2p_coord_please_pull_(served|started|declined)_total|p2p_coord_pull_intent_served_total|p2p_prefetch_(batches|digests)_total|p2p_prefetch_pullers_per_manifest_(bucket|sum|count) + regex: gantry_storage_mode_info|p2p_dht_health_score|p2p_dht_lookup_total|p2p_dht_lookup_duration_seconds_(bucket|sum|count)|gantry_peer_serve_bytes_total|gantry_origin_bytes_total|p2p_origin_pull_total|p2p_peer_fetch_total|gantry_peer_fetch_last_timestamp_seconds|p2p_peer_fetch_duration_seconds_(bucket|sum|count)|p2p_origin_pull_success_total|p2p_origin_fallback_total|p2p_in_flight_pulls|gantry_peer_fetch_bytes_total|gantry_mirror_bytes_served_total|gantry_mirror_response_completed_timestamp_seconds|gantry_containerd_commit_observed_total|gantry_containerd_commit_observed_timestamp_seconds|gantry_containerd_commit_observation_duration_seconds_(bucket|sum|count)|gantry_containerd_commit_latest_observation_duration_seconds|gantry_containerd_commit_missing_after_stream_total|p2p_coord_please_pull_(served|started|declined)_total|p2p_coord_pull_intent_served_total|p2p_prefetch_(batches|digests|groups)_total|p2p_prefetch_pullers_per_manifest_(bucket|sum|count) - action: replace targetLabel: gantry_benchmark replacement: "true" diff --git a/internal/gantry/coldstart/coldstart.go b/internal/gantry/coldstart/coldstart.go index d2bf8086e..7182179f8 100644 --- a/internal/gantry/coldstart/coldstart.go +++ b/internal/gantry/coldstart/coldstart.go @@ -111,6 +111,8 @@ type MetricsHooks struct { // `p2p_prefetch_batches_total` (count) and // `p2p_prefetch_digests_batched_total` (sum). OnPrefetchBatch func(pullers, digests int) + // OnPrefetchGroup fires once per local or remote dispatch group. + OnPrefetchGroup func(target, outcome string) } // Options configures a Resolver. diff --git a/internal/gantry/coldstart/coldstart_test.go b/internal/gantry/coldstart/coldstart_test.go index 0c45c7338..3cf3c2d52 100644 --- a/internal/gantry/coldstart/coldstart_test.go +++ b/internal/gantry/coldstart/coldstart_test.go @@ -146,7 +146,7 @@ func buildResolver(t *testing.T, coord ifaces.Coordinator, disco coldstart.Disco // buildResolverWithReplicas mirrors buildResolver but sets // PrefetchPullerReplicas so prefetch fan-out can be exercised. -func buildResolverWithReplicas(t *testing.T, coord ifaces.Coordinator, disco coldstart.Discovery, self ifaces.NodeID, members []ifaces.Node, replicas int) *coldstart.Resolver { +func buildResolverWithReplicas(t *testing.T, coord ifaces.Coordinator, disco coldstart.Discovery, self ifaces.NodeID, members []ifaces.Node, replicas int, metrics coldstart.MetricsHooks) *coldstart.Resolver { t.Helper() mems := fakes.NewMembers(self, members...) @@ -162,6 +162,7 @@ func buildResolverWithReplicas(t *testing.T, coord ifaces.Coordinator, disco col HrwK: 3, HrwScope: hrw.ScopeCluster, PrefetchPullerReplicas: replicas, + Metrics: metrics, QueryTimeout: 200 * time.Millisecond, PollManifest: 20 * time.Millisecond, PollLayer: 50 * time.Millisecond, diff --git a/internal/gantry/coldstart/prefetch.go b/internal/gantry/coldstart/prefetch.go index 4b87173a6..de66cf5ec 100644 --- a/internal/gantry/coldstart/prefetch.go +++ b/internal/gantry/coldstart/prefetch.go @@ -310,11 +310,18 @@ func (r *Resolver) PrefetchChildren(ctx context.Context, children []ChildDigest, _, err := r.opts.LocalPull.StartLocalPull(callCtx, registry, repository, kind, ds) if err != nil { failures.Add(1) + + if r.opts.Metrics.OnPrefetchGroup != nil { + r.opts.Metrics.OnPrefetchGroup("local", "error") + } + r.opts.Logger.Debug("coldstart: prefetch local pull failed", slog.String("kind", kind.String()), slog.Int("batch_size", len(ds)), slog.Any("err", err), ) + } else if r.opts.Metrics.OnPrefetchGroup != nil { + r.opts.Metrics.OnPrefetchGroup("local", "success") } }(k, digests) } @@ -333,12 +340,19 @@ func (r *Resolver) PrefetchChildren(ctx context.Context, children []ChildDigest, _, err := r.opts.Coord.PleasePull(callCtx, node, registry, repository, kind, digests) if err != nil { failures.Add(1) + + if r.opts.Metrics.OnPrefetchGroup != nil { + r.opts.Metrics.OnPrefetchGroup("remote", "error") + } + r.opts.Logger.Debug("coldstart: prefetch please_pull failed", slog.String("puller", string(node)), slog.String("kind", kind.String()), slog.Int("batch_size", len(digests)), slog.Any("err", err), ) + } else if r.opts.Metrics.OnPrefetchGroup != nil { + r.opts.Metrics.OnPrefetchGroup("remote", "success") } }(gk.node, gk.kind, ds) } diff --git a/internal/gantry/coldstart/prefetch_test.go b/internal/gantry/coldstart/prefetch_test.go index 61d739bc6..c359f2ab9 100644 --- a/internal/gantry/coldstart/prefetch_test.go +++ b/internal/gantry/coldstart/prefetch_test.go @@ -124,7 +124,7 @@ func TestPrefetchChildren_ReplicatesToTopNPullers(t *testing.T) { coord := &stubCoord{} disco := &stubDisco{health: 1.0} - r := buildResolverWithReplicas(t, coord, disco, self, cluster, 3) + r := buildResolverWithReplicas(t, coord, disco, self, cluster, 3, coldstart.MetricsHooks{}) children := []coldstart.ChildDigest{{Digest: d, Kind: ifaces.KindBlob}} if err := r.PrefetchChildren(context.Background(), children, "docker.io", "library/nginx"); err != nil { @@ -146,6 +146,65 @@ func TestPrefetchChildren_ReplicatesToTopNPullers(t *testing.T) { } } +func TestPrefetchChildren_ReportsRemoteGroupOutcomes(t *testing.T) { + cluster := clusterNodes() + self := ifaces.NodeID("n3") + + var ( + d digest.Digest + found bool + ) + + for i := 0; i < 8192; i++ { + candidate := digest.MustParse("sha256:" + digestHex(i)) + + top := hrw.TopK(cluster, candidate, 2) + if len(top) == 2 && top[0].Node.ID != self && top[1].Node.ID != self { + d = candidate + found = true + + break + } + } + + if !found { + t.Fatal("could not find digest with two remote pullers") + } + + top := hrw.TopK(cluster, d, 2) + coord := &stubCoord{pleasePullErrs: map[ifaces.NodeID]error{ + top[0].Node.ID: errors.New("dial failed"), + }} + + var ( + mu sync.Mutex + outcomes []string + ) + + r := buildResolverWithReplicas(t, coord, &stubDisco{health: 1.0}, self, cluster, 2, coldstart.MetricsHooks{ + OnPrefetchGroup: func(target, outcome string) { + mu.Lock() + defer mu.Unlock() + + outcomes = append(outcomes, target+":"+outcome) + }, + }) + + err := r.PrefetchChildren(context.Background(), []coldstart.ChildDigest{{Digest: d, Kind: ifaces.KindBlob}}, "docker.io", "library/nginx") + if !errors.Is(err, coldstart.ErrPrefetchPartial) { + t.Fatalf("PrefetchChildren error = %v, want ErrPrefetchPartial", err) + } + + mu.Lock() + defer mu.Unlock() + + sort.Strings(outcomes) + + if got, want := fmt.Sprint(outcomes), "[remote:error remote:success]"; got != want { + t.Fatalf("group outcomes = %s, want %s", got, want) + } +} + func TestPrefetchChildren_FractionScalesWithoutCap(t *testing.T) { for _, test := range []struct { nodes int From fab5beadc10f5b17efc4f8270bf2695bf8026d8b Mon Sep 17 00:00:00 2001 From: Vaibhav Patel Date: Thu, 6 Aug 2026 21:54:57 -0400 Subject: [PATCH 42/60] fix(gantry): smooth redundant prefetch dispatch Cold-start prefetch intentionally lets every requester dispatch the same HRW seeding plan, relying on target-side in-flight dedup for resilience. At 1000 nodes the unbounded implementation launched roughly 560,000 libp2p RPC groups at once. All 822 intended origin pulls started, but 51% of duplicate groups failed during connection establishment with refusals, handshake resets, dial backoff, or libp2p resource limits. Preserve the redundant fan-out design while smoothing its connection burst: - cap simultaneous outbound remote groups at 64 per manifest - rotate the deterministic target order by sender node ID and manifest - add a deterministic per-node dispatch delay in [0, 1s) - retain the existing 2s per-group timeout and target-side dedup Expose both controls through YAML, environment, and flags as prefetch_max_concurrent_groups and prefetch_dispatch_jitter. Production defaults are 64 and 1s; direct resolver tests keep jitter disabled when unset. Log each manifest's concurrency, offset, and delay for diagnosis. Tests block RPCs to prove a third group cannot start while two slots are occupied, verify different nodes derive stable but different dispatch plans, and cover config defaults, environment, flags, and validation. --- cmd/gantry/main.go | 32 +++--- deploy/gantry/configmap.yaml.tmpl | 4 + internal/gantry/coldstart/coldstart.go | 10 ++ internal/gantry/coldstart/prefetch.go | 67 ++++++++++++- .../coldstart/prefetch_internal_test.go | 58 +++++++++++ internal/gantry/coldstart/prefetch_test.go | 99 +++++++++++++++++++ internal/gantry/config/config.go | 32 +++++- internal/gantry/config/config_test.go | 68 +++++++++++++ 8 files changed, 349 insertions(+), 21 deletions(-) create mode 100644 internal/gantry/coldstart/prefetch_internal_test.go diff --git a/cmd/gantry/main.go b/cmd/gantry/main.go index c886b9b04..8f2afda73 100644 --- a/cmd/gantry/main.go +++ b/cmd/gantry/main.go @@ -454,21 +454,23 @@ func runAgent(args []string) error { if hasMultiNodeMembership(memberView) { selfZone := lookupSelfZone(memberView) realResolver := coldstart.New(coldstart.Options{ - Members: memberView, - Discovery: disco, - Coord: coordClient, - Inflight: inflightMap, - Logger: logger, - HrwK: c.HRWK, - HrwScope: hrw.ParseScope(c.HRWTopologyScope), - SelfZone: selfZone, - LocalIntent: coordServer, - LocalPull: coordServer, - PrefetchPullerReplicas: c.PrefetchPullerReplicas, - PrefetchPullerFraction: c.PrefetchPullerFraction, - TransientCooldownCap: c.OriginFailureHonorWindowCap, - TopKExpansionFactor: c.TopKExpansionFactorDegraded, - TrustedFailureClasses: parseTrustedFailureClasses(c.OriginFailureClassesTrustedClusterWide, logger), + Members: memberView, + Discovery: disco, + Coord: coordClient, + Inflight: inflightMap, + Logger: logger, + HrwK: c.HRWK, + HrwScope: hrw.ParseScope(c.HRWTopologyScope), + SelfZone: selfZone, + LocalIntent: coordServer, + LocalPull: coordServer, + PrefetchPullerReplicas: c.PrefetchPullerReplicas, + PrefetchPullerFraction: c.PrefetchPullerFraction, + PrefetchMaxConcurrentGroups: c.PrefetchMaxConcurrentGroups, + PrefetchDispatchJitter: c.PrefetchDispatchJitter, + TransientCooldownCap: c.OriginFailureHonorWindowCap, + TopKExpansionFactor: c.TopKExpansionFactorDegraded, + TrustedFailureClasses: parseTrustedFailureClasses(c.OriginFailureClassesTrustedClusterWide, logger), Metrics: coldstart.MetricsHooks{ OnRankMismatch: func(kindLabel string, _ ifaces.NodeID) { p3.hrwRankMismatch.WithLabelValues(kindLabel).Inc() diff --git a/deploy/gantry/configmap.yaml.tmpl b/deploy/gantry/configmap.yaml.tmpl index e8dcc2781..0723d1553 100644 --- a/deploy/gantry/configmap.yaml.tmpl +++ b/deploy/gantry/configmap.yaml.tmpl @@ -140,6 +140,10 @@ data: # A positive fraction overrides prefetch_puller_replicas. There is no # maximum cap other than the eligible cluster/zone node count. prefetch_puller_fraction: 0.02 + # Bound and desynchronize redundant best-effort prefetch dispatch. Without + # this, every requester opens roughly 550 libp2p RPC groups at once. + prefetch_max_concurrent_groups: 64 + prefetch_dispatch_jitter: "1s" # Coord peer authorization. Default false ships observe-only: an # inbound coord request from a peer not in the membership view only # increments p2p_coord_unauthorized_peer_total and is still served. diff --git a/internal/gantry/coldstart/coldstart.go b/internal/gantry/coldstart/coldstart.go index 7182179f8..099d65057 100644 --- a/internal/gantry/coldstart/coldstart.go +++ b/internal/gantry/coldstart/coldstart.go @@ -142,6 +142,12 @@ type Options struct { // than zero. The resolver selects ceil(eligible candidates * fraction), // with a minimum of one and no cap other than the candidate count. PrefetchPullerFraction float64 + // PrefetchMaxConcurrentGroups caps simultaneous remote dispatch groups. + // Zero uses 64. + PrefetchMaxConcurrentGroups int + // PrefetchDispatchJitter is disabled at zero. Production config defaults + // it to one second; direct tests can remain deterministic and immediate. + PrefetchDispatchJitter time.Duration // LocalIntent computes self's PullIntent synchronously, without // the libp2p coord round-trip. When non-nil, the cold-start @@ -239,6 +245,10 @@ func New(opts Options) *Resolver { opts.QueryTimeout = 2 * time.Second } + if opts.PrefetchMaxConcurrentGroups <= 0 { + opts.PrefetchMaxConcurrentGroups = 64 + } + if opts.PollManifest <= 0 { opts.PollManifest = 200 * time.Millisecond } diff --git a/internal/gantry/coldstart/prefetch.go b/internal/gantry/coldstart/prefetch.go index de66cf5ec..5f51993c1 100644 --- a/internal/gantry/coldstart/prefetch.go +++ b/internal/gantry/coldstart/prefetch.go @@ -28,17 +28,47 @@ import ( "context" "errors" "fmt" + "hash/fnv" "log/slog" "math" "sort" "sync" "sync/atomic" + "time" "github.com/Azure/unbounded/internal/gantry/digest" "github.com/Azure/unbounded/internal/gantry/hrw" "github.com/Azure/unbounded/internal/gantry/ifaces" ) +func prefetchDispatchHash(self ifaces.NodeID, children []ChildDigest) uint64 { + hasher := fnv.New64a() + _, _ = hasher.Write([]byte(self)) + _, _ = hasher.Write([]byte{0}) + + if len(children) > 0 { + _, _ = hasher.Write([]byte(children[0].Digest.String())) + } + + return hasher.Sum64() +} + +func prefetchDispatchPlan(self ifaces.NodeID, children []ChildDigest, groups int, maxJitter time.Duration) (int, time.Duration) { + dispatchHash := prefetchDispatchHash(self, children) + + offset := 0 + if groups > 1 { + offset = int(dispatchHash % uint64(groups)) + } + + delay := time.Duration(0) + if maxJitter > 0 { + delay = time.Duration(dispatchHash % uint64(maxJitter)) + } + + return offset, delay +} + // PrefetchLayers groups digests by their HRW rank-0 reachable // designated puller and issues one PleasePull RPC per puller. Digests // HRW'ing to self are diverted to the local LocalPullStarter (if @@ -236,6 +266,15 @@ func (r *Resolver) PrefetchChildren(ctx context.Context, children []ChildDigest, return groupKeys[i].kind < groupKeys[j].kind }) + + dispatchOffset, dispatchDelay := prefetchDispatchPlan(self, children, len(groupKeys), r.opts.PrefetchDispatchJitter) + if len(groupKeys) > 1 { + rotated := make([]groupKey, 0, len(groupKeys)) + rotated = append(rotated, groupKeys[dispatchOffset:]...) + rotated = append(rotated, groupKeys[:dispatchOffset]...) + groupKeys = rotated + } + // Sort self-kinds too so the goroutine launch is deterministic. selfKinds := make([]ifaces.OriginRefKind, 0, len(selfByKind)) for k := range selfByKind { @@ -276,12 +315,26 @@ func (r *Resolver) PrefetchChildren(ctx context.Context, children []ChildDigest, slog.Int("pullers", totalPullers), slog.Int("rpc_groups", len(byGroup)+len(selfByKind)), slog.Int("skipped_self", skippedSelf), + slog.Int("max_concurrent_groups", r.opts.PrefetchMaxConcurrentGroups), + slog.Int("dispatch_offset", dispatchOffset), + slog.Duration("dispatch_delay", dispatchDelay), ) if r.opts.Metrics.OnPrefetchBatch != nil { r.opts.Metrics.OnPrefetchBatch(totalPullers, totalDigests) } + if dispatchDelay > 0 { + timer := time.NewTimer(dispatchDelay) + defer timer.Stop() + + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + } + } + var ( wg sync.WaitGroup failures atomic.Int32 @@ -326,13 +379,25 @@ func (r *Resolver) PrefetchChildren(ctx context.Context, children []ChildDigest, }(k, digests) } - for _, gk := range groupKeys { + remoteSlots := make(chan struct{}, r.opts.PrefetchMaxConcurrentGroups) + +dispatchRemote: + for index, gk := range groupKeys { ds := byGroup[gk] + select { + case remoteSlots <- struct{}{}: + case <-ctx.Done(): + failures.Add(int32(len(groupKeys) - index)) + + break dispatchRemote + } + wg.Add(1) go func(node ifaces.NodeID, kind ifaces.OriginRefKind, digests []digest.Digest) { defer wg.Done() + defer func() { <-remoteSlots }() callCtx, cancel := context.WithTimeout(ctx, r.opts.QueryTimeout) defer cancel() diff --git a/internal/gantry/coldstart/prefetch_internal_test.go b/internal/gantry/coldstart/prefetch_internal_test.go new file mode 100644 index 000000000..ae8c25878 --- /dev/null +++ b/internal/gantry/coldstart/prefetch_internal_test.go @@ -0,0 +1,58 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package coldstart + +import ( + "testing" + "time" + + "github.com/Azure/unbounded/internal/gantry/digest" + "github.com/Azure/unbounded/internal/gantry/ifaces" +) + +func TestPrefetchDispatchPlanDesynchronizesNodesDeterministically(t *testing.T) { + t.Parallel() + + children := []ChildDigest{{ + Digest: digest.MustParse("sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), + Kind: ifaces.KindBlob, + }} + + offsetA, delayA := prefetchDispatchPlan("node-a", children, 554, time.Second) + offsetA2, delayA2 := prefetchDispatchPlan("node-a", children, 554, time.Second) + offsetB, delayB := prefetchDispatchPlan("node-b", children, 554, time.Second) + + if offsetA != offsetA2 || delayA != delayA2 { + t.Fatalf("same node plan changed: (%d, %v) != (%d, %v)", offsetA, delayA, offsetA2, delayA2) + } + + if offsetA == offsetB && delayA == delayB { + t.Fatalf("different nodes got the same plan: node-a=(%d,%v) node-b=(%d,%v)", offsetA, delayA, offsetB, delayB) + } + + for node, plan := range map[ifaces.NodeID]struct { + offset int + delay time.Duration + }{ + "node-a": {offset: offsetA, delay: delayA}, + "node-b": {offset: offsetB, delay: delayB}, + } { + if plan.offset < 0 || plan.offset >= 554 { + t.Errorf("%s offset = %d, want [0,554)", node, plan.offset) + } + + if plan.delay < 0 || plan.delay >= time.Second { + t.Errorf("%s delay = %v, want [0,1s)", node, plan.delay) + } + } +} + +func TestPrefetchDispatchPlanDisabledJitter(t *testing.T) { + t.Parallel() + + offset, delay := prefetchDispatchPlan("node-a", nil, 0, 0) + if offset != 0 || delay != 0 { + t.Fatalf("plan = (%d,%v), want (0,0)", offset, delay) + } +} diff --git a/internal/gantry/coldstart/prefetch_test.go b/internal/gantry/coldstart/prefetch_test.go index c359f2ab9..735065930 100644 --- a/internal/gantry/coldstart/prefetch_test.go +++ b/internal/gantry/coldstart/prefetch_test.go @@ -16,9 +16,52 @@ import ( "github.com/Azure/unbounded/internal/gantry/digest" "github.com/Azure/unbounded/internal/gantry/hrw" "github.com/Azure/unbounded/internal/gantry/ifaces" + "github.com/Azure/unbounded/internal/gantry/ifaces/fakes" + "github.com/Azure/unbounded/internal/gantry/inflight" "github.com/Azure/unbounded/internal/gantry/registryauth" ) +type blockingPrefetchCoord struct { + mu sync.Mutex + active int + maxActive int + started chan struct{} + release chan struct{} +} + +func (c *blockingPrefetchCoord) PullIntentQuery(context.Context, ifaces.NodeID, digest.Digest) (ifaces.PullIntent, error) { + return ifaces.PullIntent{}, nil +} + +func (c *blockingPrefetchCoord) PleasePull(ctx context.Context, _ ifaces.NodeID, _, _ string, _ ifaces.OriginRefKind, digests []digest.Digest) ([]ifaces.PleasePullOutcome, error) { + c.mu.Lock() + + c.active++ + if c.active > c.maxActive { + c.maxActive = c.active + } + c.mu.Unlock() + + c.started <- struct{}{} + + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-c.release: + } + + c.mu.Lock() + c.active-- + c.mu.Unlock() + + outcomes := make([]ifaces.PleasePullOutcome, len(digests)) + for index, d := range digests { + outcomes[index] = ifaces.PleasePullOutcome{Digest: d, Outcome: ifaces.PleasePullStarted} + } + + return outcomes, nil +} + // pickHRW0 returns the HRW rank-0 node ID for d across the given // cluster. Used by tests to set up "send N digests, expect M batches" // scenarios deterministically. @@ -205,6 +248,62 @@ func TestPrefetchChildren_ReportsRemoteGroupOutcomes(t *testing.T) { } } +func TestPrefetchChildren_BoundsRemoteGroupConcurrency(t *testing.T) { + nodes := make([]ifaces.Node, 8) + for index := range nodes { + nodes[index] = ifaces.Node{ID: ifaces.NodeID(fmt.Sprintf("n%d", index))} + } + + coord := &blockingPrefetchCoord{ + started: make(chan struct{}, len(nodes)), + release: make(chan struct{}), + } + resolver := coldstart.New(coldstart.Options{ + Members: fakes.NewMembers("requester", nodes...), + Discovery: &stubDisco{health: 1.0}, + Coord: coord, + Inflight: inflight.New(inflight.DefaultStalls(), time.Now), + HrwScope: hrw.ScopeCluster, + PrefetchPullerReplicas: len(nodes), + PrefetchMaxConcurrentGroups: 2, + QueryTimeout: time.Second, + }) + + d := digest.MustParse("sha256:" + digestHex(9000)) + done := make(chan error, 1) + + go func() { + done <- resolver.PrefetchChildren(context.Background(), []coldstart.ChildDigest{{Digest: d, Kind: ifaces.KindBlob}}, "docker.io", "library/nginx") + }() + + for range 2 { + select { + case <-coord.started: + case <-time.After(time.Second): + t.Fatal("timed out waiting for bounded prefetch workers") + } + } + + select { + case <-coord.started: + t.Fatal("a third remote group started while the two slots were occupied") + case <-time.After(50 * time.Millisecond): + } + + close(coord.release) + + if err := <-done; err != nil { + t.Fatalf("PrefetchChildren: %v", err) + } + + coord.mu.Lock() + defer coord.mu.Unlock() + + if coord.maxActive != 2 { + t.Fatalf("max active groups = %d, want 2", coord.maxActive) + } +} + func TestPrefetchChildren_FractionScalesWithoutCap(t *testing.T) { for _, test := range []struct { nodes int diff --git a/internal/gantry/config/config.go b/internal/gantry/config/config.go index 6aba7f647..c0e7a7a6a 100644 --- a/internal/gantry/config/config.go +++ b/internal/gantry/config/config.go @@ -266,6 +266,14 @@ type Config struct { // except by the number of eligible nodes. PrefetchPullerFraction float64 `yaml:"prefetch_puller_fraction"` + // PrefetchMaxConcurrentGroups caps simultaneous outbound prefetch groups + // per manifest. Group dispatch is best effort and target-side deduplicated. + PrefetchMaxConcurrentGroups int `yaml:"prefetch_max_concurrent_groups"` + + // PrefetchDispatchJitter spreads manifest prefetch across requesters. Each + // node derives a stable delay in [0, jitter) from itself and the manifest. + PrefetchDispatchJitter time.Duration `yaml:"prefetch_dispatch_jitter"` + // HRWTopologyScope selects "cluster" (HRW over all nodes) or "zone" // (HRW within the requester's zone) - the design doc / the design doc open question. HRWTopologyScope string `yaml:"hrw_topology_scope"` @@ -465,11 +473,13 @@ func NewDefault() *Config { UpstreamRegistries: nil, - HRWK: 3, - PrefetchPullerReplicas: 8, - PrefetchPullerFraction: 0, - HRWTopologyScope: "cluster", - ZoneLabelKey: "topology.kubernetes.io/zone", + HRWK: 3, + PrefetchPullerReplicas: 8, + PrefetchPullerFraction: 0, + PrefetchMaxConcurrentGroups: 64, + PrefetchDispatchJitter: time.Second, + HRWTopologyScope: "cluster", + ZoneLabelKey: "topology.kubernetes.io/zone", CoordPeerAuthzEnforce: false, CoordMaxDigestsPerRequest: 256, @@ -599,6 +609,8 @@ func (c *Config) LoadEnv(env func(string) string) error { setInt("HRW_K", &c.HRWK) setInt("PREFETCH_PULLER_REPLICAS", &c.PrefetchPullerReplicas) setFloat("PREFETCH_PULLER_FRACTION", &c.PrefetchPullerFraction) + setInt("PREFETCH_MAX_CONCURRENT_GROUPS", &c.PrefetchMaxConcurrentGroups) + setDur("PREFETCH_DISPATCH_JITTER", &c.PrefetchDispatchJitter) setStr("HRW_TOPOLOGY_SCOPE", &c.HRWTopologyScope) setStr("ZONE_LABEL_KEY", &c.ZoneLabelKey) setBool("COORD_PEER_AUTHZ_ENFORCE", &c.CoordPeerAuthzEnforce) @@ -662,6 +674,8 @@ func (c *Config) BindFlags(fs *flag.FlagSet) { fs.IntVar(&c.HRWK, "hrw-k", c.HRWK, "HRW top-K size") fs.IntVar(&c.PrefetchPullerReplicas, "prefetch-puller-replicas", c.PrefetchPullerReplicas, "number of HRW-ranked pullers each prefetched layer digest is pulled by (initial seeds); 1 = single puller/tightest dedup, N = N-fold peer fan-out at N origin copies") fs.Float64Var(&c.PrefetchPullerFraction, "prefetch-puller-fraction", c.PrefetchPullerFraction, "fraction of eligible HRW nodes selected as initial pullers, rounded up (0 disables and uses --prefetch-puller-replicas)") + fs.IntVar(&c.PrefetchMaxConcurrentGroups, "prefetch-max-concurrent-groups", c.PrefetchMaxConcurrentGroups, "maximum simultaneous outbound prefetch RPC groups per manifest") + fs.DurationVar(&c.PrefetchDispatchJitter, "prefetch-dispatch-jitter", c.PrefetchDispatchJitter, "maximum deterministic per-node delay before dispatching manifest prefetch") fs.StringVar(&c.HRWTopologyScope, "hrw-topology-scope", c.HRWTopologyScope, `HRW scope: "cluster" or "zone"`) fs.StringVar(&c.ZoneLabelKey, "zone-label-key", c.ZoneLabelKey, "Kubernetes node label identifying the zone (used when hrw-topology-scope=zone)") fs.BoolVar(&c.CoordPeerAuthzEnforce, "coord-peer-authz-enforce", c.CoordPeerAuthzEnforce, "reject inbound coord requests from peers not in the membership view (default false = observe-only)") @@ -857,6 +871,14 @@ func (c *Config) Validate() error { errs = append(errs, fmt.Errorf("coord_max_digests_per_request: must be >= 1, got %d", c.CoordMaxDigestsPerRequest)) } + if c.PrefetchMaxConcurrentGroups < 1 { + errs = append(errs, fmt.Errorf("prefetch_max_concurrent_groups: must be >= 1, got %d", c.PrefetchMaxConcurrentGroups)) + } + + if c.PrefetchDispatchJitter < 0 { + errs = append(errs, fmt.Errorf("prefetch_dispatch_jitter: must be >= 0, got %v", c.PrefetchDispatchJitter)) + } + if c.CoordMaxConcurrentPulls < 1 { errs = append(errs, fmt.Errorf("coord_max_concurrent_pulls: must be >= 1, got %d", c.CoordMaxConcurrentPulls)) } diff --git a/internal/gantry/config/config_test.go b/internal/gantry/config/config_test.go index 9a5bd7dfa..fa9d72d50 100644 --- a/internal/gantry/config/config_test.go +++ b/internal/gantry/config/config_test.go @@ -26,6 +26,14 @@ func TestDefaultsValidateAfterMinimalUpstream(t *testing.T) { t.Fatalf("PrefetchPullerFraction = %v, want disabled", c.PrefetchPullerFraction) } + if c.PrefetchMaxConcurrentGroups != 64 { + t.Fatalf("PrefetchMaxConcurrentGroups = %d, want 64", c.PrefetchMaxConcurrentGroups) + } + + if c.PrefetchDispatchJitter != time.Second { + t.Fatalf("PrefetchDispatchJitter = %v, want 1s", c.PrefetchDispatchJitter) + } + if c.TransferMaxConcurrentServes != 10 { t.Fatalf("TransferMaxConcurrentServes = %d, want 10", c.TransferMaxConcurrentServes) } @@ -88,6 +96,66 @@ func TestValidate_PrefetchPullerFractionBounds(t *testing.T) { } } +func TestPrefetchDispatchConfig(t *testing.T) { + t.Run("environment", func(t *testing.T) { + c := NewDefault() + + err := c.LoadEnv(func(key string) string { + switch key { + case "GANTRY_PREFETCH_MAX_CONCURRENT_GROUPS": + return "32" + case "GANTRY_PREFETCH_DISPATCH_JITTER": + return "750ms" + default: + return "" + } + }) + if err != nil { + t.Fatalf("LoadEnv: %v", err) + } + + if c.PrefetchMaxConcurrentGroups != 32 || c.PrefetchDispatchJitter != 750*time.Millisecond { + t.Fatalf("prefetch dispatch config = %d, %v", c.PrefetchMaxConcurrentGroups, c.PrefetchDispatchJitter) + } + }) + + t.Run("flags", func(t *testing.T) { + c := NewDefault() + flags := flag.NewFlagSet("test", flag.ContinueOnError) + c.BindFlags(flags) + + if err := flags.Parse([]string{"--prefetch-max-concurrent-groups=32", "--prefetch-dispatch-jitter=750ms"}); err != nil { + t.Fatalf("Parse: %v", err) + } + + if c.PrefetchMaxConcurrentGroups != 32 || c.PrefetchDispatchJitter != 750*time.Millisecond { + t.Fatalf("prefetch dispatch config = %d, %v", c.PrefetchMaxConcurrentGroups, c.PrefetchDispatchJitter) + } + }) +} + +func TestValidate_PrefetchDispatchBounds(t *testing.T) { + for _, test := range []struct { + name string + mutate func(*Config) + want string + }{ + {name: "zero groups", mutate: func(c *Config) { c.PrefetchMaxConcurrentGroups = 0 }, want: "prefetch_max_concurrent_groups"}, + {name: "negative jitter", mutate: func(c *Config) { c.PrefetchDispatchJitter = -time.Second }, want: "prefetch_dispatch_jitter"}, + } { + t.Run(test.name, func(t *testing.T) { + c := NewDefault() + c.UpstreamRegistries = []UpstreamRegistry{{Name: "r", Endpoint: "https://r"}} + test.mutate(c) + + err := c.Validate() + if err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("want %s error, got %v", test.want, err) + } + }) + } +} + func TestValidate_RequiresUpstream(t *testing.T) { c := NewDefault() From 02293919a7aa473b1fb290185bd77755e2e1c4d1 Mon Sep 17 00:00:00 2001 From: Vaibhav Patel Date: Thu, 6 Aug 2026 22:37:20 -0400 Subject: [PATCH 43/60] feat(gantry-benchmark): add live pull monitor Add a standalone gantry-benchmark-monitor binary for the active Gantry-cold phase. It discovers the active run, Job start time, workload shape, and current DaemonSet revision, then executes one server-side aggregated Prometheus range query per refresh. Render two rolling per-phase-minute tables: - peer outcomes: busy, hit, stall, notfound, unavailable, totals, and first-six-minute shares - layer delivery: GB moved, aggregate and per-node throughput, cumulative percentage, and total payload progress The display redraws every second and marks the current partial minute. It states the 10-second Prometheus scrape cadence so repeated values between scrapes are explicit. Queries aggregate before returning data; the monitor never downloads per-pod series. Add make -C hack/gantry-benchmark monitor, using the idempotent deploy config's dedicated kubeconfig. --once --no-clear emits a non-interactive snapshot. Tests cover Prometheus parsing, minute-bin counter deltas, partial-minute rendering, totals and percentages, number formatting, and revision-scoped PromQL. Validate it against a live 1000-node run: at 9m53s the monitor rendered 317999 busy outcomes and 42.656 TB delivered (99.3%). --- hack/cmd/gantry-benchmark-monitor/main.go | 471 ++++++++++++++++++ hack/cmd/gantry-benchmark-monitor/monitor.go | 334 +++++++++++++ .../gantry-benchmark-monitor/monitor_test.go | 146 ++++++ hack/gantry-benchmark/Makefile | 12 +- hack/gantry-benchmark/README.md | 14 + 5 files changed, 975 insertions(+), 2 deletions(-) create mode 100644 hack/cmd/gantry-benchmark-monitor/main.go create mode 100644 hack/cmd/gantry-benchmark-monitor/monitor.go create mode 100644 hack/cmd/gantry-benchmark-monitor/monitor_test.go diff --git a/hack/cmd/gantry-benchmark-monitor/main.go b/hack/cmd/gantry-benchmark-monitor/main.go new file mode 100644 index 000000000..157e31122 --- /dev/null +++ b/hack/cmd/gantry-benchmark-monitor/main.go @@ -0,0 +1,471 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "net/url" + "os" + "os/exec" + "os/signal" + "strconv" + "strings" + "syscall" + "time" + + "golang.org/x/term" +) + +const ( + defaultBenchmarkNamespace = "gantry-benchmark" + defaultGantryNamespace = "gantry-system" + defaultMonitoringNS = "monitoring" + defaultPrometheusService = "kps-kube-prometheus-stack-prometheus" + stateConfigMapName = "gantry-benchmark-state" + gantryPhaseLabel = "gantry-cold" +) + +type monitorConfig struct { + kubectl string + kubeconfig string + benchmarkNamespace string + gantryNamespace string + monitoringNamespace string + prometheusService string + runID string + refreshInterval time.Duration + once bool + noClear bool +} + +type kubectlRunner struct { + binary string + kubeconfig string +} + +func (r kubectlRunner) run(ctx context.Context, args ...string) ([]byte, error) { + if r.kubeconfig != "" { + args = append([]string{"--kubeconfig", r.kubeconfig}, args...) + } + + command := exec.CommandContext(ctx, r.binary, args...) + + output, err := command.CombinedOutput() + if err != nil { + return nil, fmt.Errorf("kubectl %s: %w: %s", strings.Join(args, " "), err, strings.TrimSpace(string(output))) + } + + return output, nil +} + +type benchmarkState struct { + RunID string `json:"run_id"` + MonitoringNamespace string `json:"monitoring_namespace"` + PrometheusService string `json:"prometheus_service"` + NodeCount int `json:"node_count"` + ImageSizeMiB int `json:"image_size_mib"` +} + +type monitorSession struct { + runID string + jobName string + phaseStart time.Time + nodeCount int + expectedBytes float64 + monitoringNamespace string + prometheusService string + revision string +} + +type jobStatus struct { + Succeeded int + Active int + Failed int + Complete bool +} + +func envDefault(name, fallback string) string { + if value := os.Getenv(name); value != "" { + return value + } + + return fallback +} + +func parseConfig(args []string) (monitorConfig, error) { + config := monitorConfig{} + flags := flag.NewFlagSet("gantry-benchmark-monitor", flag.ContinueOnError) + flags.SetOutput(os.Stderr) + flags.Usage = func() { + fmt.Fprintln(flags.Output(), "Usage: gantry-benchmark-monitor [options]") //nolint:errcheck // best-effort help output + fmt.Fprintln(flags.Output(), "Live per-minute Gantry peer outcomes and layer-byte delivery.") //nolint:errcheck // best-effort help output + flags.PrintDefaults() + } + + flags.StringVar(&config.kubectl, "kubectl", envDefault("KUBECTL", "kubectl"), "kubectl executable") + flags.StringVar(&config.kubeconfig, "kubeconfig", os.Getenv("KUBECONFIG"), "kubeconfig path") + flags.StringVar(&config.benchmarkNamespace, "namespace", envDefault("BENCHMARK_NAMESPACE", defaultBenchmarkNamespace), "benchmark namespace") + flags.StringVar(&config.gantryNamespace, "gantry-namespace", envDefault("GANTRY_NAMESPACE", defaultGantryNamespace), "Gantry namespace") + flags.StringVar(&config.monitoringNamespace, "monitoring-namespace", envDefault("MONITORING_NAMESPACE", defaultMonitoringNS), "monitoring namespace") + flags.StringVar(&config.prometheusService, "prometheus-service", envDefault("PROMETHEUS_SERVICE", defaultPrometheusService), "Prometheus service") + flags.StringVar(&config.runID, "run-id", "", "run ID (default: active benchmark state)") + flags.DurationVar(&config.refreshInterval, "refresh", time.Second, "display and query refresh interval") + flags.BoolVar(&config.once, "once", false, "print one snapshot and exit") + flags.BoolVar(&config.noClear, "no-clear", false, "do not redraw the terminal") + + if err := flags.Parse(args); err != nil { + return monitorConfig{}, err + } + + if config.refreshInterval <= 0 { + return monitorConfig{}, fmt.Errorf("refresh must be positive, got %s", config.refreshInterval) + } + + return config, nil +} + +func loadBenchmarkState(ctx context.Context, runner kubectlRunner, config monitorConfig) (benchmarkState, error) { + output, err := runner.run(ctx, "-n", config.benchmarkNamespace, "get", "configmap", stateConfigMapName, "-o", "json") + if err != nil { + return benchmarkState{}, err + } + + var configMap struct { + Data map[string]string `json:"data"` + } + if err := json.Unmarshal(output, &configMap); err != nil { + return benchmarkState{}, fmt.Errorf("decode benchmark state ConfigMap: %w", err) + } + + raw := configMap.Data["state.json"] + if raw == "" { + return benchmarkState{}, errors.New("benchmark state ConfigMap has no state.json") + } + + var state benchmarkState + if err := json.Unmarshal([]byte(raw), &state); err != nil { + return benchmarkState{}, fmt.Errorf("decode benchmark state: %w", err) + } + + return state, nil +} + +func loadGantryRevision(ctx context.Context, runner kubectlRunner, namespace string) (string, error) { + output, err := runner.run(ctx, "-n", namespace, "get", "pods", "-l", "app.kubernetes.io/name=gantry", "-o", "json") + if err != nil { + return "", err + } + + var pods struct { + Items []struct { + Metadata struct { + Labels map[string]string `json:"labels"` + } `json:"metadata"` + Status struct { + Conditions []struct { + Type string `json:"type"` + Status string `json:"status"` + } `json:"conditions"` + } `json:"status"` + } `json:"items"` + } + if err := json.Unmarshal(output, &pods); err != nil { + return "", fmt.Errorf("decode Gantry pods: %w", err) + } + + for _, pod := range pods.Items { + ready := false + + for _, condition := range pod.Status.Conditions { + if condition.Type == "Ready" && condition.Status == "True" { + ready = true + break + } + } + + if !ready { + continue + } + + if revision := pod.Metadata.Labels["controller-revision-hash"]; revision != "" { + return revision, nil + } + } + + return "", errors.New("no Ready Gantry pod has controller-revision-hash") +} + +func loadJob(ctx context.Context, runner kubectlRunner, namespace, runID string) (monitorSession, jobStatus, error) { + selector := "gantry.unbounded-cloud.io/run-id=" + runID + ",gantry.unbounded-cloud.io/phase=" + gantryPhaseLabel + + output, err := runner.run(ctx, "-n", namespace, "get", "jobs", "-l", selector, "-o", "json") + if err != nil { + return monitorSession{}, jobStatus{}, err + } + + var jobs struct { + Items []struct { + Metadata struct { + Name string `json:"name"` + } `json:"metadata"` + Spec struct { + Completions int `json:"completions"` + } `json:"spec"` + Status struct { + StartTime string `json:"startTime"` + Succeeded int `json:"succeeded"` + Active int `json:"active"` + Failed int `json:"failed"` + Conditions []struct { + Type string `json:"type"` + Status string `json:"status"` + } `json:"conditions"` + } `json:"status"` + } `json:"items"` + } + if err := json.Unmarshal(output, &jobs); err != nil { + return monitorSession{}, jobStatus{}, fmt.Errorf("decode Gantry pull Job: %w", err) + } + + if len(jobs.Items) == 0 { + return monitorSession{}, jobStatus{}, errors.New("gantry-cold pull Job has not started") + } + + job := jobs.Items[len(jobs.Items)-1] + if job.Status.StartTime == "" { + return monitorSession{}, jobStatus{}, errors.New("gantry-cold pull Job has no startTime") + } + + startedAt, err := time.Parse(time.RFC3339, job.Status.StartTime) + if err != nil { + return monitorSession{}, jobStatus{}, fmt.Errorf("parse Job startTime %q: %w", job.Status.StartTime, err) + } + + status := jobStatus{Succeeded: job.Status.Succeeded, Active: job.Status.Active, Failed: job.Status.Failed} + for _, condition := range job.Status.Conditions { + if condition.Type == "Complete" && condition.Status == "True" { + status.Complete = true + } + } + + return monitorSession{jobName: job.Metadata.Name, phaseStart: startedAt, nodeCount: job.Spec.Completions}, status, nil +} + +func discoverSession(ctx context.Context, runner kubectlRunner, config monitorConfig) (monitorSession, jobStatus, error) { + state, err := loadBenchmarkState(ctx, runner, config) + if err != nil { + return monitorSession{}, jobStatus{}, err + } + + runID := config.runID + if runID == "" { + runID = state.RunID + } + + if runID == "" { + return monitorSession{}, jobStatus{}, errors.New("active benchmark state has no run ID") + } + + session, status, err := loadJob(ctx, runner, config.benchmarkNamespace, runID) + if err != nil { + return monitorSession{}, jobStatus{}, err + } + + revision, err := loadGantryRevision(ctx, runner, config.gantryNamespace) + if err != nil { + return monitorSession{}, jobStatus{}, err + } + + session.runID = runID + if session.nodeCount == 0 { + session.nodeCount = state.NodeCount + } + + session.expectedBytes = float64(state.NodeCount) * float64(state.ImageSizeMiB) * 1024 * 1024 + + session.monitoringNamespace = config.monitoringNamespace + if state.MonitoringNamespace != "" { + session.monitoringNamespace = state.MonitoringNamespace + } + + session.prometheusService = config.prometheusService + if state.PrometheusService != "" { + session.prometheusService = state.PrometheusService + } + + session.revision = revision + + return session, status, nil +} + +func refreshJobStatus(ctx context.Context, runner kubectlRunner, config monitorConfig, session monitorSession) (jobStatus, error) { + _, status, err := loadJob(ctx, runner, config.benchmarkNamespace, session.runID) + return status, err +} + +func prometheusExpression(session monitorSession, gantryNamespace string) string { + labels := fmt.Sprintf(`namespace=%s,gantry_benchmark="true",controller_revision_hash=%s`, strconv.Quote(gantryNamespace), strconv.Quote(session.revision)) + peer := fmt.Sprintf(`sum by (outcome) (p2p_peer_fetch_total{%s})`, labels) + bytes := fmt.Sprintf(`sum(gantry_mirror_bytes_served_total{%s,kind="layer"})`, labels) + + return peer + " or " + bytes +} + +func queryPrometheusRange(ctx context.Context, runner kubectlRunner, session monitorSession, gantryNamespace string, now time.Time) (rangeResponse, error) { + start := session.phaseStart.Add(-15 * time.Second) + rawPath := fmt.Sprintf( + "/api/v1/namespaces/%s/services/http:%s:9090/proxy/api/v1/query_range?query=%s&start=%s&end=%s&step=1", + session.monitoringNamespace, + session.prometheusService, + url.QueryEscape(prometheusExpression(session, gantryNamespace)), + url.QueryEscape(start.UTC().Format(time.RFC3339Nano)), + url.QueryEscape(now.UTC().Format(time.RFC3339Nano)), + ) + + output, err := runner.run(ctx, "get", "--raw", rawPath) + if err != nil { + return rangeResponse{}, err + } + + return parseRangeResponse(output) +} + +func renderWaiting(config monitorConfig, err error) { + if !config.noClear && term.IsTerminal(int(os.Stdout.Fd())) { + fmt.Print("\033[H\033[2J") + } + + fmt.Printf("Gantry benchmark live monitor\n\ntime: %s\nstatus: waiting\nreason: %v\n", time.Now().UTC().Format(time.RFC3339), err) +} + +func runMonitor(ctx context.Context, config monitorConfig) error { + runner := kubectlRunner{binary: config.kubectl, kubeconfig: config.kubeconfig} + + var ( + session *monitorSession + lastSnapshot monitorSnapshot + ) + + for { + now := time.Now().UTC() + + var status jobStatus + + if session == nil { + discovered, job, err := discoverSession(ctx, runner, config) + if err != nil { + renderWaiting(config, err) + + if config.once { + return err + } + + if err := waitForNext(ctx, config.refreshInterval); err != nil { + return err + } + + continue + } + + session = &discovered + status = job + } else { + job, err := refreshJobStatus(ctx, runner, config, *session) + if err != nil { + renderWaiting(config, err) + + if err := waitForNext(ctx, config.refreshInterval); err != nil { + return err + } + + continue + } + + status = job + } + + queryCtx, cancel := context.WithTimeout(ctx, 15*time.Second) + response, err := queryPrometheusRange(queryCtx, runner, *session, config.gantryNamespace, now) + + cancel() + + if err != nil { + renderWaiting(config, err) + + if config.once { + return err + } + + if err := waitForNext(ctx, config.refreshInterval); err != nil { + return err + } + + continue + } + + lastSnapshot = aggregateRange(response, session.phaseStart, now, session.expectedBytes, session.nodeCount) + lastSnapshot.RunID = session.runID + lastSnapshot.JobName = session.jobName + lastSnapshot.PhaseStart = session.phaseStart + lastSnapshot.Now = now + lastSnapshot.RefreshInterval = config.refreshInterval + lastSnapshot.Job = status + + if !config.noClear && term.IsTerminal(int(os.Stdout.Fd())) { + fmt.Print("\033[H\033[2J") + } + + fmt.Print(renderSnapshot(lastSnapshot)) + + if config.once || status.Complete { + return nil + } + + if err := waitForNext(ctx, config.refreshInterval); err != nil { + return err + } + } +} + +func waitForNext(ctx context.Context, interval time.Duration) error { + timer := time.NewTimer(interval) + defer timer.Stop() + + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + +func main() { + config, err := parseConfig(os.Args[1:]) + if err != nil { + if errors.Is(err, flag.ErrHelp) { + return + } + + fmt.Fprintln(os.Stderr, err) + os.Exit(2) + } + + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + + isTTY := term.IsTerminal(int(os.Stdout.Fd())) && !config.noClear + if isTTY { + fmt.Print("\033[?1049h") + defer fmt.Print("\033[?1049l") + } + + if err := runMonitor(ctx, config); err != nil && !errors.Is(err, context.Canceled) { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} diff --git a/hack/cmd/gantry-benchmark-monitor/monitor.go b/hack/cmd/gantry-benchmark-monitor/monitor.go new file mode 100644 index 000000000..a94e1a7cc --- /dev/null +++ b/hack/cmd/gantry-benchmark-monitor/monitor.go @@ -0,0 +1,334 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package main + +import ( + "encoding/json" + "fmt" + "math" + "sort" + "strconv" + "strings" + "time" +) + +type rangeSample struct { + Timestamp time.Time + Value float64 +} + +type rangeSeries struct { + Metric map[string]string + Samples []rangeSample +} + +type rangeResponse struct { + Series []rangeSeries +} + +type minuteBin struct { + Minute int + PeerOutcomes map[string]float64 + Bytes float64 +} + +type monitorSnapshot struct { + RunID string + JobName string + PhaseStart time.Time + Now time.Time + LatestSample time.Time + RefreshInterval time.Duration + NodeCount int + ExpectedBytes float64 + Bins []minuteBin + PeerTotals map[string]float64 + TotalBytes float64 + Job jobStatus +} + +func parseRangeResponse(raw []byte) (rangeResponse, error) { + var envelope struct { + Status string `json:"status"` + Data struct { + Result []struct { + Metric map[string]string `json:"metric"` + Values [][2]json.RawMessage `json:"values"` + } `json:"result"` + } `json:"data"` + } + if err := json.Unmarshal(raw, &envelope); err != nil { + return rangeResponse{}, fmt.Errorf("decode Prometheus range response: %w", err) + } + + if envelope.Status != "success" { + return rangeResponse{}, fmt.Errorf("prometheus range query status is %q", envelope.Status) + } + + response := rangeResponse{Series: make([]rangeSeries, 0, len(envelope.Data.Result))} + for _, rawSeries := range envelope.Data.Result { + series := rangeSeries{Metric: rawSeries.Metric, Samples: make([]rangeSample, 0, len(rawSeries.Values))} + for _, pair := range rawSeries.Values { + var timestamp float64 + if err := json.Unmarshal(pair[0], ×tamp); err != nil { + return rangeResponse{}, fmt.Errorf("decode Prometheus sample timestamp: %w", err) + } + + var text string + if err := json.Unmarshal(pair[1], &text); err != nil { + return rangeResponse{}, fmt.Errorf("decode Prometheus sample value: %w", err) + } + + value, err := strconv.ParseFloat(text, 64) + if err != nil { + return rangeResponse{}, fmt.Errorf("parse Prometheus sample value %q: %w", text, err) + } + + if math.IsNaN(value) || math.IsInf(value, 0) { + continue + } + + seconds, fraction := math.Modf(timestamp) + series.Samples = append(series.Samples, rangeSample{ + Timestamp: time.Unix(int64(seconds), int64(fraction*float64(time.Second))).UTC(), + Value: value, + }) + } + + response.Series = append(response.Series, series) + } + + return response, nil +} + +func aggregateRange(response rangeResponse, phaseStart, now time.Time, expectedBytes float64, nodeCount int) monitorSnapshot { + elapsed := now.Sub(phaseStart) + if elapsed < 0 { + elapsed = 0 + } + + binCount := int(elapsed/time.Minute) + 1 + + bins := make([]minuteBin, binCount) + for index := range bins { + bins[index] = minuteBin{Minute: index, PeerOutcomes: map[string]float64{}} + } + + totals := map[string]float64{} + latest := time.Time{} + + for _, series := range response.Series { + sort.Slice(series.Samples, func(i, j int) bool { + return series.Samples[i].Timestamp.Before(series.Samples[j].Timestamp) + }) + + if len(series.Samples) == 0 { + continue + } + + outcome := series.Metric["outcome"] + + previous := series.Samples[0].Value + if series.Samples[0].Timestamp.After(latest) { + latest = series.Samples[0].Timestamp + } + + for _, sample := range series.Samples[1:] { + if sample.Timestamp.After(latest) { + latest = sample.Timestamp + } + + delta := sample.Value - previous + + previous = sample.Value + if delta <= 0 || sample.Timestamp.Before(phaseStart) { + continue + } + + minute := int(sample.Timestamp.Sub(phaseStart) / time.Minute) + if minute < 0 || minute >= len(bins) { + continue + } + + if outcome == "" { + bins[minute].Bytes += delta + continue + } + + bins[minute].PeerOutcomes[outcome] += delta + totals[outcome] += delta + } + } + + totalBytes := 0.0 + for _, bin := range bins { + totalBytes += bin.Bytes + } + + return monitorSnapshot{ + LatestSample: latest, + NodeCount: nodeCount, + ExpectedBytes: expectedBytes, + Bins: bins, + PeerTotals: totals, + TotalBytes: totalBytes, + } +} + +func commaInteger(value float64) string { + text := strconv.FormatInt(int64(math.Round(value)), 10) + + start := 0 + if strings.HasPrefix(text, "-") { + start = 1 + } + + for index := len(text) - 3; index > start; index -= 3 { + text = text[:index] + "," + text[index:] + } + + return text +} + +func percentage(numerator, denominator float64) float64 { + if denominator <= 0 { + return 0 + } + + return numerator / denominator * 100 +} + +func binDuration(snapshot monitorSnapshot, minute int) float64 { + if minute < len(snapshot.Bins)-1 { + return 60 + } + + seconds := snapshot.Now.Sub(snapshot.PhaseStart.Add(time.Duration(minute) * time.Minute)).Seconds() + if seconds < 1 { + return 1 + } + + if seconds > 60 { + return 60 + } + + return seconds +} + +func renderPeerTable(builder *strings.Builder, snapshot monitorSnapshot) { + fmt.Fprintln(builder, "=== Peer fetch outcomes by phase minute ===") + fmt.Fprintf(builder, "%4s %12s %10s %10s %10s %12s\n", "min", "busy", "hit", "stall", "notfound", "unavailable") + + for _, bin := range snapshot.Bins { + minute := strconv.Itoa(bin.Minute) + if bin.Minute == len(snapshot.Bins)-1 { + minute += "*" + } + + fmt.Fprintf(builder, "%4s %12s %10s %10s %10s %12s\n", + minute, + commaInteger(bin.PeerOutcomes["busy"]), + commaInteger(bin.PeerOutcomes["hit"]), + commaInteger(bin.PeerOutcomes["stall"]), + commaInteger(bin.PeerOutcomes["notfound"]), + commaInteger(bin.PeerOutcomes["unavailable"]), + ) + } + + fmt.Fprintf(builder, "\nTOTAL %10s %10s %10s %10s %12s\n", + commaInteger(snapshot.PeerTotals["busy"]), + commaInteger(snapshot.PeerTotals["hit"]), + commaInteger(snapshot.PeerTotals["stall"]), + commaInteger(snapshot.PeerTotals["notfound"]), + commaInteger(snapshot.PeerTotals["unavailable"]), + ) + + firstSixBusy := 0.0 + firstSixHit := 0.0 + + for _, bin := range snapshot.Bins { + if bin.Minute >= 6 { + break + } + + firstSixBusy += bin.PeerOutcomes["busy"] + firstSixHit += bin.PeerOutcomes["hit"] + } + + if snapshot.PeerTotals["busy"] > 0 { + fmt.Fprintf(builder, "\nbusy in first 6 min: %s of %s = %.1f%%\n", + commaInteger(firstSixBusy), commaInteger(snapshot.PeerTotals["busy"]), percentage(firstSixBusy, snapshot.PeerTotals["busy"])) + } + + if snapshot.PeerTotals["hit"] > 0 { + fmt.Fprintf(builder, "hit in first 6 min: %s of %s = %.1f%%\n", + commaInteger(firstSixHit), commaInteger(snapshot.PeerTotals["hit"]), percentage(firstSixHit, snapshot.PeerTotals["hit"])) + } +} + +func renderByteTable(builder *strings.Builder, snapshot monitorSnapshot) { + fmt.Fprintln(builder, "=== Layer bytes delivered by phase minute ===") + fmt.Fprintf(builder, "%4s %12s %16s %14s %8s\n", "min", "GB moved", "GB/s all-nodes", "MB/s per node", "cum %") + + cumulative := 0.0 + for _, bin := range snapshot.Bins { + cumulative += bin.Bytes + seconds := binDuration(snapshot, bin.Minute) + allNodesGBs := bin.Bytes / seconds / 1e9 + + perNodeMBs := 0.0 + if snapshot.NodeCount > 0 { + perNodeMBs = bin.Bytes / seconds / float64(snapshot.NodeCount) / 1e6 + } + + minute := strconv.Itoa(bin.Minute) + if bin.Minute == len(snapshot.Bins)-1 { + minute += "*" + } + + fmt.Fprintf(builder, "%4s %12s %16.1f %14.1f %7.1f%%\n", + minute, + commaInteger(bin.Bytes/1e9), + allNodesGBs, + perNodeMBs, + percentage(cumulative, snapshot.ExpectedBytes), + ) + } + + fmt.Fprintf(builder, "\ntotal %.3f TB of %.3f TB (%.1f%%)\n", + snapshot.TotalBytes/1e12, + snapshot.ExpectedBytes/1e12, + percentage(snapshot.TotalBytes, snapshot.ExpectedBytes), + ) +} + +func renderSnapshot(snapshot monitorSnapshot) string { + var builder strings.Builder + + elapsed := snapshot.Now.Sub(snapshot.PhaseStart) + if elapsed < 0 { + elapsed = 0 + } + + fmt.Fprintln(&builder, "Gantry benchmark live monitor") + fmt.Fprintf(&builder, "time: %s\n", snapshot.Now.UTC().Format(time.RFC3339)) + fmt.Fprintf(&builder, "run: %s\n", snapshot.RunID) + fmt.Fprintf(&builder, "job: %s\n", snapshot.JobName) + fmt.Fprintf(&builder, "phase started: %s (elapsed %s)\n", snapshot.PhaseStart.UTC().Format(time.RFC3339), elapsed.Round(time.Second)) + fmt.Fprintf(&builder, "pods: %d/%d succeeded, %d active, %d failed\n", snapshot.Job.Succeeded, snapshot.NodeCount, snapshot.Job.Active, snapshot.Job.Failed) + fmt.Fprintf(&builder, "display refresh: %s; Prometheus scrape cadence: 10s (values repeat between scrapes)\n", snapshot.RefreshInterval) + + if !snapshot.LatestSample.IsZero() { + fmt.Fprintf(&builder, "latest query sample: %s\n", snapshot.LatestSample.UTC().Format(time.RFC3339)) + } + + fmt.Fprintln(&builder, "*: current partial minute") + fmt.Fprintln(&builder) + + renderPeerTable(&builder, snapshot) + fmt.Fprintln(&builder) + renderByteTable(&builder, snapshot) + + return builder.String() +} diff --git a/hack/cmd/gantry-benchmark-monitor/monitor_test.go b/hack/cmd/gantry-benchmark-monitor/monitor_test.go new file mode 100644 index 000000000..bbd330824 --- /dev/null +++ b/hack/cmd/gantry-benchmark-monitor/monitor_test.go @@ -0,0 +1,146 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package main + +import ( + "fmt" + "strings" + "testing" + "time" +) + +func prometheusFixture(start time.Time) []byte { + return []byte(fmt.Sprintf(`{ + "status":"success", + "data":{"resultType":"matrix","result":[ + {"metric":{"outcome":"busy"},"values":[[%f,"100"],[%f,"110"],[%f,"130"],[%f,"135"]]}, + {"metric":{"outcome":"hit"},"values":[[%f,"20"],[%f,"22"],[%f,"25"]]}, + {"metric":{},"values":[[%f,"1000"],[%f,"60000001000"],[%f,"120000001000"],[%f,"150000001000"]]} + ]} +}`, + float64(start.Add(-10*time.Second).Unix()), + float64(start.Add(10*time.Second).Unix()), + float64(start.Add(70*time.Second).Unix()), + float64(start.Add(130*time.Second).Unix()), + float64(start.Add(-10*time.Second).Unix()), + float64(start.Add(30*time.Second).Unix()), + float64(start.Add(90*time.Second).Unix()), + float64(start.Add(-10*time.Second).Unix()), + float64(start.Add(59*time.Second).Unix()), + float64(start.Add(119*time.Second).Unix()), + float64(start.Add(130*time.Second).Unix()), + )) +} + +func TestParseAndAggregateRange(t *testing.T) { + start := time.Date(2026, 8, 7, 1, 0, 0, 0, time.UTC) + + response, err := parseRangeResponse(prometheusFixture(start)) + if err != nil { + t.Fatalf("parseRangeResponse: %v", err) + } + + snapshot := aggregateRange(response, start, start.Add(150*time.Second), 200e9, 1000) + if len(snapshot.Bins) != 3 { + t.Fatalf("bins = %d, want 3", len(snapshot.Bins)) + } + + checks := []struct { + minute int + busy float64 + hit float64 + bytes float64 + }{ + {minute: 0, busy: 10, hit: 2, bytes: 60e9}, + {minute: 1, busy: 20, hit: 3, bytes: 60e9}, + {minute: 2, busy: 5, hit: 0, bytes: 30e9}, + } + for _, check := range checks { + bin := snapshot.Bins[check.minute] + if bin.PeerOutcomes["busy"] != check.busy || bin.PeerOutcomes["hit"] != check.hit || bin.Bytes != check.bytes { + t.Errorf("minute %d = busy %.0f hit %.0f bytes %.0f; want %.0f %.0f %.0f", + check.minute, bin.PeerOutcomes["busy"], bin.PeerOutcomes["hit"], bin.Bytes, check.busy, check.hit, check.bytes) + } + } + + if snapshot.PeerTotals["busy"] != 35 || snapshot.PeerTotals["hit"] != 5 { + t.Fatalf("peer totals = %#v", snapshot.PeerTotals) + } + + if snapshot.TotalBytes != 150e9 { + t.Fatalf("total bytes = %.0f, want %.0f", snapshot.TotalBytes, 150e9) + } +} + +func TestRenderSnapshotIncludesBothLiveTables(t *testing.T) { + start := time.Date(2026, 8, 7, 1, 0, 0, 0, time.UTC) + + response, err := parseRangeResponse(prometheusFixture(start)) + if err != nil { + t.Fatal(err) + } + + snapshot := aggregateRange(response, start, start.Add(150*time.Second), 200e9, 1000) + snapshot.RunID = "run-1" + snapshot.JobName = "gantry-benchmark-gantry-cold-run-1" + snapshot.PhaseStart = start + snapshot.Now = start.Add(150 * time.Second) + snapshot.RefreshInterval = time.Second + snapshot.Job = jobStatus{Succeeded: 12, Active: 988} + + output := renderSnapshot(snapshot) + for _, want := range []string{ + "=== Peer fetch outcomes by phase minute ===", + "busy", + "notfound", + "TOTAL", + "busy in first 6 min: 35 of 35 = 100.0%", + "=== Layer bytes delivered by phase minute ===", + "MB/s per node", + "2*", + "total 0.150 TB of 0.200 TB (75.0%)", + "pods: 12/1000 succeeded, 988 active, 0 failed", + "Prometheus scrape cadence: 10s", + } { + if !strings.Contains(output, want) { + t.Errorf("output is missing %q:\n%s", want, output) + } + } +} + +func TestPrometheusExpressionScopesCurrentRevision(t *testing.T) { + expression := prometheusExpression(monitorSession{revision: "gantry-abc123"}, "gantry-system") + for _, want := range []string{ + `p2p_peer_fetch_total`, + `gantry_mirror_bytes_served_total`, + `namespace="gantry-system"`, + `controller_revision_hash="gantry-abc123"`, + `kind="layer"`, + ` or `, + } { + if !strings.Contains(expression, want) { + t.Errorf("expression %q is missing %q", expression, want) + } + } +} + +func TestCommaInteger(t *testing.T) { + for value, want := range map[float64]string{ + 0: "0", + 999: "999", + 1000: "1,000", + 1162435: "1,162,435", + -1234567: "-1,234,567", + } { + if got := commaInteger(value); got != want { + t.Errorf("commaInteger(%.0f) = %q, want %q", value, got, want) + } + } +} + +func TestParseRangeResponseRejectsFailure(t *testing.T) { + if _, err := parseRangeResponse([]byte(`{"status":"error"}`)); err == nil { + t.Fatal("parseRangeResponse succeeded for an error response") + } +} diff --git a/hack/gantry-benchmark/Makefile b/hack/gantry-benchmark/Makefile index 2ee15ad28..a029b025e 100644 --- a/hack/gantry-benchmark/Makefile +++ b/hack/gantry-benchmark/Makefile @@ -2,13 +2,14 @@ REPO_ROOT := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))/../..) ENV_FILE ?= $(CURDIR)/env.local DEPLOY_CONFIG ?= $(CURDIR)/deploy.env -.PHONY: help test deploy deploy-plan deploy-status operator-vm-check operator-vm-provision operator-vm-status operator-vm-watch proxy-image proxy-push enable prepare prepare-adopt prepare-gantry prepare-gantry-fresh prepare-gantry-adopt preflight run run-gantry status disable +.PHONY: help test monitor deploy deploy-plan deploy-status operator-vm-check operator-vm-provision operator-vm-status operator-vm-watch proxy-image proxy-push enable prepare prepare-adopt prepare-gantry prepare-gantry-fresh prepare-gantry-adopt preflight run run-gantry status disable help: ## Show benchmark targets @echo "" @echo "Usage: make -C hack/gantry-benchmark [ENV_FILE=path]" @echo "" @echo " test Run focused proxy and benchmark tests" + @echo " monitor Live peer-outcome and layer-byte tables for the active Gantry phase" @echo " deploy Idempotently deploy the complete benchmark stack" @echo " deploy-plan Print the resolved deployment contract without mutation" @echo " deploy-status Report deployment readiness without mutation" @@ -32,7 +33,14 @@ help: ## Show benchmark targets test: $(MAKE) operator-vm-check - cd "$(REPO_ROOT)" && GOTOOLCHAIN=auto go test ./hack/cmd/acr-origin-proxy ./hack/cmd/gantry-benchmark + cd "$(REPO_ROOT)" && GOTOOLCHAIN=auto go test ./hack/cmd/acr-origin-proxy ./hack/cmd/gantry-benchmark ./hack/cmd/gantry-benchmark-monitor + +monitor: + set -a; \ + . "$(DEPLOY_CONFIG)"; \ + set +a; \ + export KUBECONFIG="$${DEPLOY_KUBECONFIG:-$(REPO_ROOT)/tmp/$${DEPLOYMENT_NAME}/kubeconfig}"; \ + cd "$(REPO_ROOT)" && GOTOOLCHAIN=auto go run ./hack/cmd/gantry-benchmark-monitor $(MONITOR_ARGS) operator-vm-check: bash -n deploy.sh operator-vm-provision.sh operator-vm-bootstrap.sh operator-vm-build-images.sh operator-vm-run.sh operator-vm-status.sh operator-vm-watch.sh diff --git a/hack/gantry-benchmark/README.md b/hack/gantry-benchmark/README.md index 19ea128db..6efbdea1b 100644 --- a/hack/gantry-benchmark/README.md +++ b/hack/gantry-benchmark/README.md @@ -215,6 +215,20 @@ machine-readable live push byte percentage, so the view reports that limitation instead of estimating it. Use `operator-vm-status` for a single snapshot. Override the refresh cadence with `WATCH_INTERVAL_SECONDS` (default 30). +For the Gantry pull phase, use the dedicated live monitor from the workstation: + +```bash +make -C hack/gantry-benchmark monitor +``` + +It redraws every second and shows per-phase-minute peer outcomes (`busy`, +`hit`, `stall`, `notfound`, `unavailable`) alongside layer bytes, aggregate +and per-node throughput, and cumulative payload percentage. The monitor uses +one server-side aggregated Prometheus range query per refresh; it does not +download per-pod series. Prometheus scrapes Gantry every 10 seconds, so the +screen updates each second while counter values advance at scrape cadence. +Use `MONITOR_ARGS="--once --no-clear"` for a single non-interactive snapshot. + Artifacts persist on the VM under `/var/lib/gantry-benchmark/artifacts//`; `latest` points at the newest run. By default the operator is a `Standard_D32ds_v5` with a dedicated 512 GiB From e8b41acd44edcbeaac1c333977196e08d7c4f1f7 Mon Sep 17 00:00:00 2001 From: Vaibhav Patel Date: Thu, 6 Aug 2026 22:45:48 -0400 Subject: [PATCH 44/60] feat(gantry-benchmark): show live pod states in monitor Add a Kubernetes Pod tracker to the standalone live monitor. It performs one initial list for the active Gantry-cold Job, then maintains local state from watch events instead of polling 1000 Pod objects each second. The watch relists and reconnects if the API stream closes. Render completed, running, creating, image-pull, and failed counts on every one-second refresh. Show unscheduled and unclassified waiting states only when nonzero, and surface transient watch errors without stopping Prometheus monitoring. Highlight the header on a TTY with a bold cyan title, dim run metadata, and a bold Pod status line. Redirected output remains plain, and NO_COLOR disables styling. --no-clear still permits color because it controls redraw behavior rather than output styling. Tests cover every Pod phase/waiting classification, count aggregation, watch-error propagation, plain rendering, and ANSI header rendering. Validate against the live run at completion: pods: 1000/1000 completed | 0 running | 0 creating | 0 image-pull | 0 failed --- hack/cmd/gantry-benchmark-monitor/main.go | 30 ++ hack/cmd/gantry-benchmark-monitor/monitor.go | 41 ++- .../gantry-benchmark-monitor/monitor_test.go | 32 ++- hack/cmd/gantry-benchmark-monitor/pods.go | 265 ++++++++++++++++++ .../cmd/gantry-benchmark-monitor/pods_test.go | 85 ++++++ hack/gantry-benchmark/README.md | 12 +- 6 files changed, 454 insertions(+), 11 deletions(-) create mode 100644 hack/cmd/gantry-benchmark-monitor/pods.go create mode 100644 hack/cmd/gantry-benchmark-monitor/pods_test.go diff --git a/hack/cmd/gantry-benchmark-monitor/main.go b/hack/cmd/gantry-benchmark-monitor/main.go index 157e31122..63ce64276 100644 --- a/hack/cmd/gantry-benchmark-monitor/main.go +++ b/hack/cmd/gantry-benchmark-monitor/main.go @@ -348,6 +348,7 @@ func runMonitor(ctx context.Context, config monitorConfig) error { var ( session *monitorSession + podTracker *podStateTracker lastSnapshot monitorSnapshot ) @@ -374,6 +375,25 @@ func runMonitor(ctx context.Context, config monitorConfig) error { session = &discovered status = job + + tracker, err := newPodStateTracker(ctx, config.kubeconfig, config.benchmarkNamespace, session.jobName) + if err != nil { + session = nil + + renderWaiting(config, err) + + if config.once { + return err + } + + if err := waitForNext(ctx, config.refreshInterval); err != nil { + return err + } + + continue + } + + podTracker = tracker } else { job, err := refreshJobStatus(ctx, runner, config, *session) if err != nil { @@ -415,6 +435,16 @@ func runMonitor(ctx context.Context, config monitorConfig) error { lastSnapshot.Now = now lastSnapshot.RefreshInterval = config.refreshInterval lastSnapshot.Job = status + lastSnapshot.Color = term.IsTerminal(int(os.Stdout.Fd())) && os.Getenv("NO_COLOR") == "" + + if podTracker != nil { + counts, podErr := podTracker.snapshot() + + lastSnapshot.PodStates = counts + if podErr != nil { + lastSnapshot.PodStateError = podErr.Error() + } + } if !config.noClear && term.IsTerminal(int(os.Stdout.Fd())) { fmt.Print("\033[H\033[2J") diff --git a/hack/cmd/gantry-benchmark-monitor/monitor.go b/hack/cmd/gantry-benchmark-monitor/monitor.go index a94e1a7cc..653a6cd9c 100644 --- a/hack/cmd/gantry-benchmark-monitor/monitor.go +++ b/hack/cmd/gantry-benchmark-monitor/monitor.go @@ -46,6 +46,9 @@ type monitorSnapshot struct { PeerTotals map[string]float64 TotalBytes float64 Job jobStatus + PodStates podStateCounts + PodStateError string + Color bool } func parseRangeResponse(raw []byte) (rangeResponse, error) { @@ -311,19 +314,45 @@ func renderSnapshot(snapshot monitorSnapshot) string { elapsed = 0 } - fmt.Fprintln(&builder, "Gantry benchmark live monitor") - fmt.Fprintf(&builder, "time: %s\n", snapshot.Now.UTC().Format(time.RFC3339)) + titleStart, metaStart, statusStart, reset := "", "", "", "" + if snapshot.Color { + titleStart = "\033[1;36m" + metaStart = "\033[2m" + statusStart = "\033[1m" + reset = "\033[0m" + } + + fmt.Fprintf(&builder, "%sGantry benchmark live monitor%s\n", titleStart, reset) + fmt.Fprintf(&builder, "%stime: %s\n", metaStart, snapshot.Now.UTC().Format(time.RFC3339)) fmt.Fprintf(&builder, "run: %s\n", snapshot.RunID) fmt.Fprintf(&builder, "job: %s\n", snapshot.JobName) - fmt.Fprintf(&builder, "phase started: %s (elapsed %s)\n", snapshot.PhaseStart.UTC().Format(time.RFC3339), elapsed.Round(time.Second)) - fmt.Fprintf(&builder, "pods: %d/%d succeeded, %d active, %d failed\n", snapshot.Job.Succeeded, snapshot.NodeCount, snapshot.Job.Active, snapshot.Job.Failed) - fmt.Fprintf(&builder, "display refresh: %s; Prometheus scrape cadence: 10s (values repeat between scrapes)\n", snapshot.RefreshInterval) + fmt.Fprintf(&builder, "phase started: %s (elapsed %s)%s\n", snapshot.PhaseStart.UTC().Format(time.RFC3339), elapsed.Round(time.Second), reset) + fmt.Fprintf(&builder, "%spods: %d/%d completed | %d running | %d creating | %d image-pull | %d failed%s\n", + statusStart, + snapshot.PodStates.Completed, + snapshot.NodeCount, + snapshot.PodStates.Running, + snapshot.PodStates.Creating, + snapshot.PodStates.ImagePull, + snapshot.PodStates.Failed, + reset, + ) + + if snapshot.PodStates.Unscheduled > 0 || snapshot.PodStates.Other > 0 { + fmt.Fprintf(&builder, "%spod detail: %d unscheduled, %d other%s\n", metaStart, snapshot.PodStates.Unscheduled, snapshot.PodStates.Other, reset) + } + + if snapshot.PodStateError != "" { + fmt.Fprintf(&builder, "%spod watch: %s%s\n", metaStart, snapshot.PodStateError, reset) + } + + fmt.Fprintf(&builder, "%sdisplay refresh: %s; Prometheus scrape cadence: 10s (values repeat between scrapes)\n", metaStart, snapshot.RefreshInterval) if !snapshot.LatestSample.IsZero() { fmt.Fprintf(&builder, "latest query sample: %s\n", snapshot.LatestSample.UTC().Format(time.RFC3339)) } - fmt.Fprintln(&builder, "*: current partial minute") + fmt.Fprintf(&builder, "*: current partial minute%s\n", reset) fmt.Fprintln(&builder) renderPeerTable(&builder, snapshot) diff --git a/hack/cmd/gantry-benchmark-monitor/monitor_test.go b/hack/cmd/gantry-benchmark-monitor/monitor_test.go index bbd330824..3336cb21a 100644 --- a/hack/cmd/gantry-benchmark-monitor/monitor_test.go +++ b/hack/cmd/gantry-benchmark-monitor/monitor_test.go @@ -88,6 +88,7 @@ func TestRenderSnapshotIncludesBothLiveTables(t *testing.T) { snapshot.Now = start.Add(150 * time.Second) snapshot.RefreshInterval = time.Second snapshot.Job = jobStatus{Succeeded: 12, Active: 988} + snapshot.PodStates = podStateCounts{Completed: 12, Running: 20, Creating: 960, ImagePull: 8} output := renderSnapshot(snapshot) for _, want := range []string{ @@ -100,7 +101,7 @@ func TestRenderSnapshotIncludesBothLiveTables(t *testing.T) { "MB/s per node", "2*", "total 0.150 TB of 0.200 TB (75.0%)", - "pods: 12/1000 succeeded, 988 active, 0 failed", + "pods: 12/1000 completed | 20 running | 960 creating | 8 image-pull | 0 failed", "Prometheus scrape cadence: 10s", } { if !strings.Contains(output, want) { @@ -109,6 +110,35 @@ func TestRenderSnapshotIncludesBothLiveTables(t *testing.T) { } } +func TestRenderSnapshotHighlightsHeaderWhenColorEnabled(t *testing.T) { + t.Parallel() + + start := time.Date(2026, 8, 7, 1, 0, 0, 0, time.UTC) + snapshot := monitorSnapshot{ + RunID: "run-1", + JobName: "job-1", + PhaseStart: start, + Now: start.Add(time.Minute), + RefreshInterval: time.Second, + NodeCount: 1000, + Bins: []minuteBin{{Minute: 0, PeerOutcomes: map[string]float64{}}}, + PeerTotals: map[string]float64{}, + PodStates: podStateCounts{Completed: 1, Running: 2, Creating: 997}, + Color: true, + } + + output := renderSnapshot(snapshot) + for _, want := range []string{ + "\033[1;36mGantry benchmark live monitor\033[0m", + "\033[2mtime:", + "\033[1mpods: 1/1000 completed | 2 running | 997 creating", + } { + if !strings.Contains(output, want) { + t.Errorf("colored output is missing %q:\n%q", want, output) + } + } +} + func TestPrometheusExpressionScopesCurrentRevision(t *testing.T) { expression := prometheusExpression(monitorSession{revision: "gantry-abc123"}, "gantry-system") for _, want := range []string{ diff --git a/hack/cmd/gantry-benchmark-monitor/pods.go b/hack/cmd/gantry-benchmark-monitor/pods.go new file mode 100644 index 000000000..e42f794af --- /dev/null +++ b/hack/cmd/gantry-benchmark-monitor/pods.go @@ -0,0 +1,265 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package main + +import ( + "context" + "fmt" + "sync" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" +) + +type podState string + +const ( + podStateCreating podState = "creating" + podStateImagePull podState = "image-pull" + podStateRunning podState = "running" + podStateCompleted podState = "completed" + podStateFailed podState = "failed" + podStateUnscheduled podState = "unscheduled" + podStateOther podState = "other" +) + +type podStateCounts struct { + Creating int + ImagePull int + Running int + Completed int + Failed int + Unscheduled int + Other int +} + +type podStateTracker struct { + mu sync.RWMutex + states map[string]podState + err error +} + +func loadKubeConfig(path string) (clientcmd.ClientConfig, error) { + rules := clientcmd.NewDefaultClientConfigLoadingRules() + if path != "" { + rules.ExplicitPath = path + } + + return clientcmd.NewNonInteractiveDeferredLoadingClientConfig(rules, &clientcmd.ConfigOverrides{}), nil +} + +func newPodStateTracker(ctx context.Context, kubeconfig, namespace, jobName string) (*podStateTracker, error) { + clientConfig, err := loadKubeConfig(kubeconfig) + if err != nil { + return nil, err + } + + restConfig, err := clientConfig.ClientConfig() + if err != nil { + return nil, fmt.Errorf("load Kubernetes client config: %w", err) + } + + client, err := kubernetes.NewForConfig(restConfig) + if err != nil { + return nil, fmt.Errorf("create Kubernetes client: %w", err) + } + + tracker := &podStateTracker{states: map[string]podState{}} + if err := tracker.replaceFromList(ctx, client, namespace, jobName); err != nil { + return nil, err + } + + go tracker.run(ctx, client, namespace, jobName) + + return tracker, nil +} + +func (t *podStateTracker) replaceFromList(ctx context.Context, client kubernetes.Interface, namespace, jobName string) error { + list, err := client.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{LabelSelector: "job-name=" + jobName}) + if err != nil { + return fmt.Errorf("list pull Job pods: %w", err) + } + + states := make(map[string]podState, len(list.Items)) + for index := range list.Items { + pod := &list.Items[index] + states[string(pod.UID)] = classifyPod(pod) + } + + t.mu.Lock() + t.states = states + t.err = nil + t.mu.Unlock() + + return nil +} + +func (t *podStateTracker) run(ctx context.Context, client kubernetes.Interface, namespace, jobName string) { + for ctx.Err() == nil { + list, err := client.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{LabelSelector: "job-name=" + jobName}) + if err != nil { + t.setError(err) + + if !waitForRetry(ctx) { + return + } + + continue + } + + states := make(map[string]podState, len(list.Items)) + for index := range list.Items { + pod := &list.Items[index] + states[string(pod.UID)] = classifyPod(pod) + } + + t.replace(states) + + watcher, err := client.CoreV1().Pods(namespace).Watch(ctx, metav1.ListOptions{ + LabelSelector: "job-name=" + jobName, + ResourceVersion: list.ResourceVersion, + AllowWatchBookmarks: true, + }) + if err != nil { + t.setError(err) + + if !waitForRetry(ctx) { + return + } + + continue + } + + closed := t.consumeWatch(ctx, watcher) + watcher.Stop() + + if !closed { + return + } + } +} + +func (t *podStateTracker) consumeWatch(ctx context.Context, watcher watch.Interface) bool { + for { + select { + case <-ctx.Done(): + return false + case event, ok := <-watcher.ResultChan(): + if !ok { + return true + } + + pod, ok := event.Object.(*corev1.Pod) + if !ok { + if event.Type == watch.Error { + t.setError(fmt.Errorf("pod watch returned an error event")) + return true + } + + continue + } + + key := string(pod.UID) + + t.mu.Lock() + if event.Type == watch.Deleted { + delete(t.states, key) + } else { + t.states[key] = classifyPod(pod) + } + + t.err = nil + t.mu.Unlock() + } + } +} + +func waitForRetry(ctx context.Context) bool { + timer := time.NewTimer(time.Second) + defer timer.Stop() + + select { + case <-ctx.Done(): + return false + case <-timer.C: + return true + } +} + +func (t *podStateTracker) replace(states map[string]podState) { + t.mu.Lock() + t.states = states + t.err = nil + t.mu.Unlock() +} + +func (t *podStateTracker) setError(err error) { + t.mu.Lock() + t.err = err + t.mu.Unlock() +} + +func (t *podStateTracker) snapshot() (podStateCounts, error) { + t.mu.RLock() + defer t.mu.RUnlock() + + counts := podStateCounts{} + + for _, state := range t.states { + switch state { + case podStateCreating: + counts.Creating++ + case podStateImagePull: + counts.ImagePull++ + case podStateRunning: + counts.Running++ + case podStateCompleted: + counts.Completed++ + case podStateFailed: + counts.Failed++ + case podStateUnscheduled: + counts.Unscheduled++ + case podStateOther: + counts.Other++ + } + } + + return counts, t.err +} + +func classifyPod(pod *corev1.Pod) podState { + switch pod.Status.Phase { + case corev1.PodSucceeded: + return podStateCompleted + case corev1.PodFailed: + return podStateFailed + case corev1.PodRunning: + return podStateRunning + } + + for _, status := range pod.Status.ContainerStatuses { + if status.Name != "pull" || status.State.Waiting == nil { + continue + } + + switch status.State.Waiting.Reason { + case "ImagePullBackOff", "ErrImagePull", "RegistryUnavailable": + return podStateImagePull + case "ContainerCreating", "PodInitializing": + return podStateCreating + default: + return podStateOther + } + } + + if pod.Spec.NodeName == "" { + return podStateUnscheduled + } + + return podStateCreating +} diff --git a/hack/cmd/gantry-benchmark-monitor/pods_test.go b/hack/cmd/gantry-benchmark-monitor/pods_test.go new file mode 100644 index 000000000..eacaf0e23 --- /dev/null +++ b/hack/cmd/gantry-benchmark-monitor/pods_test.go @@ -0,0 +1,85 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package main + +import ( + "errors" + "testing" + + corev1 "k8s.io/api/core/v1" +) + +func waitingPod(reason, node string) *corev1.Pod { + return &corev1.Pod{ + Spec: corev1.PodSpec{NodeName: node}, + Status: corev1.PodStatus{ + Phase: corev1.PodPending, + ContainerStatuses: []corev1.ContainerStatus{{ + Name: "pull", + State: corev1.ContainerState{ + Waiting: &corev1.ContainerStateWaiting{Reason: reason}, + }, + }}, + }, + } +} + +func TestClassifyPod(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + pod *corev1.Pod + want podState + }{ + {name: "completed", pod: &corev1.Pod{Status: corev1.PodStatus{Phase: corev1.PodSucceeded}}, want: podStateCompleted}, + {name: "failed", pod: &corev1.Pod{Status: corev1.PodStatus{Phase: corev1.PodFailed}}, want: podStateFailed}, + {name: "running", pod: &corev1.Pod{Status: corev1.PodStatus{Phase: corev1.PodRunning}}, want: podStateRunning}, + {name: "creating", pod: waitingPod("ContainerCreating", "node-a"), want: podStateCreating}, + {name: "initializing", pod: waitingPod("PodInitializing", "node-a"), want: podStateCreating}, + {name: "pull backoff", pod: waitingPod("ImagePullBackOff", "node-a"), want: podStateImagePull}, + {name: "pull error", pod: waitingPod("ErrImagePull", "node-a"), want: podStateImagePull}, + {name: "registry unavailable", pod: waitingPod("RegistryUnavailable", "node-a"), want: podStateImagePull}, + {name: "other waiting", pod: waitingPod("CreateContainerConfigError", "node-a"), want: podStateOther}, + {name: "unscheduled", pod: &corev1.Pod{Status: corev1.PodStatus{Phase: corev1.PodPending}}, want: podStateUnscheduled}, + {name: "scheduled pulling", pod: &corev1.Pod{Spec: corev1.PodSpec{NodeName: "node-a"}, Status: corev1.PodStatus{Phase: corev1.PodPending}}, want: podStateCreating}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + if got := classifyPod(test.pod); got != test.want { + t.Fatalf("classifyPod() = %q, want %q", got, test.want) + } + }) + } +} + +func TestPodStateTrackerSnapshot(t *testing.T) { + t.Parallel() + + tracker := &podStateTracker{ + states: map[string]podState{ + "a": podStateCompleted, + "b": podStateRunning, + "c": podStateCreating, + "d": podStateImagePull, + "e": podStateFailed, + "f": podStateUnscheduled, + "g": podStateOther, + }, + err: errors.New("watch reconnecting"), + } + + counts, err := tracker.snapshot() + if err == nil || err.Error() != "watch reconnecting" { + t.Fatalf("snapshot error = %v", err) + } + + want := podStateCounts{Completed: 1, Running: 1, Creating: 1, ImagePull: 1, Failed: 1, Unscheduled: 1, Other: 1} + if counts != want { + t.Fatalf("counts = %#v, want %#v", counts, want) + } +} diff --git a/hack/gantry-benchmark/README.md b/hack/gantry-benchmark/README.md index 6efbdea1b..e4c740024 100644 --- a/hack/gantry-benchmark/README.md +++ b/hack/gantry-benchmark/README.md @@ -223,10 +223,14 @@ make -C hack/gantry-benchmark monitor It redraws every second and shows per-phase-minute peer outcomes (`busy`, `hit`, `stall`, `notfound`, `unavailable`) alongside layer bytes, aggregate -and per-node throughput, and cumulative payload percentage. The monitor uses -one server-side aggregated Prometheus range query per refresh; it does not -download per-pod series. Prometheus scrapes Gantry every 10 seconds, so the -screen updates each second while counter values advance at scrape cadence. +and per-node throughput, cumulative payload percentage, and live Pod counts for +completed, running, creating, image-pull failures, and failed Pods. Pod counts +come from one Kubernetes list followed by watch events rather than polling all +1000 Pod objects. The header uses ANSI emphasis on a TTY and remains plain when +redirected or piped. The monitor uses one server-side aggregated Prometheus +range query per refresh; it does not download per-pod metric series. Prometheus +scrapes Gantry every 10 seconds, so the screen updates each second while counter +values advance at scrape cadence. Use `MONITOR_ARGS="--once --no-clear"` for a single non-interactive snapshot. Artifacts persist on the VM under From f8a520d82947b4617f9281379eaf49d62b9bfd1d Mon Sep 17 00:00:00 2001 From: Vaibhav Patel Date: Thu, 6 Aug 2026 22:52:38 -0400 Subject: [PATCH 45/60] fix(gantry): raise peer fetch ceiling to 15 minutes PeerFetchTimeout is an absolute request deadline, not a no-progress timeout. With the previous 60s value, progressing 1 GiB transfers were classified as stalls after delivering a median 747 MB; sampled P95 was 1.022 GB and some attempts timed out only about 1.6 MB before completion. The run recorded 30829 such events at a mean 60.0007s. Raise the complete peer-request safety ceiling to 15m in defaults and the production deployment. The HTTP/2 transport retains its separate 10s read-idle health probe, so dead connections are detected independently of the total transfer duration. Live stream-through still preserves and resumes a verified prefix if the ceiling is reached. Update the latency analysis to identify 60s stalls as historical behavior and document the new default. --- deploy/gantry/configmap.yaml.tmpl | 11 +++----- .../gantry-benchmark/PULL-LATENCY-ANALYSIS.md | 25 +++++++++---------- internal/gantry/config/config.go | 17 +++++-------- internal/gantry/config/config_test.go | 4 +-- 4 files changed, 24 insertions(+), 33 deletions(-) diff --git a/deploy/gantry/configmap.yaml.tmpl b/deploy/gantry/configmap.yaml.tmpl index 0723d1553..51fd92985 100644 --- a/deploy/gantry/configmap.yaml.tmpl +++ b/deploy/gantry/configmap.yaml.tmpl @@ -159,13 +159,10 @@ data: # one node. Each pull holds an HTTP body, containerd writer, # goroutine, and lease. coord_max_concurrent_pulls: 16 - # Deliberately SHORT. A requester stuck on a lockstep-saturated seed must - # bail at this deadline and re-select a fresher finisher-seed rather than - # ride the slow stream to completion; that bail-and-re-select (with the - # strict containerd hosts.toml) drives the cold-start cascade. Too high - # (e.g. 1h) removes re-selection and collapses to single-seed bandwidth - # (the ~12min lockstep). Matches upstream gantry's hardcoded 60s. - peer_fetch_timeout: "60s" + # Safety ceiling for a complete peer request. Progressing large layers must + # not switch providers only because their total transfer exceeds 60s. The + # HTTP/2 transport separately probes connections after 10s without reads. + peer_fetch_timeout: "15m" # Peer re-discovery loop. On a cache miss the mirror repeatedly re-runs DHT # FindProviders and retries peer fetches for up to peer_rediscover_budget to # pick up finisher-seeds that advertise mid-swarm (the cold-start cascade), diff --git a/hack/gantry-benchmark/PULL-LATENCY-ANALYSIS.md b/hack/gantry-benchmark/PULL-LATENCY-ANALYSIS.md index 774eb94bb..9d4c7e35c 100644 --- a/hack/gantry-benchmark/PULL-LATENCY-ANALYSIS.md +++ b/hack/gantry-benchmark/PULL-LATENCY-ANALYSIS.md @@ -124,12 +124,12 @@ occur in the first six minutes and 56% in the second minute alone, falling to zero by minute eleven. At 1.5ms each they cost about 1.7 seconds per node in total. -Second, the stalls are not lost work. The 60.0007s mean is `PeerFetchTimeout` -firing, but `livePeerStream` streams through to the containerd-facing response -and records the verified byte offset, and re-selection resumes from that offset. -A stall costs a DHT lookup and a redial, not the delivered prefix. Stalls also -hold steady at roughly 3,700 per minute from minute four to minute ten, which is -exactly when delivery runs at peak rate. +Second, the stalls are not lost work. These historical runs used a 60s +`PeerFetchTimeout`, and the measured 60.0007s mean is that absolute deadline +firing on progressing transfers. `livePeerStream` streams through to the +containerd-facing response, records the verified byte offset, and resumes from +that offset after re-selection. The default is now 15m; the HTTP/2 transport's +10s idle-health probe remains the dead-connection detector. What does explain the gap is the rate at which layer bytes reach nodes: @@ -237,9 +237,9 @@ Gantry-to-baseline P95 ratio of 1.0. Every other sample in `RESULTS.md` used needed to parallelize it, so this is the largest untested lever. 4. The Gantry phase is slower than baseline because of its cold start. Delivery takes four minutes to reach full rate while baseline is there in one, costing - about 2.6 minutes. Neither the 429 storm nor the 60s stalls are the cost: - the first is a startup transient at 1.5ms each, and the second preserves the - delivered prefix and occurs while delivery is at peak rate. + about 2.6 minutes. Neither the 429 storm nor the historical 60s stalls are + the cost: the first is a startup transient at 1.5ms each, and the second + preserves the delivered prefix and occurs while delivery is at peak rate. 5. The ramp exists because every node walks the manifest in the same order, so the swarm seeds one layer position at a time instead of all 40 at once. Layer completions arrive in strict waves about 7 seconds apart. Baseline shows the @@ -248,10 +248,9 @@ Gantry-to-baseline P95 ratio of 1.0. Every other sample in `RESULTS.md` used 6. Once warm, Gantry delivers faster than pulling from the registry, peaking near 350 MB/s per node against about 182 MB/s for baseline. 7. `PeerFetchTimeout` is a total request deadline rather than a no-progress - deadline, so the throughput a stream must sustain to survive it scales with - layer size: 17.9 MB/s for a 1 GiB layer, 716 MB/s for a 40 GiB one. This did - not dominate these runs, but it does not scale to larger layers. containerd's - own `image_pull_progress_timeout` uses no-progress semantics by contrast. + deadline. The 60s setting used by these runs imposed a size-dependent rate + floor: 17.9 MB/s for a 1 GiB layer and 716 MB/s for a 40 GiB one. The default + is now 15m, while the transport retains its 10s idle-health probe. 8. Gantry's value on this workload is the 99.5% reduction in registry egress and origin pulls, not pod startup latency, which stays 15-22% above baseline. diff --git a/internal/gantry/config/config.go b/internal/gantry/config/config.go index c0e7a7a6a..70842454c 100644 --- a/internal/gantry/config/config.go +++ b/internal/gantry/config/config.go @@ -303,16 +303,11 @@ type Config struct { CoordMaxConcurrentPulls int `yaml:"coord_max_concurrent_pulls"` // PeerFetchTimeout caps the complete peer request, including streaming and - // committing the response body. It is deliberately SHORT (60s): a requester - // stuck on a lockstep-saturated seed must bail and re-select a fresher - // finisher-seed rather than ride the slow stream to completion. Live peer - // streams resume from the verified byte offset when re-selecting, so this - // deadline does not discard an already-delivered prefix. That bail-and- - // re-select is what drives the cold-start cascade (paired with the - // strict containerd hosts.toml, where an exhausted fetch 503s and containerd - // retries Gantry, re-discovering recent finishers). Setting this too high - // (e.g. 1h) removes the re-selection and collapses distribution to the - // single-seed bandwidth bound (the ~12min lockstep regression). + // committing the response body. The default is 15m so progressing large + // layers are not forced to switch providers at a size-dependent throughput + // threshold. The HTTP/2 transport separately probes connections after 10s + // without reads, so dead connections are detected before this safety ceiling. + // Live stream-through still preserves a verified prefix if the ceiling fires. PeerFetchTimeout time.Duration `yaml:"peer_fetch_timeout"` // PeerRediscoverBudget bounds the total wall-clock time the mirror keeps @@ -484,7 +479,7 @@ func NewDefault() *Config { CoordPeerAuthzEnforce: false, CoordMaxDigestsPerRequest: 256, CoordMaxConcurrentPulls: 16, - PeerFetchTimeout: 60 * time.Second, + PeerFetchTimeout: 15 * time.Minute, PeerRediscoverBudget: 5 * time.Minute, // re-discovery cascade on by default (validated at 300 nodes) PeerRediscoverBackoff: time.Second, // pause between re-discovery rounds TransferMaxConcurrentServes: 10, // serve cap preserves bandwidth per large-layer stream diff --git a/internal/gantry/config/config_test.go b/internal/gantry/config/config_test.go index fa9d72d50..80ae2528c 100644 --- a/internal/gantry/config/config_test.go +++ b/internal/gantry/config/config_test.go @@ -14,8 +14,8 @@ import ( func TestDefaultsValidateAfterMinimalUpstream(t *testing.T) { c := NewDefault() - if c.PeerFetchTimeout != 60*time.Second { - t.Fatalf("PeerFetchTimeout = %v, want 60s", c.PeerFetchTimeout) + if c.PeerFetchTimeout != 15*time.Minute { + t.Fatalf("PeerFetchTimeout = %v, want 15m", c.PeerFetchTimeout) } if c.AdvertiseReconcileInterval != time.Minute { From dff502dac1eda96d6f117c34995ef1aa93b3b011 Mon Sep 17 00:00:00 2001 From: Vaibhav Patel Date: Thu, 6 Aug 2026 23:36:08 -0400 Subject: [PATCH 46/60] fix(gantry): bound cold-start coordination storms Limit remote speculative prefetch to three deterministic manifest coordinators while preserving every consumer's self-selected local pulls. This keeps the configured seed count per layer but prevents every manifest consumer from repeating the same remote dispatch plan. Route dynamic membership bootstrap through the existing 8/4/32 cascade and merge TCP/QUIC addresses by peer ID before dialing. Replace stale peerstore addresses with membership-published Pod addresses before coordination RPCs so learned loopback addresses are not retried. Preserve caller cancellation and deadline errors from containerd ReaderAt operations. Canceled mirror requests now terminate quietly instead of being reported and counted as storage backend outages; genuine backend failures remain ErrUnavailable. --- cmd/gantry/main.go | 11 ++- cmd/gantry/membership_peer_resolver_test.go | 6 ++ deploy/gantry/configmap.yaml.tmpl | 6 +- internal/gantry/coldstart/coldstart.go | 7 ++ internal/gantry/coldstart/prefetch.go | 43 ++++++++++++ internal/gantry/coldstart/prefetch_test.go | 67 +++++++++++++++++++ internal/gantry/config/config.go | 12 ++++ internal/gantry/config/config_test.go | 17 +++-- internal/gantry/containerdstore/store.go | 8 +++ internal/gantry/containerdstore/store_test.go | 41 ++++++++++++ internal/gantry/discovery/discovery.go | 67 +++++++++++++++---- internal/gantry/discovery/discovery_test.go | 19 ++++++ internal/gantry/mirror/mirror.go | 5 ++ 13 files changed, 286 insertions(+), 23 deletions(-) diff --git a/cmd/gantry/main.go b/cmd/gantry/main.go index 8f2afda73..fe81d9fe8 100644 --- a/cmd/gantry/main.go +++ b/cmd/gantry/main.go @@ -466,6 +466,7 @@ func runAgent(args []string) error { LocalPull: coordServer, PrefetchPullerReplicas: c.PrefetchPullerReplicas, PrefetchPullerFraction: c.PrefetchPullerFraction, + PrefetchCoordinatorReplicas: c.PrefetchCoordinatorReplicas, PrefetchMaxConcurrentGroups: c.PrefetchMaxConcurrentGroups, PrefetchDispatchJitter: c.PrefetchDispatchJitter, TransientCooldownCap: c.OriginFailureHonorWindowCap, @@ -1347,8 +1348,12 @@ func membershipPeerIDResolver(mv ifaces.Members, ps peerstore.Peerstore, logger addrs = append(addrs, info.Addrs...) } - if ps != nil && len(addrs) > 0 { - ps.AddAddrs(pid, addrs, peerstore.AddressTTL) + if ps != nil { + ps.ClearAddrs(pid) + + if len(addrs) > 0 { + ps.AddAddrs(pid, addrs, peerstore.AddressTTL) + } } return pid, true @@ -2089,7 +2094,7 @@ func (p *layerPrefetchAdapter) OnManifestServed(ctx context.Context, registry, r return } - if err := p.resolver.PrefetchChildren(ctx, pending, registry, repository); err != nil { + if err := p.resolver.PrefetchManifestChildren(ctx, manifestDigest, pending, registry, repository); err != nil { p.logger.Debug("prefetch: PrefetchChildren reported errors", slog.String("manifest", manifestDigest.String()), slog.Int("children", len(pending)), diff --git a/cmd/gantry/membership_peer_resolver_test.go b/cmd/gantry/membership_peer_resolver_test.go index 7486083f8..366af2313 100644 --- a/cmd/gantry/membership_peer_resolver_test.go +++ b/cmd/gantry/membership_peer_resolver_test.go @@ -9,6 +9,7 @@ import ( libp2p "github.com/libp2p/go-libp2p" "github.com/libp2p/go-libp2p/core/peer" + "github.com/libp2p/go-libp2p/core/peerstore" "github.com/multiformats/go-multiaddr" "github.com/Azure/unbounded/internal/gantry/ifaces" @@ -38,6 +39,11 @@ func TestMembershipPeerIDResolverInstallsPodAddresses(t *testing.T) { PeerID: target.ID().String(), P2PAddrs: []string{announced}, }) + caller.Peerstore().AddAddr( + target.ID(), + multiaddr.StringCast("/ip4/127.0.0.1/tcp/4001"), + peerstore.PermanentAddrTTL, + ) resolve := membershipPeerIDResolver(members, caller.Peerstore(), slog.Default()) diff --git a/deploy/gantry/configmap.yaml.tmpl b/deploy/gantry/configmap.yaml.tmpl index 51fd92985..d291b889f 100644 --- a/deploy/gantry/configmap.yaml.tmpl +++ b/deploy/gantry/configmap.yaml.tmpl @@ -140,8 +140,10 @@ data: # A positive fraction overrides prefetch_puller_replicas. There is no # maximum cap other than the eligible cluster/zone node count. prefetch_puller_fraction: 0.02 - # Bound and desynchronize redundant best-effort prefetch dispatch. Without - # this, every requester opens roughly 550 libp2p RPC groups at once. + # Only three deterministic manifest consumers dispatch remote groups; all + # consumers still start any self-selected local seed. This bounds redundant + # fan-out without reducing the configured pullers per layer. + prefetch_coordinator_replicas: 3 prefetch_max_concurrent_groups: 64 prefetch_dispatch_jitter: "1s" # Coord peer authorization. Default false ships observe-only: an diff --git a/internal/gantry/coldstart/coldstart.go b/internal/gantry/coldstart/coldstart.go index 099d65057..fbcaabbff 100644 --- a/internal/gantry/coldstart/coldstart.go +++ b/internal/gantry/coldstart/coldstart.go @@ -142,6 +142,13 @@ type Options struct { // than zero. The resolver selects ceil(eligible candidates * fraction), // with a minimum of one and no cap other than the candidate count. PrefetchPullerFraction float64 + // PrefetchCoordinatorReplicas limits remote speculative dispatch to the + // top-N HRW nodes for a shared coordination key (the manifest digest for + // production callers). Every caller still starts any self-selected local + // pull, and the demand path recovers when none of the coordinators consumed + // the manifest. Zero preserves all-caller dispatch for direct callers and + // tests. + PrefetchCoordinatorReplicas int // PrefetchMaxConcurrentGroups caps simultaneous remote dispatch groups. // Zero uses 64. PrefetchMaxConcurrentGroups int diff --git a/internal/gantry/coldstart/prefetch.go b/internal/gantry/coldstart/prefetch.go index 5f51993c1..0544c46a2 100644 --- a/internal/gantry/coldstart/prefetch.go +++ b/internal/gantry/coldstart/prefetch.go @@ -69,6 +69,20 @@ func prefetchDispatchPlan(self ifaces.NodeID, children []ChildDigest, groups int return offset, delay } +func coordinatesRemotePrefetch(self ifaces.NodeID, candidates []ifaces.Node, key digest.Digest, replicas int) bool { + if replicas <= 0 || replicas >= len(candidates) { + return true + } + + for _, candidate := range hrw.TopK(candidates, key, replicas) { + if candidate.Node.ID == self { + return true + } + } + + return false +} + // PrefetchLayers groups digests by their HRW rank-0 reachable // designated puller and issues one PleasePull RPC per puller. Digests // HRW'ing to self are diverted to the local LocalPullStarter (if @@ -141,6 +155,23 @@ type ChildDigest struct { // into blob on the wire) leaves // p2p_origin_pull_total{kind="config"} permanently zero. func (r *Resolver) PrefetchChildren(ctx context.Context, children []ChildDigest, registry, repository string) error { + var coordinationKey digest.Digest + if len(children) > 0 { + coordinationKey = children[0].Digest + } + + return r.prefetchChildren(ctx, coordinationKey, children, registry, repository) +} + +// PrefetchManifestChildren uses the manifest digest as the stable remote +// coordinator election key. Callers that parsed a manifest should prefer +// this method because local cache filtering can produce different first-child +// digests on different nodes. +func (r *Resolver) PrefetchManifestChildren(ctx context.Context, manifestDigest digest.Digest, children []ChildDigest, registry, repository string) error { + return r.prefetchChildren(ctx, manifestDigest, children, registry, repository) +} + +func (r *Resolver) prefetchChildren(ctx context.Context, coordinationKey digest.Digest, children []ChildDigest, registry, repository string) error { if registry == "" || repository == "" { return fmt.Errorf("%w: registry=%q repository=%q", ErrPrefetchInvalid, registry, repository) @@ -241,6 +272,16 @@ func (r *Resolver) PrefetchChildren(ctx context.Context, children []ChildDigest, } } + remoteCoordinator := coordinatesRemotePrefetch( + self, + candidates, + coordinationKey, + r.opts.PrefetchCoordinatorReplicas, + ) + if !remoteCoordinator { + byGroup = nil + } + if len(byGroup) == 0 && len(selfByKind) == 0 { r.opts.Logger.Debug("coldstart: prefetch had no remote pullers", slog.Int("children", len(children)), @@ -315,6 +356,8 @@ func (r *Resolver) PrefetchChildren(ctx context.Context, children []ChildDigest, slog.Int("pullers", totalPullers), slog.Int("rpc_groups", len(byGroup)+len(selfByKind)), slog.Int("skipped_self", skippedSelf), + slog.Bool("remote_coordinator", remoteCoordinator), + slog.Int("coordinator_replicas", r.opts.PrefetchCoordinatorReplicas), slog.Int("max_concurrent_groups", r.opts.PrefetchMaxConcurrentGroups), slog.Int("dispatch_offset", dispatchOffset), slog.Duration("dispatch_delay", dispatchDelay), diff --git a/internal/gantry/coldstart/prefetch_test.go b/internal/gantry/coldstart/prefetch_test.go index 735065930..0b98ebf9f 100644 --- a/internal/gantry/coldstart/prefetch_test.go +++ b/internal/gantry/coldstart/prefetch_test.go @@ -8,6 +8,7 @@ import ( "errors" "fmt" "sort" + "strings" "sync" "testing" "time" @@ -675,6 +676,72 @@ func TestPrefetchChildren_PropagatesDelegatedAuthorization(t *testing.T) { } } +func TestPrefetchChildren_RemoteDispatchUsesDeterministicCoordinators(t *testing.T) { + cluster := clusterNodes() + digests := findManyDigestsForPullers(t, cluster, map[ifaces.NodeID]int{ + "n0": 1, + "n1": 1, + "n2": 1, + "n3": 1, + }) + + children := make([]coldstart.ChildDigest, 0, len(digests)) + for _, d := range digests { + children = append(children, coldstart.ChildDigest{Digest: d, Kind: ifaces.KindBlob}) + } + + callersWithRemoteDispatch := 0 + totalRemoteGroups := 0 + totalLocalGroups := 0 + manifestDigest := digest.MustParse("sha256:" + strings.Repeat("f", 64)) + + for _, node := range cluster { + coord := &stubCoord{} + localPull := &stubLocalPull{} + now := time.Now + resolver := coldstart.New(coldstart.Options{ + Members: fakes.NewMembers(node.ID, cluster...), + Discovery: &stubDisco{health: 1.0}, + Coord: coord, + Inflight: inflight.New(inflight.DefaultStalls(), now), + Now: now, + HrwScope: hrw.ScopeCluster, + LocalPull: localPull, + PrefetchCoordinatorReplicas: 1, + QueryTimeout: 200 * time.Millisecond, + }) + + if err := resolver.PrefetchManifestChildren(context.Background(), manifestDigest, children, "docker.io", "library/nginx"); err != nil { + t.Fatalf("PrefetchChildren for %s: %v", node.ID, err) + } + + coord.mu.Lock() + groups := len(coord.pleasePullCalls) + coord.mu.Unlock() + + if groups > 0 { + callersWithRemoteDispatch++ + totalRemoteGroups += groups + } + + localPull.mu.Lock() + totalLocalGroups += len(localPull.digests) + localPull.mu.Unlock() + } + + if callersWithRemoteDispatch != 1 { + t.Fatalf("callers with remote dispatch = %d, want 1", callersWithRemoteDispatch) + } + + if totalRemoteGroups != len(cluster)-1 { + t.Fatalf("remote groups = %d, want %d", totalRemoteGroups, len(cluster)-1) + } + + if totalLocalGroups != len(cluster) { + t.Fatalf("local groups = %d, want %d", totalLocalGroups, len(cluster)) + } +} + // TestPrefetchChildren_DistinctPullersBatchedPerKind covers the // cross-product case: 2 kinds × 2 distinct pullers -> 4 PleasePull // RPCs. Confirms the (puller, kind) grouping key works in both diff --git a/internal/gantry/config/config.go b/internal/gantry/config/config.go index 70842454c..82ba7465a 100644 --- a/internal/gantry/config/config.go +++ b/internal/gantry/config/config.go @@ -266,6 +266,11 @@ type Config struct { // except by the number of eligible nodes. PrefetchPullerFraction float64 `yaml:"prefetch_puller_fraction"` + // PrefetchCoordinatorReplicas limits remote speculative prefetch dispatch + // to a deterministic HRW-ranked subset of manifest consumers. Local + // self-selected pulls still run on every consumer. The default is 3. + PrefetchCoordinatorReplicas int `yaml:"prefetch_coordinator_replicas"` + // PrefetchMaxConcurrentGroups caps simultaneous outbound prefetch groups // per manifest. Group dispatch is best effort and target-side deduplicated. PrefetchMaxConcurrentGroups int `yaml:"prefetch_max_concurrent_groups"` @@ -471,6 +476,7 @@ func NewDefault() *Config { HRWK: 3, PrefetchPullerReplicas: 8, PrefetchPullerFraction: 0, + PrefetchCoordinatorReplicas: 3, PrefetchMaxConcurrentGroups: 64, PrefetchDispatchJitter: time.Second, HRWTopologyScope: "cluster", @@ -604,6 +610,7 @@ func (c *Config) LoadEnv(env func(string) string) error { setInt("HRW_K", &c.HRWK) setInt("PREFETCH_PULLER_REPLICAS", &c.PrefetchPullerReplicas) setFloat("PREFETCH_PULLER_FRACTION", &c.PrefetchPullerFraction) + setInt("PREFETCH_COORDINATOR_REPLICAS", &c.PrefetchCoordinatorReplicas) setInt("PREFETCH_MAX_CONCURRENT_GROUPS", &c.PrefetchMaxConcurrentGroups) setDur("PREFETCH_DISPATCH_JITTER", &c.PrefetchDispatchJitter) setStr("HRW_TOPOLOGY_SCOPE", &c.HRWTopologyScope) @@ -669,6 +676,7 @@ func (c *Config) BindFlags(fs *flag.FlagSet) { fs.IntVar(&c.HRWK, "hrw-k", c.HRWK, "HRW top-K size") fs.IntVar(&c.PrefetchPullerReplicas, "prefetch-puller-replicas", c.PrefetchPullerReplicas, "number of HRW-ranked pullers each prefetched layer digest is pulled by (initial seeds); 1 = single puller/tightest dedup, N = N-fold peer fan-out at N origin copies") fs.Float64Var(&c.PrefetchPullerFraction, "prefetch-puller-fraction", c.PrefetchPullerFraction, "fraction of eligible HRW nodes selected as initial pullers, rounded up (0 disables and uses --prefetch-puller-replicas)") + fs.IntVar(&c.PrefetchCoordinatorReplicas, "prefetch-coordinator-replicas", c.PrefetchCoordinatorReplicas, "number of deterministic manifest consumers allowed to dispatch remote speculative prefetch groups") fs.IntVar(&c.PrefetchMaxConcurrentGroups, "prefetch-max-concurrent-groups", c.PrefetchMaxConcurrentGroups, "maximum simultaneous outbound prefetch RPC groups per manifest") fs.DurationVar(&c.PrefetchDispatchJitter, "prefetch-dispatch-jitter", c.PrefetchDispatchJitter, "maximum deterministic per-node delay before dispatching manifest prefetch") fs.StringVar(&c.HRWTopologyScope, "hrw-topology-scope", c.HRWTopologyScope, `HRW scope: "cluster" or "zone"`) @@ -856,6 +864,10 @@ func (c *Config) Validate() error { errs = append(errs, fmt.Errorf("prefetch_puller_fraction: must be between 0 and 1, got %g", c.PrefetchPullerFraction)) } + if c.PrefetchCoordinatorReplicas < 1 { + errs = append(errs, fmt.Errorf("prefetch_coordinator_replicas: must be >= 1, got %d", c.PrefetchCoordinatorReplicas)) + } + switch c.HRWTopologyScope { case "cluster", "zone": default: diff --git a/internal/gantry/config/config_test.go b/internal/gantry/config/config_test.go index 80ae2528c..f2c0c3cf2 100644 --- a/internal/gantry/config/config_test.go +++ b/internal/gantry/config/config_test.go @@ -26,6 +26,10 @@ func TestDefaultsValidateAfterMinimalUpstream(t *testing.T) { t.Fatalf("PrefetchPullerFraction = %v, want disabled", c.PrefetchPullerFraction) } + if c.PrefetchCoordinatorReplicas != 3 { + t.Fatalf("PrefetchCoordinatorReplicas = %d, want 3", c.PrefetchCoordinatorReplicas) + } + if c.PrefetchMaxConcurrentGroups != 64 { t.Fatalf("PrefetchMaxConcurrentGroups = %d, want 64", c.PrefetchMaxConcurrentGroups) } @@ -102,6 +106,8 @@ func TestPrefetchDispatchConfig(t *testing.T) { err := c.LoadEnv(func(key string) string { switch key { + case "GANTRY_PREFETCH_COORDINATOR_REPLICAS": + return "5" case "GANTRY_PREFETCH_MAX_CONCURRENT_GROUPS": return "32" case "GANTRY_PREFETCH_DISPATCH_JITTER": @@ -114,8 +120,8 @@ func TestPrefetchDispatchConfig(t *testing.T) { t.Fatalf("LoadEnv: %v", err) } - if c.PrefetchMaxConcurrentGroups != 32 || c.PrefetchDispatchJitter != 750*time.Millisecond { - t.Fatalf("prefetch dispatch config = %d, %v", c.PrefetchMaxConcurrentGroups, c.PrefetchDispatchJitter) + if c.PrefetchCoordinatorReplicas != 5 || c.PrefetchMaxConcurrentGroups != 32 || c.PrefetchDispatchJitter != 750*time.Millisecond { + t.Fatalf("prefetch dispatch config = %d, %d, %v", c.PrefetchCoordinatorReplicas, c.PrefetchMaxConcurrentGroups, c.PrefetchDispatchJitter) } }) @@ -124,12 +130,12 @@ func TestPrefetchDispatchConfig(t *testing.T) { flags := flag.NewFlagSet("test", flag.ContinueOnError) c.BindFlags(flags) - if err := flags.Parse([]string{"--prefetch-max-concurrent-groups=32", "--prefetch-dispatch-jitter=750ms"}); err != nil { + if err := flags.Parse([]string{"--prefetch-coordinator-replicas=5", "--prefetch-max-concurrent-groups=32", "--prefetch-dispatch-jitter=750ms"}); err != nil { t.Fatalf("Parse: %v", err) } - if c.PrefetchMaxConcurrentGroups != 32 || c.PrefetchDispatchJitter != 750*time.Millisecond { - t.Fatalf("prefetch dispatch config = %d, %v", c.PrefetchMaxConcurrentGroups, c.PrefetchDispatchJitter) + if c.PrefetchCoordinatorReplicas != 5 || c.PrefetchMaxConcurrentGroups != 32 || c.PrefetchDispatchJitter != 750*time.Millisecond { + t.Fatalf("prefetch dispatch config = %d, %d, %v", c.PrefetchCoordinatorReplicas, c.PrefetchMaxConcurrentGroups, c.PrefetchDispatchJitter) } }) } @@ -140,6 +146,7 @@ func TestValidate_PrefetchDispatchBounds(t *testing.T) { mutate func(*Config) want string }{ + {name: "zero coordinators", mutate: func(c *Config) { c.PrefetchCoordinatorReplicas = 0 }, want: "prefetch_coordinator_replicas"}, {name: "zero groups", mutate: func(c *Config) { c.PrefetchMaxConcurrentGroups = 0 }, want: "prefetch_max_concurrent_groups"}, {name: "negative jitter", mutate: func(c *Config) { c.PrefetchDispatchJitter = -time.Second }, want: "prefetch_dispatch_jitter"}, } { diff --git a/internal/gantry/containerdstore/store.go b/internal/gantry/containerdstore/store.go index 2096c3a0a..c383249f8 100644 --- a/internal/gantry/containerdstore/store.go +++ b/internal/gantry/containerdstore/store.go @@ -204,6 +204,10 @@ func (s *Store) withNS(ctx context.Context) context.Context { func (s *Store) Has(ctx context.Context, d gdigest.Digest) (bool, error) { ra, err := s.cs.ReaderAt(s.withNS(ctx), ocispec.Descriptor{Digest: godigest.Digest(d.String())}) if err != nil { + if ctxErr := ctx.Err(); ctxErr != nil { + return false, ctxErr + } + if errors.Is(err, cerrdefs.ErrNotFound) { if s.metrics.OnMiss != nil { s.metrics.OnMiss() @@ -250,6 +254,10 @@ func (s *Store) Open(ctx context.Context, d gdigest.Digest) (io.ReadCloser, int6 ra, err := s.cs.ReaderAt(s.withNS(ctx), desc) if err != nil { + if ctxErr := ctx.Err(); ctxErr != nil { + return nil, 0, ctxErr + } + if errors.Is(err, cerrdefs.ErrNotFound) { if s.metrics.OnMiss != nil { s.metrics.OnMiss() diff --git a/internal/gantry/containerdstore/store_test.go b/internal/gantry/containerdstore/store_test.go index 5a9d2eedf..27ab878be 100644 --- a/internal/gantry/containerdstore/store_test.go +++ b/internal/gantry/containerdstore/store_test.go @@ -559,6 +559,47 @@ func TestStore_OpenUnavailableMappedToErrUnavailable(t *testing.T) { } } +func TestStore_ContextErrorsAreNotMappedToUnavailable(t *testing.T) { + for _, test := range []struct { + name string + err error + }{ + {name: "canceled", err: context.Canceled}, + {name: "deadline", err: context.DeadlineExceeded}, + } { + t.Run(test.name, func(t *testing.T) { + cs := &flakyReaderStore{fakeStore: newFake(), failErr: test.err} + payload := []byte("caller-context-" + test.name) + cs.put(godigest.FromBytes(payload), payload) + + var ctx context.Context + + if errors.Is(test.err, context.DeadlineExceeded) { + var cancel context.CancelFunc + + ctx, cancel = context.WithDeadline(context.Background(), time.Now().Add(-time.Second)) + defer cancel() + } else { + var cancel context.CancelFunc + + ctx, cancel = context.WithCancel(context.Background()) + cancel() + } + + s := New(cs) + d := mustDigest(t, payload) + + if _, err := s.Has(ctx, d); !errors.Is(err, test.err) { + t.Fatalf("Has err = %v, want %v", err, test.err) + } + + if _, _, err := s.Open(ctx, d); !errors.Is(err, test.err) { + t.Fatalf("Open err = %v, want %v", err, test.err) + } + }) + } +} + // flakyReaderStore wraps fakeStore and forces ReaderAt to fail with a // non-NotFound error so we can exercise the unavailable path. type flakyReaderStore struct { diff --git a/internal/gantry/discovery/discovery.go b/internal/gantry/discovery/discovery.go index 3436988c7..6eaa530ae 100644 --- a/internal/gantry/discovery/discovery.go +++ b/internal/gantry/discovery/discovery.go @@ -278,8 +278,8 @@ func (h *Host) Addrs() []multiaddr.Multiaddr { return h.h.Addrs() } // stream handler to the same host that runs the DHT. func (h *Host) LibP2P() host.Host { return h.h } -// ConnectPeers dials a set of multiaddr strings in parallel with a 5s -// per-peer timeout. Used by main.go to seed the DHT routing table from +// ConnectPeers dials a set of multiaddr strings through the bounded 8/4/32 +// bootstrap cascade. Used by main.go to seed the DHT routing table from // the membership view (the design doc): after members.WaitForSync, every Ready // peer with a published p2p multiaddr is fed back into the libp2p host // so kad-dht has direct-connect seeds even without operator-supplied @@ -293,6 +293,8 @@ func (h *Host) ConnectPeers(ctx context.Context, multiaddrs []string) int { } pool := make([]peer.AddrInfo, 0, len(multiaddrs)) + positions := make(map[peer.ID]int, len(multiaddrs)) + for _, p := range multiaddrs { ai, err := peer.AddrInfoFromString(p) if err != nil { @@ -309,14 +311,14 @@ func (h *Host) ConnectPeers(ctx context.Context, multiaddrs []string) int { continue } - pool = append(pool, *ai) + mergePeerAddrInfo(&pool, positions, *ai) } if len(pool) == 0 { return 0 } - return h.dialBatch(ctx, pool) + return h.dialBootstrapPool(ctx, pool) } // RoutingTableSize returns the current kad-dht routing-table size. @@ -456,14 +458,10 @@ func (h *Host) dialBootstrap(ctx context.Context, peers []string) { return } - const ( - batchSize = 8 - successQuorum = 4 - totalDialBudget = 32 - ) - // Parse all peers up-front; drop unparseable ones. pool := make([]peer.AddrInfo, 0, len(peers)) + positions := make(map[peer.ID]int, len(peers)) + for _, p := range peers { ai, err := peer.AddrInfoFromString(p) if err != nil { @@ -475,18 +473,58 @@ func (h *Host) dialBootstrap(ctx context.Context, peers []string) { continue } - pool = append(pool, *ai) + if ai.ID == h.h.ID() { + continue + } + + mergePeerAddrInfo(&pool, positions, *ai) } if len(pool) == 0 { return } + h.dialBootstrapPool(ctx, pool) +} + +func mergePeerAddrInfo(pool *[]peer.AddrInfo, positions map[peer.ID]int, candidate peer.AddrInfo) { + if index, ok := positions[candidate.ID]; ok { + existing := &(*pool)[index] + seen := make(map[string]struct{}, len(existing.Addrs)) + + for _, addr := range existing.Addrs { + seen[addr.String()] = struct{}{} + } + + for _, addr := range candidate.Addrs { + if _, ok := seen[addr.String()]; ok { + continue + } + + existing.Addrs = append(existing.Addrs, addr) + seen[addr.String()] = struct{}{} + } + + return + } + + positions[candidate.ID] = len(*pool) + *pool = append(*pool, candidate) +} + +func (h *Host) dialBootstrapPool(ctx context.Context, pool []peer.AddrInfo) int { + const ( + batchSize = 8 + successQuorum = 4 + totalDialBudget = 32 + ) + // Fisher–Yates shuffle for unbiased random subsets. rng := newBootstrapRand() rng.Shuffle(len(pool), func(i, j int) { pool[i], pool[j] = pool[j], pool[i] }) dialed := 0 + connected := 0 cursor := 0 for cursor < len(pool) && dialed < totalDialBudget { @@ -504,11 +542,14 @@ func (h *Host) dialBootstrap(ctx context.Context, peers []string) { successes := h.dialBatch(ctx, batch) dialed += len(batch) + connected += successes - if successes >= successQuorum { - return + if connected >= successQuorum { + return connected } } + + return connected } // dialBatch fans out parallel Connect attempts against batch with a 5s diff --git a/internal/gantry/discovery/discovery_test.go b/internal/gantry/discovery/discovery_test.go index ad2ff0ca5..4c3f916d4 100644 --- a/internal/gantry/discovery/discovery_test.go +++ b/internal/gantry/discovery/discovery_test.go @@ -48,6 +48,25 @@ func TestDigestToCID_Deterministic(t *testing.T) { } } +func TestMergePeerAddrInfoCombinesAddressesByPeer(t *testing.T) { + peerID := peer.ID("peer-a") + tcp := multiaddr.StringCast("/ip4/10.0.0.1/tcp/4001") + quic := multiaddr.StringCast("/ip4/10.0.0.1/udp/4001/quic-v1") + pool := []peer.AddrInfo{} + positions := map[peer.ID]int{} + + mergePeerAddrInfo(&pool, positions, peer.AddrInfo{ID: peerID, Addrs: []multiaddr.Multiaddr{tcp}}) + mergePeerAddrInfo(&pool, positions, peer.AddrInfo{ID: peerID, Addrs: []multiaddr.Multiaddr{tcp, quic}}) + + if len(pool) != 1 { + t.Fatalf("peer count = %d, want 1", len(pool)) + } + + if len(pool[0].Addrs) != 2 { + t.Fatalf("address count = %d, want 2", len(pool[0].Addrs)) + } +} + func TestHostBringUpEphemeral(t *testing.T) { // Smoke test: New with an ephemeral identity returns a usable host; // Provide on a fresh DHT errors because there are no peers yet, but diff --git a/internal/gantry/mirror/mirror.go b/internal/gantry/mirror/mirror.go index 90442d264..2f41d2ff3 100644 --- a/internal/gantry/mirror/mirror.go +++ b/internal/gantry/mirror/mirror.go @@ -975,6 +975,11 @@ func (s *Server) serveLocalHit(ctx context.Context, w http.ResponseWriter, r *ht return false } + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + logger.Debug("mirror: cache open canceled", slog.Any("err", err)) + return true + } + var eun *ifaces.ErrUnavailable if errors.As(err, &eun) { logger.Warn("mirror: storage unavailable", slog.Any("err", err)) From da2d3c551e791f6c808950c146521d98a8677e53 Mon Sep 17 00:00:00 2001 From: Vaibhav Patel Date: Fri, 7 Aug 2026 08:22:35 -0400 Subject: [PATCH 47/60] feat(gantry): expose per-layer download progress --- cmd/gantry/agent_metrics.go | 101 ++++++++++++++++++++ cmd/gantry/layer_progress_test.go | 97 +++++++++++++++++++ cmd/gantry/main.go | 39 +++++--- cmd/gantry/prefetch_manifest_test.go | 30 ++++++ internal/gantry/mirror/byte_metrics_test.go | 25 ++--- internal/gantry/mirror/mirror.go | 14 +-- 6 files changed, 276 insertions(+), 30 deletions(-) create mode 100644 cmd/gantry/layer_progress_test.go diff --git a/cmd/gantry/agent_metrics.go b/cmd/gantry/agent_metrics.go index 44f78668c..2af0ff2ee 100644 --- a/cmd/gantry/agent_metrics.go +++ b/cmd/gantry/agent_metrics.go @@ -12,12 +12,17 @@ package main // dependencies between subsystems instead of declaring metric metadata. import ( + "strconv" + "sync" "sync/atomic" + "time" "github.com/prometheus/client_golang/prometheus" + "github.com/Azure/unbounded/internal/gantry/digest" "github.com/Azure/unbounded/internal/gantry/ifaces" "github.com/Azure/unbounded/internal/gantry/inflight" + "github.com/Azure/unbounded/internal/gantry/manifest" "github.com/Azure/unbounded/internal/gantry/metrics" ) @@ -84,6 +89,7 @@ type phase2Metrics struct { peerFetchBytes *prometheus.CounterVec mirrorServeBytes *prometheus.CounterVec mirrorCompletedAt *prometheus.GaugeVec + layerCompletedAt *prometheus.GaugeVec peerFetchDur *prometheus.HistogramVec peerDialSuccess prometheus.Counter peerDialFailure prometheus.Counter @@ -130,6 +136,10 @@ func newPhase2Metrics(reg *metrics.Registry) *phase2Metrics { Name: "gantry_mirror_response_completed_timestamp_seconds", Help: "Unix timestamp when a complete response body was most recently written to the local containerd client, labeled by content kind and source path.", }, []string{"kind", "source"}), + layerCompletedAt: reg.NewGaugeVec("mirror", prometheus.GaugeOpts{ + Name: "gantry_layer_download_completed_timestamp_seconds", + Help: "Unix timestamp when a current-image layer response completed to the local containerd client. Zero means pending. Labels are bounded to the current manifest and deleted when the manifest changes.", + }, []string{"node", "image_digest", "layer_digest", "layer_index"}), peerFetchDur: reg.NewHistogramVec("mirror", prometheus.HistogramOpts{ Name: "p2p_peer_fetch_duration_seconds", Help: "End-to-end peer-fetch latency from FetchFromPeer dial to terminal outcome (hit = cache commit, error/stall/notfound = first failing branch). Together with p2p_peer_fetch_total{outcome} this isolates dial vs. body vs. commit-time-digest-verification slowness.", @@ -212,6 +222,97 @@ func newPhase2Metrics(reg *metrics.Registry) *phase2Metrics { return p } +type layerProgressTracker struct { + mu sync.Mutex + gauge *prometheus.GaugeVec + node string + now func() time.Time + manifest digest.Digest + layers map[digest.Digest]string + completedLayers map[digest.Digest]struct{} + earlyCompleted map[digest.Digest]time.Time + oldLabels [][]string +} + +const maxEarlyLayerCompletions = 256 + +func newLayerProgressTracker(gauge *prometheus.GaugeVec, node string, now func() time.Time) *layerProgressTracker { + return &layerProgressTracker{ + gauge: gauge, + node: node, + now: now, + layers: map[digest.Digest]string{}, + completedLayers: map[digest.Digest]struct{}{}, + earlyCompleted: map[digest.Digest]time.Time{}, + } +} + +func (t *layerProgressTracker) observeManifest(manifestDigest digest.Digest, children []manifest.TypedChild) { + t.mu.Lock() + defer t.mu.Unlock() + + if t.manifest == manifestDigest { + return + } + + for _, labels := range t.oldLabels { + t.gauge.DeleteLabelValues(labels...) + } + + t.manifest = manifestDigest + t.layers = make(map[digest.Digest]string, len(children)) + t.completedLayers = make(map[digest.Digest]struct{}, len(children)) + t.oldLabels = t.oldLabels[:0] + + layerIndex := 0 + + for _, child := range children { + if child.Kind != ifaces.KindBlob { + continue + } + + index := strconv.Itoa(layerIndex) + labels := []string{t.node, manifestDigest.String(), child.Digest.String(), index} + t.layers[child.Digest] = index + t.oldLabels = append(t.oldLabels, labels) + + completedAt, completed := t.earlyCompleted[child.Digest] + if completed { + t.gauge.WithLabelValues(labels...).Set(float64(completedAt.UnixNano()) / float64(time.Second)) + t.completedLayers[child.Digest] = struct{}{} + } else { + t.gauge.WithLabelValues(labels...).Set(0) + } + + layerIndex++ + } + + t.earlyCompleted = map[digest.Digest]time.Time{} +} + +func (t *layerProgressTracker) completed(d digest.Digest) { + t.mu.Lock() + defer t.mu.Unlock() + + index, ok := t.layers[d] + if !ok { + if _, exists := t.earlyCompleted[d]; !exists && len(t.earlyCompleted) < maxEarlyLayerCompletions { + t.earlyCompleted[d] = t.now() + } + + return + } + + if _, ok := t.completedLayers[d]; ok { + return + } + + t.gauge.WithLabelValues(t.node, t.manifest.String(), d.String(), index). + Set(float64(t.now().UnixNano()) / float64(time.Second)) + + t.completedLayers[d] = struct{}{} +} + // phase3Metrics groups the instruments owned by // HRW-rank-mismatch detection, DHT-false-empty observability, top-K // probe hit rate, in-flight pull gauge, cold-start latency, and coord // stream counters. diff --git a/cmd/gantry/layer_progress_test.go b/cmd/gantry/layer_progress_test.go new file mode 100644 index 000000000..e4cc5b477 --- /dev/null +++ b/cmd/gantry/layer_progress_test.go @@ -0,0 +1,97 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package main + +import ( + "strings" + "testing" + "time" + + "github.com/Azure/unbounded/internal/gantry/digest" + "github.com/Azure/unbounded/internal/gantry/ifaces" + "github.com/Azure/unbounded/internal/gantry/manifest" + "github.com/Azure/unbounded/internal/gantry/metrics" +) + +func TestLayerProgressTrackerBoundsSeriesToCurrentManifest(t *testing.T) { + registry := metrics.New() + phase := newPhase2Metrics(registry) + now := time.Unix(123, 500_000_000) + tracker := newLayerProgressTracker(phase.layerCompletedAt, "node-a", func() time.Time { return now }) + + manifestA := digest.MustParse("sha256:" + strings.Repeat("a", 64)) + configA := digest.MustParse("sha256:" + strings.Repeat("b", 64)) + layerA0 := digest.MustParse("sha256:" + strings.Repeat("c", 64)) + layerA1 := digest.MustParse("sha256:" + strings.Repeat("d", 64)) + + tracker.completed(layerA0) + + tracker.observeManifest(manifestA, []manifest.TypedChild{ + {Digest: configA, Kind: ifaces.KindConfig}, + {Digest: layerA0, Kind: ifaces.KindBlob}, + {Digest: layerA1, Kind: ifaces.KindBlob}, + }) + tracker.completed(layerA1) + + now = time.Unix(200, 0) + + tracker.completed(layerA1) + + series := layerProgressSeries(t, registry) + if len(series) != 2 { + t.Fatalf("series after first manifest = %d, want 2", len(series)) + } + + if got := series[layerA0.String()]; got != 123.5 { + t.Fatalf("early completed layer value = %v, want 123.5", got) + } + + if got := series[layerA1.String()]; got != 123.5 { + t.Fatalf("completed layer value = %v, want 123.5", got) + } + + manifestB := digest.MustParse("sha256:" + strings.Repeat("e", 64)) + layerB0 := digest.MustParse("sha256:" + strings.Repeat("f", 64)) + tracker.observeManifest(manifestB, []manifest.TypedChild{{Digest: layerB0, Kind: ifaces.KindBlob}}) + + series = layerProgressSeries(t, registry) + if len(series) != 1 { + t.Fatalf("series after manifest replacement = %d, want 1", len(series)) + } + + if _, ok := series[layerB0.String()]; !ok { + t.Fatalf("current layer %s missing from series %v", layerB0, series) + } +} + +func layerProgressSeries(t *testing.T, registry *metrics.Registry) map[string]float64 { + t.Helper() + + families, err := registry.PrometheusRegistry().Gather() + if err != nil { + t.Fatalf("Gather: %v", err) + } + + result := map[string]float64{} + + for _, family := range families { + if family.GetName() != "gantry_layer_download_completed_timestamp_seconds" { + continue + } + + for _, metric := range family.Metric { + digestLabel := "" + + for _, label := range metric.Label { + if label.GetName() == "layer_digest" { + digestLabel = label.GetValue() + } + } + + result[digestLabel] = metric.GetGauge().GetValue() + } + } + + return result +} diff --git a/cmd/gantry/main.go b/cmd/gantry/main.go index fe81d9fe8..21be9d6e7 100644 --- a/cmd/gantry/main.go +++ b/cmd/gantry/main.go @@ -149,6 +149,7 @@ func runAgent(args []string) error { reg.RegisterDefaultCollectors() inst := newPhase1Metrics(reg) p2 := newPhase2Metrics(reg) + layerProgress := newLayerProgressTracker(p2.layerCompletedAt, c.NodeName, time.Now) p9 := newPhase9Metrics(reg) // Storage mode info: emit a single time-series at 1 for the // active backend so dashboards can filter by it. @@ -498,7 +499,7 @@ func runAgent(args []string) error { }, }) coldStartResolver = coldStartAdapter{r: realResolver} - layerPrefetcher = newLayerPrefetcher(realResolver, cstore, logger) + layerPrefetcher = newLayerPrefetcher(realResolver, cstore, logger, layerProgress.observeManifest) logger.Info("cold-start orchestrator wired", slog.Int("hrw_k", c.HRWK), slog.String("hrw_scope", c.HRWTopologyScope), @@ -507,6 +508,10 @@ func runAgent(args []string) error { logger.Info("cold-start orchestrator disabled (single-self membership; no Kubernetes informer)") } + if layerPrefetcher == nil { + layerPrefetcher = newLayerPrefetcher(nil, cstore, logger, layerProgress.observeManifest) + } + // - direct-origin-fallback direct-origin fallback controller (the design doc). Wired // only when the cold-start resolver is also wired; without // orchestration there is no `ErrColdStartExhausted` path to gate. @@ -600,8 +605,9 @@ func runAgent(args []string) error { p2.mirrorServeBytes.WithLabelValues(kind, source).Add(float64(bytes)) }, ), - mirror.WithMirrorResponseCompletedHook(func(kind, source string) { + mirror.WithMirrorResponseCompletedHook(func(d digest.Digest, kind, source string) { p2.mirrorCompletedAt.WithLabelValues(kind, source).SetToCurrentTime() + layerProgress.completed(d) }), mirror.WithOriginStreamMetrics( func(kind string) { p9.originStreamStarted.WithLabelValues(kind).Inc() }, @@ -1906,9 +1912,10 @@ func (a coldStartAdapter) Resolve(ctx context.Context, d digest.Digest, kind ifa // The implementation runs in a goroutine spawned by the mirror; it // MUST NOT panic. All errors are logged at DEBUG. type layerPrefetchAdapter struct { - resolver *coldstart.Resolver - cache ifaces.LocalContentStore - logger *slog.Logger + resolver *coldstart.Resolver + cache ifaces.LocalContentStore + logger *slog.Logger + onManifest func(digest.Digest, []manifest.TypedChild) } // maxManifestBytes caps the size of a manifest body the prefetcher @@ -1967,11 +1974,17 @@ func advertiseOnCommit(ctx context.Context, adv *advertise.Advertiser, store ifa } } -func newLayerPrefetcher(r *coldstart.Resolver, cache ifaces.LocalContentStore, logger *slog.Logger) mirror.LayerPrefetcher { +func newLayerPrefetcher( + r *coldstart.Resolver, + cache ifaces.LocalContentStore, + logger *slog.Logger, + onManifest func(digest.Digest, []manifest.TypedChild), +) mirror.LayerPrefetcher { return &layerPrefetchAdapter{ - resolver: r, - cache: cache, - logger: logger.With(slog.String("subsystem", "prefetch")), + resolver: r, + cache: cache, + logger: logger.With(slog.String("subsystem", "prefetch")), + onManifest: onManifest, } } @@ -2013,7 +2026,7 @@ func (p *layerPrefetchAdapter) openManifest(ctx context.Context, d digest.Digest } func (p *layerPrefetchAdapter) OnManifestServed(ctx context.Context, registry, repository string, manifestDigest digest.Digest) { - if p.resolver == nil { + if p.resolver == nil && p.onManifest == nil { return } // Use a fresh deadline so the prefetch survives the request @@ -2064,7 +2077,11 @@ func (p *layerPrefetchAdapter) OnManifestServed(ctx context.Context, registry, r return } - if len(children) == 0 { + if p.onManifest != nil { + p.onManifest(manifestDigest, children) + } + + if len(children) == 0 || p.resolver == nil { // Image index or no children - nothing to fan out. return } diff --git a/cmd/gantry/prefetch_manifest_test.go b/cmd/gantry/prefetch_manifest_test.go index 00ca033c1..1b0246e58 100644 --- a/cmd/gantry/prefetch_manifest_test.go +++ b/cmd/gantry/prefetch_manifest_test.go @@ -15,6 +15,7 @@ import ( "github.com/Azure/unbounded/internal/gantry/digest" "github.com/Azure/unbounded/internal/gantry/ifaces" + "github.com/Azure/unbounded/internal/gantry/manifest" ) // delayedManifestStore returns ErrNotFound until availableAfter opens, which @@ -116,3 +117,32 @@ func TestOpenManifestGivesUpWhenNeverCommitted(t *testing.T) { t.Fatalf("open attempts = %d, want more than one before giving up", store.openCount()) } } + +func TestLayerPrefetchAdapterReportsManifestChildrenWithoutResolver(t *testing.T) { + manifestDigest := testDigest(t, "a") + configDigest := testDigest(t, "b") + layer0 := testDigest(t, "c") + layer1 := testDigest(t, "d") + body := `{"schemaVersion":2,"config":{"digest":"` + configDigest.String() + `"},"layers":[{"digest":"` + layer0.String() + `"},{"digest":"` + layer1.String() + `"}]}` + store := &delayedManifestStore{body: body, ready: true} + + var ( + gotManifest digest.Digest + gotChildren int + ) + + adapter := &layerPrefetchAdapter{ + cache: store, + logger: slog.Default(), + onManifest: func(observed digest.Digest, children []manifest.TypedChild) { + gotManifest = observed + gotChildren = len(children) + }, + } + + adapter.OnManifestServed(context.Background(), "registry.example", "pull", manifestDigest) + + if gotManifest != manifestDigest || gotChildren != 3 { + t.Fatalf("manifest callback = %s with %d children, want %s with 3", gotManifest, gotChildren, manifestDigest) + } +} diff --git a/internal/gantry/mirror/byte_metrics_test.go b/internal/gantry/mirror/byte_metrics_test.go index 269d1bf8e..b549bfbf2 100644 --- a/internal/gantry/mirror/byte_metrics_test.go +++ b/internal/gantry/mirror/byte_metrics_test.go @@ -21,6 +21,7 @@ import ( ) type byteObservation struct { + digest digest.Digest kind string source string bytes int64 @@ -65,8 +66,8 @@ func TestMirrorByteMetricsCacheSource(t *testing.T) { mirror.WithByteMetrics(func(kind, source string, bytes int64) { served = append(served, byteObservation{kind: kind, source: source, bytes: bytes}) }), - mirror.WithMirrorResponseCompletedHook(func(kind, source string) { - completed = append(completed, byteObservation{kind: kind, source: source}) + mirror.WithMirrorResponseCompletedHook(func(completedDigest digest.Digest, kind, source string) { + completed = append(completed, byteObservation{digest: completedDigest, kind: kind, source: source}) }), ) @@ -79,8 +80,8 @@ func TestMirrorByteMetricsCacheSource(t *testing.T) { t.Fatalf("served observations = %+v, want [%+v]", served, want) } - if len(completed) != 1 || completed[0].kind != want.kind || completed[0].source != want.source { - t.Fatalf("completed observations = %+v, want kind=%s source=%s", completed, want.kind, want.source) + if len(completed) != 1 || completed[0].digest != d || completed[0].kind != want.kind || completed[0].source != want.source { + t.Fatalf("completed observations = %+v, want digest=%s kind=%s source=%s", completed, d, want.kind, want.source) } } @@ -111,8 +112,8 @@ func TestMirrorByteMetricsPeerSource(t *testing.T) { served = append(served, byteObservation{kind: kind, source: source, bytes: bytes}) }, ), - mirror.WithMirrorResponseCompletedHook(func(kind, source string) { - completed = append(completed, byteObservation{kind: kind, source: source}) + mirror.WithMirrorResponseCompletedHook(func(completedDigest digest.Digest, kind, source string) { + completed = append(completed, byteObservation{digest: completedDigest, kind: kind, source: source}) }), ) @@ -130,8 +131,8 @@ func TestMirrorByteMetricsPeerSource(t *testing.T) { t.Fatalf("served observations = %+v, want [%+v]", served, wantServed) } - if len(completed) != 1 || completed[0].kind != wantServed.kind || completed[0].source != wantServed.source { - t.Fatalf("completed observations = %+v, want kind=%s source=%s", completed, wantServed.kind, wantServed.source) + if len(completed) != 1 || completed[0].digest != d || completed[0].kind != wantServed.kind || completed[0].source != wantServed.source { + t.Fatalf("completed observations = %+v, want digest=%s kind=%s source=%s", completed, d, wantServed.kind, wantServed.source) } } @@ -171,8 +172,8 @@ func TestMirrorByteMetricsOriginSource(t *testing.T) { mirror.WithByteMetrics(func(kind, source string, bytes int64) { served = append(served, byteObservation{kind: kind, source: source, bytes: bytes}) }), - mirror.WithMirrorResponseCompletedHook(func(kind, source string) { - completed = append(completed, byteObservation{kind: kind, source: source}) + mirror.WithMirrorResponseCompletedHook(func(completedDigest digest.Digest, kind, source string) { + completed = append(completed, byteObservation{digest: completedDigest, kind: kind, source: source}) }), ) @@ -190,7 +191,7 @@ func TestMirrorByteMetricsOriginSource(t *testing.T) { t.Fatalf("served observations = %+v, want [%+v]", served, wantServed) } - if len(completed) != 1 || completed[0].kind != wantServed.kind || completed[0].source != wantServed.source { - t.Fatalf("completed observations = %+v, want kind=%s source=%s", completed, wantServed.kind, wantServed.source) + if len(completed) != 1 || completed[0].digest != d || completed[0].kind != wantServed.kind || completed[0].source != wantServed.source { + t.Fatalf("completed observations = %+v, want digest=%s kind=%s source=%s", completed, d, wantServed.kind, wantServed.source) } } diff --git a/internal/gantry/mirror/mirror.go b/internal/gantry/mirror/mirror.go index 2f41d2ff3..d9d41f644 100644 --- a/internal/gantry/mirror/mirror.go +++ b/internal/gantry/mirror/mirror.go @@ -201,7 +201,7 @@ type metricsHooks struct { onLiveStreamCompleted func(d digest.Digest) onPeerFetch func(outcome string) onMirrorBytesServed func(kind, source string, bytes int64) - onMirrorResponseCompleted func(kind, source string) + onMirrorResponseCompleted func(d digest.Digest, kind, source string) onPeerFetchLatency func(outcome string, d time.Duration) onPeerDialResult func(success bool) onDhtLookup func(outcome string, dur time.Duration) @@ -370,7 +370,7 @@ func WithLiveStreamCompletedHook(onCompleted func(d digest.Digest)) Option { // WithMirrorResponseCompletedHook registers a callback after a complete GET // response body has been written successfully to the local containerd client. // It is not fired for HEAD requests, partial streams, or failed copies. -func WithMirrorResponseCompletedHook(onCompleted func(kind, source string)) Option { +func WithMirrorResponseCompletedHook(onCompleted func(d digest.Digest, kind, source string)) Option { return func(s *Server) { s.metrics.onMirrorResponseCompleted = onCompleted } @@ -964,7 +964,7 @@ func (s *Server) serveLocalHit(ctx context.Context, w http.ResponseWriter, r *ht if err != nil { logger.Debug("mirror: copy from cache failed", slog.Any("err", err)) } else { - s.fireMirrorResponseCompleted(kind, "cache") + s.fireMirrorResponseCompleted(d, kind, "cache") } return true @@ -1111,7 +1111,7 @@ func (s *Server) serveFromOrigin(ctx context.Context, w http.ResponseWriter, d d } s.fireOriginStreamCompleted(kind) - s.fireMirrorResponseCompleted(kind, "origin") + s.fireMirrorResponseCompleted(d, kind, "origin") s.fireLiveStreamCompleted(d) s.recordNegCacheSuccess(d) @@ -1817,7 +1817,7 @@ func (s *Server) fetchOneProvider(ctx context.Context, w http.ResponseWriter, r s.bumpPeerFetch("hit") s.bumpPeerFetchLatency("hit", fetchStart) - s.fireMirrorResponseCompleted(kind, "peer") + s.fireMirrorResponseCompleted(d, kind, "peer") s.fireLiveStreamCompleted(d) return peerAttemptResult{outcome: peerFetchOutcomeHit, served: true} @@ -2294,12 +2294,12 @@ func (s *Server) fireMirrorBytesServed(kind ifaces.OriginRefKind, source string, s.metrics.onMirrorBytesServed(kind.MetricLabel(), source, bytes) } -func (s *Server) fireMirrorResponseCompleted(kind ifaces.OriginRefKind, source string) { +func (s *Server) fireMirrorResponseCompleted(d digest.Digest, kind ifaces.OriginRefKind, source string) { if s.metrics.onMirrorResponseCompleted == nil { return } - s.metrics.onMirrorResponseCompleted(kind.MetricLabel(), source) + s.metrics.onMirrorResponseCompleted(d, kind.MetricLabel(), source) } func (s *Server) fireOriginStreamStarted(kind ifaces.OriginRefKind) { From 5f34a23908b3048b57ac7aa2b75a5d743f8e2969 Mon Sep 17 00:00:00 2001 From: Vaibhav Patel Date: Fri, 7 Aug 2026 08:22:42 -0400 Subject: [PATCH 48/60] feat(benchmark): show per-node image progress --- hack/cmd/gantry-benchmark-monitor/grid.go | 367 ++++++++++++++++++ .../cmd/gantry-benchmark-monitor/grid_test.go | 105 +++++ hack/cmd/gantry-benchmark-monitor/main.go | 115 +++++- hack/cmd/gantry-benchmark-monitor/monitor.go | 6 + .../gantry-benchmark-monitor/monitor_test.go | 22 ++ hack/cmd/gantry-benchmark-monitor/pods.go | 45 ++- .../cmd/gantry-benchmark-monitor/pods_test.go | 18 + hack/gantry-benchmark/RESULTS.md | 49 ++- .../manifests/monitoring.yaml.tmpl | 91 ++++- 9 files changed, 795 insertions(+), 23 deletions(-) create mode 100644 hack/cmd/gantry-benchmark-monitor/grid.go create mode 100644 hack/cmd/gantry-benchmark-monitor/grid_test.go diff --git a/hack/cmd/gantry-benchmark-monitor/grid.go b/hack/cmd/gantry-benchmark-monitor/grid.go new file mode 100644 index 000000000..1edd21bb1 --- /dev/null +++ b/hack/cmd/gantry-benchmark-monitor/grid.go @@ -0,0 +1,367 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package main + +import ( + "encoding/json" + "fmt" + "math" + "sort" + "strconv" + "strings" + "time" +) + +const defaultNodesPerPage = 64 + +type instantSeries struct { + Metric map[string]string + Timestamp time.Time + Value float64 +} + +type instantResponse struct { + Series []instantSeries +} + +type progressLayer struct { + Index int + Digest string +} + +type progressGrid struct { + Image string + ImageDigest string + Nodes []string + Layers []progressLayer + Downloaded map[string]map[int]time.Time + Unpacked map[string]map[string]time.Time + ImageStart map[string]time.Time + ImageDone map[string]time.Time + Latest time.Time +} + +func parseInstantResponse(raw []byte) (instantResponse, error) { + var envelope struct { + Status string `json:"status"` + Data struct { + Result []struct { + Metric map[string]string `json:"metric"` + Value [2]json.RawMessage `json:"value"` + } `json:"result"` + } `json:"data"` + } + if err := json.Unmarshal(raw, &envelope); err != nil { + return instantResponse{}, fmt.Errorf("decode Prometheus instant response: %w", err) + } + + if envelope.Status != "success" { + return instantResponse{}, fmt.Errorf("prometheus instant query status is %q", envelope.Status) + } + + response := instantResponse{Series: make([]instantSeries, 0, len(envelope.Data.Result))} + for _, rawSeries := range envelope.Data.Result { + var timestamp float64 + if err := json.Unmarshal(rawSeries.Value[0], ×tamp); err != nil { + return instantResponse{}, fmt.Errorf("decode Prometheus instant timestamp: %w", err) + } + + var text string + if err := json.Unmarshal(rawSeries.Value[1], &text); err != nil { + return instantResponse{}, fmt.Errorf("decode Prometheus instant value: %w", err) + } + + value, err := strconv.ParseFloat(text, 64) + if err != nil { + return instantResponse{}, fmt.Errorf("parse Prometheus instant value %q: %w", text, err) + } + + if math.IsNaN(value) || math.IsInf(value, 0) { + continue + } + + seconds, fraction := math.Modf(timestamp) + response.Series = append(response.Series, instantSeries{ + Metric: rawSeries.Metric, + Timestamp: time.Unix(int64(seconds), int64(fraction*float64(time.Second))).UTC(), + Value: value, + }) + } + + return response, nil +} + +func imageDigest(reference string) string { + _, digest, ok := strings.Cut(reference, "@") + if ok { + return digest + } + + return "" +} + +func imageMatches(reference, target, targetDigest string) bool { + if reference == target { + return true + } + + return targetDigest != "" && imageDigest(reference) == targetDigest +} + +func aggregateProgressGrid(response instantResponse, nodes []string, image string) progressGrid { + grid := progressGrid{ + Image: image, + ImageDigest: imageDigest(image), + Nodes: append([]string(nil), nodes...), + Downloaded: map[string]map[int]time.Time{}, + Unpacked: map[string]map[string]time.Time{}, + ImageStart: map[string]time.Time{}, + ImageDone: map[string]time.Time{}, + } + sort.Strings(grid.Nodes) + + layers := map[int]string{} + + for _, series := range response.Series { + if series.Timestamp.After(grid.Latest) { + grid.Latest = series.Timestamp + } + + name, node := series.Metric["__name__"], series.Metric["node"] + + if node == "" { + continue + } + + switch name { + case "gantry_layer_download_completed_timestamp_seconds": + if grid.ImageDigest != "" && series.Metric["image_digest"] != grid.ImageDigest { + continue + } + + index, err := strconv.Atoi(series.Metric["layer_index"]) + if err != nil || index < 0 { + continue + } + + digest := series.Metric["layer_digest"] + if digest == "" { + continue + } + + layers[index] = digest + + if series.Value > 0 { + if grid.Downloaded[node] == nil { + grid.Downloaded[node] = map[int]time.Time{} + } + + grid.Downloaded[node][index] = time.Unix(0, int64(series.Value*float64(time.Second))).UTC() + } + case "gantry_benchmark_layer_unpacked_timestamp_seconds": + if !imageMatches(series.Metric["image"], image, grid.ImageDigest) || series.Value <= 0 { + continue + } + + digest := series.Metric["layer_digest"] + if digest == "" { + continue + } + + if grid.Unpacked[node] == nil { + grid.Unpacked[node] = map[string]time.Time{} + } + + grid.Unpacked[node][digest] = time.Unix(seriesValueSeconds(series.Value), 0).UTC() + case "gantry_benchmark_image_unpack_started_timestamp_seconds": + if imageMatches(series.Metric["image"], image, grid.ImageDigest) && series.Value > 0 { + grid.ImageStart[node] = time.Unix(seriesValueSeconds(series.Value), 0).UTC() + } + case "gantry_benchmark_image_unpacked_timestamp_seconds": + if imageMatches(series.Metric["image"], image, grid.ImageDigest) && series.Value > 0 { + grid.ImageDone[node] = time.Unix(seriesValueSeconds(series.Value), 0).UTC() + } + } + } + + indexes := make([]int, 0, len(layers)) + for index := range layers { + indexes = append(indexes, index) + } + + sort.Ints(indexes) + + grid.Layers = make([]progressLayer, 0, len(indexes)) + for _, index := range indexes { + grid.Layers = append(grid.Layers, progressLayer{Index: index, Digest: layers[index]}) + } + + return grid +} + +func seriesValueSeconds(value float64) int64 { + return int64(math.Floor(value)) +} + +func phaseMinuteCell(completedAt, phaseStart time.Time) byte { + if completedAt.IsZero() { + return '.' + } + + minute := int(completedAt.Sub(phaseStart) / time.Minute) + if minute < 0 { + minute = 0 + } + + if minute > 35 { + minute = 35 + } + + const digits = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" + + return digits[minute] +} + +func unpackCell(grid progressGrid, node string) byte { + if !grid.ImageDone[node].IsZero() { + return '#' + } + + if len(grid.Layers) == 0 { + return '.' + } + + completed := 0 + + for _, layer := range grid.Layers { + if !grid.Unpacked[node][layer.Digest].IsZero() { + completed++ + } + } + + if completed == 0 { + if !grid.ImageStart[node].IsZero() { + return '0' + } + + return '.' + } + + level := int(math.Ceil(float64(completed) / float64(len(grid.Layers)) * 9)) + if level < 1 { + level = 1 + } + + if level > 9 { + level = 9 + } + + return byte('0' + level) +} + +func pageNodes(nodes []string, page, perPage int) ([]string, int, int) { + if perPage <= 0 { + perPage = defaultNodesPerPage + } + + totalPages := max(1, (len(nodes)+perPage-1)/perPage) + + if page < 1 { + page = 1 + } + + if page > totalPages { + page = totalPages + } + + start := (page - 1) * perPage + end := min(start+perPage, len(nodes)) + + return nodes[start:end], page, totalPages +} + +func renderProgressGrids(builder *strings.Builder, snapshot monitorSnapshot) { + if snapshot.GridError != "" { + fmt.Fprintf(builder, "\nprogress grids unavailable: %s\n", snapshot.GridError) + return + } + + grid := snapshot.Progress + if len(grid.Nodes) == 0 || len(grid.Layers) == 0 { + fmt.Fprintln(builder, "\nprogress grids: waiting for per-layer samples") + return + } + + nodes, page, pages := pageNodes(grid.Nodes, snapshot.NodePage, snapshot.NodesPerPage) + + fmt.Fprintln(builder, "\n=== Layer downloads x nodes ===") + fmt.Fprintf(builder, "page %d/%d; nodes %d; showing %s .. %s\n", page, pages, len(grid.Nodes), nodes[0], nodes[len(nodes)-1]) + renderNodeHeader(builder, nodes) + + downloaded := 0 + + for _, layer := range grid.Layers { + fmt.Fprintf(builder, "L%02d %s ", layer.Index, shortDigest(layer.Digest)) + + for _, node := range nodes { + cell := phaseMinuteCell(grid.Downloaded[node][layer.Index], snapshot.PhaseStart) + builder.WriteByte(cell) + + if cell != '.' { + downloaded++ + } + } + + builder.WriteByte('\n') + } + + fmt.Fprintf(builder, "shown cells downloaded: %d/%d; legend: .=pending, 0-9/A-Z=completion phase minute (Z=35+)\n", downloaded, len(nodes)*len(grid.Layers)) + + fmt.Fprintln(builder, "\n=== Image unpack x nodes ===") + fmt.Fprintf(builder, "image %s\n", shortDigest(grid.ImageDigest)) + renderNodeHeader(builder, nodes) + fmt.Fprintf(builder, "image ") + + for _, node := range nodes { + builder.WriteByte(unpackCell(grid, node)) + } + + builder.WriteByte('\n') + fmt.Fprintln(builder, "legend: .=not started, 0=started, 1-9=unpacked layer decile, #=image unpacked") +} + +func renderNodeHeader(builder *strings.Builder, nodes []string) { + for position := 0; position < 3; position++ { + if position == 0 { + fmt.Fprint(builder, "node[-3:] ") + } else { + fmt.Fprint(builder, " ") + } + + for _, node := range nodes { + suffix := node + + if len(suffix) > 3 { + suffix = suffix[len(suffix)-3:] + } + + for len(suffix) < 3 { + suffix = " " + suffix + } + + builder.WriteByte(suffix[position]) + } + + builder.WriteByte('\n') + } +} + +func shortDigest(value string) string { + value = strings.TrimPrefix(value, "sha256:") + if len(value) > 8 { + return value[:8] + } + + return value +} diff --git a/hack/cmd/gantry-benchmark-monitor/grid_test.go b/hack/cmd/gantry-benchmark-monitor/grid_test.go new file mode 100644 index 000000000..e86af5694 --- /dev/null +++ b/hack/cmd/gantry-benchmark-monitor/grid_test.go @@ -0,0 +1,105 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package main + +import ( + "fmt" + "strings" + "testing" + "time" +) + +func TestParseAndAggregateProgressGrid(t *testing.T) { + phaseStart := time.Unix(1_000, 0).UTC() + image := "registry.example/pull@sha256:image" + raw := fmt.Sprintf(`{"status":"success","data":{"result":[ + {"metric":{"__name__":"gantry_layer_download_completed_timestamp_seconds","node":"node-a","image_digest":"sha256:image","layer_digest":"sha256:aaa","layer_index":"0"},"value":[%d,"%d"]}, + {"metric":{"__name__":"gantry_layer_download_completed_timestamp_seconds","node":"node-b","image_digest":"sha256:image","layer_digest":"sha256:aaa","layer_index":"0"},"value":[%d,"0"]}, + {"metric":{"__name__":"gantry_layer_download_completed_timestamp_seconds","node":"node-a","image_digest":"sha256:image","layer_digest":"sha256:bbb","layer_index":"1"},"value":[%d,"%d"]}, + {"metric":{"__name__":"gantry_benchmark_image_unpack_started_timestamp_seconds","node":"node-a","image":"%s"},"value":[%d,"%d"]}, + {"metric":{"__name__":"gantry_benchmark_layer_unpacked_timestamp_seconds","node":"node-a","image":"%s","layer_digest":"sha256:aaa"},"value":[%d,"%d"]}, + {"metric":{"__name__":"gantry_benchmark_image_unpacked_timestamp_seconds","node":"node-b","image":"%s"},"value":[%d,"%d"]} + ]}}`, + phaseStart.Unix()+100, phaseStart.Unix()+30, + phaseStart.Unix()+100, + phaseStart.Unix()+100, phaseStart.Unix()+90, + image, phaseStart.Unix()+100, phaseStart.Unix()+5, + image, phaseStart.Unix()+100, phaseStart.Unix()+80, + image, phaseStart.Unix()+100, phaseStart.Unix()+95, + ) + + response, err := parseInstantResponse([]byte(raw)) + if err != nil { + t.Fatalf("parseInstantResponse: %v", err) + } + + grid := aggregateProgressGrid(response, []string{"node-b", "node-a", "node-c"}, image) + if len(grid.Layers) != 2 || grid.Layers[0].Digest != "sha256:aaa" || grid.Layers[1].Digest != "sha256:bbb" { + t.Fatalf("layers = %#v", grid.Layers) + } + + if got := phaseMinuteCell(grid.Downloaded["node-a"][0], phaseStart); got != '0' { + t.Fatalf("layer 0 cell = %q, want 0", got) + } + + if got := phaseMinuteCell(grid.Downloaded["node-a"][1], phaseStart); got != '1' { + t.Fatalf("layer 1 cell = %q, want 1", got) + } + + if got := unpackCell(grid, "node-a"); got != '5' { + t.Fatalf("node-a unpack cell = %q, want 5", got) + } + + if got := unpackCell(grid, "node-b"); got != '#' { + t.Fatalf("node-b unpack cell = %q, want #", got) + } + + if got := unpackCell(grid, "node-c"); got != '.' { + t.Fatalf("node-c unpack cell = %q, want .", got) + } +} + +func TestRenderProgressGridsPagesNodes(t *testing.T) { + phaseStart := time.Unix(1_000, 0).UTC() + grid := progressGrid{ + Image: "registry.example/pull@sha256:image", + ImageDigest: "sha256:image", + Nodes: []string{"node-001", "node-002", "node-003"}, + Layers: []progressLayer{{Index: 0, Digest: "sha256:aaaaaaaa"}}, + Downloaded: map[string]map[int]time.Time{ + "node-001": {0: phaseStart.Add(2 * time.Minute)}, + }, + Unpacked: map[string]map[string]time.Time{}, + ImageStart: map[string]time.Time{}, + ImageDone: map[string]time.Time{}, + } + + var builder strings.Builder + renderProgressGrids(&builder, monitorSnapshot{ + PhaseStart: phaseStart, + Progress: grid, + NodePage: 1, + NodesPerPage: 2, + }) + + output := builder.String() + for _, want := range []string{ + "=== Layer downloads x nodes ===", + "page 1/2; nodes 3; showing node-001 .. node-002", + "L00 aaaaaaaa 2.", + "=== Image unpack x nodes ===", + "image ..", + } { + if !strings.Contains(output, want) { + t.Fatalf("rendered grid missing %q:\n%s", want, output) + } + } +} + +func TestPageNodesClampsPage(t *testing.T) { + nodes, page, pages := pageNodes([]string{"a", "b", "c"}, 99, 2) + if page != 2 || pages != 2 || len(nodes) != 1 || nodes[0] != "c" { + t.Fatalf("pageNodes = %v, page %d/%d", nodes, page, pages) + } +} diff --git a/hack/cmd/gantry-benchmark-monitor/main.go b/hack/cmd/gantry-benchmark-monitor/main.go index 63ce64276..8f59737fa 100644 --- a/hack/cmd/gantry-benchmark-monitor/main.go +++ b/hack/cmd/gantry-benchmark-monitor/main.go @@ -28,6 +28,7 @@ const ( defaultPrometheusService = "kps-kube-prometheus-stack-prometheus" stateConfigMapName = "gantry-benchmark-state" gantryPhaseLabel = "gantry-cold" + gridQueryInterval = 10 * time.Second ) type monitorConfig struct { @@ -39,6 +40,8 @@ type monitorConfig struct { prometheusService string runID string refreshInterval time.Duration + nodePage int + nodesPerPage int once bool noClear bool } @@ -80,6 +83,7 @@ type monitorSession struct { monitoringNamespace string prometheusService string revision string + image string } type jobStatus struct { @@ -102,8 +106,8 @@ func parseConfig(args []string) (monitorConfig, error) { flags := flag.NewFlagSet("gantry-benchmark-monitor", flag.ContinueOnError) flags.SetOutput(os.Stderr) flags.Usage = func() { - fmt.Fprintln(flags.Output(), "Usage: gantry-benchmark-monitor [options]") //nolint:errcheck // best-effort help output - fmt.Fprintln(flags.Output(), "Live per-minute Gantry peer outcomes and layer-byte delivery.") //nolint:errcheck // best-effort help output + fmt.Fprintln(flags.Output(), "Usage: gantry-benchmark-monitor [options]") //nolint:errcheck // best-effort help output + fmt.Fprintln(flags.Output(), "Live peer traffic, layer downloads, image unpacking, and Pod state.") //nolint:errcheck // best-effort help output flags.PrintDefaults() } @@ -115,6 +119,8 @@ func parseConfig(args []string) (monitorConfig, error) { flags.StringVar(&config.prometheusService, "prometheus-service", envDefault("PROMETHEUS_SERVICE", defaultPrometheusService), "Prometheus service") flags.StringVar(&config.runID, "run-id", "", "run ID (default: active benchmark state)") flags.DurationVar(&config.refreshInterval, "refresh", time.Second, "display and query refresh interval") + flags.IntVar(&config.nodePage, "node-page", 1, "1-based node page shown in progress grids") + flags.IntVar(&config.nodesPerPage, "nodes-per-page", defaultGridColumns(), "node columns shown per progress-grid page") flags.BoolVar(&config.once, "once", false, "print one snapshot and exit") flags.BoolVar(&config.noClear, "no-clear", false, "do not redraw the terminal") @@ -126,6 +132,14 @@ func parseConfig(args []string) (monitorConfig, error) { return monitorConfig{}, fmt.Errorf("refresh must be positive, got %s", config.refreshInterval) } + if config.nodePage < 1 { + return monitorConfig{}, fmt.Errorf("node-page must be at least 1, got %d", config.nodePage) + } + + if config.nodesPerPage < 1 { + return monitorConfig{}, fmt.Errorf("nodes-per-page must be at least 1, got %d", config.nodesPerPage) + } + return config, nil } @@ -215,6 +229,14 @@ func loadJob(ctx context.Context, runner kubectlRunner, namespace, runID string) } `json:"metadata"` Spec struct { Completions int `json:"completions"` + Template struct { + Spec struct { + Containers []struct { + Name string `json:"name"` + Image string `json:"image"` + } `json:"containers"` + } `json:"spec"` + } `json:"template"` } `json:"spec"` Status struct { StartTime string `json:"startTime"` @@ -253,7 +275,25 @@ func loadJob(ctx context.Context, runner kubectlRunner, namespace, runID string) } } - return monitorSession{jobName: job.Metadata.Name, phaseStart: startedAt, nodeCount: job.Spec.Completions}, status, nil + image := "" + + for _, container := range job.Spec.Template.Spec.Containers { + if container.Name == "pull" { + image = container.Image + break + } + } + + return monitorSession{jobName: job.Metadata.Name, phaseStart: startedAt, nodeCount: job.Spec.Completions, image: image}, status, nil +} + +func defaultGridColumns() int { + width, _, err := term.GetSize(int(os.Stdout.Fd())) + if err != nil { + return defaultNodesPerPage + } + + return min(96, max(16, width-14)) } func discoverSession(ctx context.Context, runner kubectlRunner, config monitorConfig) (monitorSession, jobStatus, error) { @@ -335,6 +375,46 @@ func queryPrometheusRange(ctx context.Context, runner kubectlRunner, session mon return parseRangeResponse(output) } +func progressExpression(session monitorSession, config monitorConfig) string { + downloadLabels := fmt.Sprintf( + `namespace=%s,gantry_benchmark="true",controller_revision_hash=%s`, + strconv.Quote(config.gantryNamespace), + strconv.Quote(session.revision), + ) + + observerLabels := fmt.Sprintf(`namespace=%s,gantry_benchmark="true"`, strconv.Quote(config.benchmarkNamespace)) + + if digest := imageDigest(session.image); digest != "" { + downloadLabels += `,image_digest=` + strconv.Quote(digest) + } + + if session.image != "" { + observerLabels += `,image=` + strconv.Quote(session.image) + } + + return fmt.Sprintf(`gantry_layer_download_completed_timestamp_seconds{%s} or {__name__=~"gantry_benchmark_(image_unpack_started|image_unpacked|layer_unpacked)_timestamp_seconds",%s}`, + downloadLabels, + observerLabels, + ) +} + +func queryProgress(ctx context.Context, runner kubectlRunner, session monitorSession, config monitorConfig, now time.Time) (instantResponse, error) { + rawPath := fmt.Sprintf( + "/api/v1/namespaces/%s/services/http:%s:9090/proxy/api/v1/query?query=%s&time=%s", + session.monitoringNamespace, + session.prometheusService, + url.QueryEscape(progressExpression(session, config)), + url.QueryEscape(now.UTC().Format(time.RFC3339Nano)), + ) + + output, err := runner.run(ctx, "get", "--raw", rawPath) + if err != nil { + return instantResponse{}, err + } + + return parseInstantResponse(output) +} + func renderWaiting(config monitorConfig, err error) { if !config.noClear && term.IsTerminal(int(os.Stdout.Fd())) { fmt.Print("\033[H\033[2J") @@ -350,6 +430,9 @@ func runMonitor(ctx context.Context, config monitorConfig) error { session *monitorSession podTracker *podStateTracker lastSnapshot monitorSnapshot + lastProgress progressGrid + gridError string + nextGridAt time.Time ) for { @@ -436,9 +519,14 @@ func runMonitor(ctx context.Context, config monitorConfig) error { lastSnapshot.RefreshInterval = config.refreshInterval lastSnapshot.Job = status lastSnapshot.Color = term.IsTerminal(int(os.Stdout.Fd())) && os.Getenv("NO_COLOR") == "" + lastSnapshot.NodePage = config.nodePage + lastSnapshot.NodesPerPage = config.nodesPerPage + + var nodes []string if podTracker != nil { counts, podErr := podTracker.snapshot() + nodes = podTracker.snapshotNodes() lastSnapshot.PodStates = counts if podErr != nil { @@ -446,6 +534,27 @@ func runMonitor(ctx context.Context, config monitorConfig) error { } } + if status.Complete || !now.Before(nextGridAt) { + gridCtx, gridCancel := context.WithTimeout(ctx, 15*time.Second) + gridResponse, gridErr := queryProgress(gridCtx, runner, *session, config, now) + + gridCancel() + + if gridErr != nil { + gridError = gridErr.Error() + } else { + lastProgress = aggregateProgressGrid(gridResponse, nodes, session.image) + gridError = "" + } + + nextGridAt = now.Add(gridQueryInterval) + } else if len(nodes) > 0 { + lastProgress.Nodes = append(lastProgress.Nodes[:0], nodes...) + } + + lastSnapshot.Progress = lastProgress + lastSnapshot.GridError = gridError + if !config.noClear && term.IsTerminal(int(os.Stdout.Fd())) { fmt.Print("\033[H\033[2J") } diff --git a/hack/cmd/gantry-benchmark-monitor/monitor.go b/hack/cmd/gantry-benchmark-monitor/monitor.go index 653a6cd9c..29cff39d7 100644 --- a/hack/cmd/gantry-benchmark-monitor/monitor.go +++ b/hack/cmd/gantry-benchmark-monitor/monitor.go @@ -48,6 +48,10 @@ type monitorSnapshot struct { Job jobStatus PodStates podStateCounts PodStateError string + Progress progressGrid + GridError string + NodePage int + NodesPerPage int Color bool } @@ -346,6 +350,8 @@ func renderSnapshot(snapshot monitorSnapshot) string { fmt.Fprintf(&builder, "%spod watch: %s%s\n", metaStart, snapshot.PodStateError, reset) } + renderProgressGrids(&builder, snapshot) + fmt.Fprintf(&builder, "%sdisplay refresh: %s; Prometheus scrape cadence: 10s (values repeat between scrapes)\n", metaStart, snapshot.RefreshInterval) if !snapshot.LatestSample.IsZero() { diff --git a/hack/cmd/gantry-benchmark-monitor/monitor_test.go b/hack/cmd/gantry-benchmark-monitor/monitor_test.go index 3336cb21a..4b0421fcc 100644 --- a/hack/cmd/gantry-benchmark-monitor/monitor_test.go +++ b/hack/cmd/gantry-benchmark-monitor/monitor_test.go @@ -155,6 +155,28 @@ func TestPrometheusExpressionScopesCurrentRevision(t *testing.T) { } } +func TestProgressExpressionScopesCurrentImage(t *testing.T) { + expression := progressExpression(monitorSession{ + revision: "gantry-abc123", + image: "registry.example/pull@sha256:image", + }, monitorConfig{ + gantryNamespace: "gantry-system", + benchmarkNamespace: "gantry-benchmark", + }) + + for _, want := range []string{ + `gantry_layer_download_completed_timestamp_seconds`, + `controller_revision_hash="gantry-abc123"`, + `image_digest="sha256:image"`, + `gantry_benchmark_(image_unpack_started|image_unpacked|layer_unpacked)_timestamp_seconds`, + `image="registry.example/pull@sha256:image"`, + } { + if !strings.Contains(expression, want) { + t.Errorf("expression %q is missing %q", expression, want) + } + } +} + func TestCommaInteger(t *testing.T) { for value, want := range map[float64]string{ 0: "0", diff --git a/hack/cmd/gantry-benchmark-monitor/pods.go b/hack/cmd/gantry-benchmark-monitor/pods.go index e42f794af..e47df9307 100644 --- a/hack/cmd/gantry-benchmark-monitor/pods.go +++ b/hack/cmd/gantry-benchmark-monitor/pods.go @@ -6,6 +6,7 @@ package main import ( "context" "fmt" + "sort" "sync" "time" @@ -41,6 +42,7 @@ type podStateCounts struct { type podStateTracker struct { mu sync.RWMutex states map[string]podState + nodes map[string]string err error } @@ -69,7 +71,7 @@ func newPodStateTracker(ctx context.Context, kubeconfig, namespace, jobName stri return nil, fmt.Errorf("create Kubernetes client: %w", err) } - tracker := &podStateTracker{states: map[string]podState{}} + tracker := &podStateTracker{states: map[string]podState{}, nodes: map[string]string{}} if err := tracker.replaceFromList(ctx, client, namespace, jobName); err != nil { return nil, err } @@ -86,13 +88,18 @@ func (t *podStateTracker) replaceFromList(ctx context.Context, client kubernetes } states := make(map[string]podState, len(list.Items)) + nodes := make(map[string]string, len(list.Items)) + for index := range list.Items { pod := &list.Items[index] - states[string(pod.UID)] = classifyPod(pod) + key := string(pod.UID) + states[key] = classifyPod(pod) + nodes[key] = pod.Spec.NodeName } t.mu.Lock() t.states = states + t.nodes = nodes t.err = nil t.mu.Unlock() @@ -113,12 +120,16 @@ func (t *podStateTracker) run(ctx context.Context, client kubernetes.Interface, } states := make(map[string]podState, len(list.Items)) + nodes := make(map[string]string, len(list.Items)) + for index := range list.Items { pod := &list.Items[index] - states[string(pod.UID)] = classifyPod(pod) + key := string(pod.UID) + states[key] = classifyPod(pod) + nodes[key] = pod.Spec.NodeName } - t.replace(states) + t.replace(states, nodes) watcher, err := client.CoreV1().Pods(namespace).Watch(ctx, metav1.ListOptions{ LabelSelector: "job-name=" + jobName, @@ -169,8 +180,10 @@ func (t *podStateTracker) consumeWatch(ctx context.Context, watcher watch.Interf t.mu.Lock() if event.Type == watch.Deleted { delete(t.states, key) + delete(t.nodes, key) } else { t.states[key] = classifyPod(pod) + t.nodes[key] = pod.Spec.NodeName } t.err = nil @@ -191,9 +204,10 @@ func waitForRetry(ctx context.Context) bool { } } -func (t *podStateTracker) replace(states map[string]podState) { +func (t *podStateTracker) replace(states map[string]podState, nodes map[string]string) { t.mu.Lock() t.states = states + t.nodes = nodes t.err = nil t.mu.Unlock() } @@ -232,6 +246,27 @@ func (t *podStateTracker) snapshot() (podStateCounts, error) { return counts, t.err } +func (t *podStateTracker) snapshotNodes() []string { + t.mu.RLock() + defer t.mu.RUnlock() + + seen := make(map[string]struct{}, len(t.nodes)) + for _, node := range t.nodes { + if node != "" { + seen[node] = struct{}{} + } + } + + nodes := make([]string, 0, len(seen)) + for node := range seen { + nodes = append(nodes, node) + } + + sort.Strings(nodes) + + return nodes +} + func classifyPod(pod *corev1.Pod) podState { switch pod.Status.Phase { case corev1.PodSucceeded: diff --git a/hack/cmd/gantry-benchmark-monitor/pods_test.go b/hack/cmd/gantry-benchmark-monitor/pods_test.go index eacaf0e23..679f2e356 100644 --- a/hack/cmd/gantry-benchmark-monitor/pods_test.go +++ b/hack/cmd/gantry-benchmark-monitor/pods_test.go @@ -83,3 +83,21 @@ func TestPodStateTrackerSnapshot(t *testing.T) { t.Fatalf("counts = %#v, want %#v", counts, want) } } + +func TestPodStateTrackerSnapshotNodes(t *testing.T) { + t.Parallel() + + tracker := &podStateTracker{nodes: map[string]string{ + "a": "node-c", + "b": "node-a", + "c": "node-c", + "d": "", + }} + + got := tracker.snapshotNodes() + want := []string{"node-a", "node-c"} + + if len(got) != len(want) || got[0] != want[0] || got[1] != want[1] { + t.Fatalf("snapshotNodes() = %v, want %v", got, want) + } +} diff --git a/hack/gantry-benchmark/RESULTS.md b/hack/gantry-benchmark/RESULTS.md index e715fd95b..764c8133a 100644 --- a/hack/gantry-benchmark/RESULTS.md +++ b/hack/gantry-benchmark/RESULTS.md @@ -30,6 +30,7 @@ containerd download and unpack concurrency, see | **1000 nodes - Canada Central ACR** | **40 GiB** | **47.178 TB** | **245.878 GB** | **99.479%** | **1000 / 2** | **99.800%** | | **1000 nodes - Canada Central ACR rerun** 3 | **40 GiB** | **45.008 TB** | **154.787 GB** | **99.656%** | **1000 / 2** | **99.800%** | | **1000 nodes - Canada Central ACR, 6 downloads** 4 | **40 GiB** | **47.165 TB** | **256.512 GB** | **99.456%** | **1000 / 4** | **99.600%** | +| **1000 nodes - Canada Central, bounded coordination** 5 | **40 GiB** | **47.165 TB** | **945.400 GB** | **97.996%** | **1000 / 1** | **99.900%** | | **1000 nodes - UK South ACR** | **40 GiB** | **53.369 TB** | **219.262 GB** | **99.589%** | **1254 / 5** | **99.601%** | | **1000 nodes - East US ACR** | **40 GiB** | **47.562 TB** | **182.317 GB** | **99.617%** | **1004 / 4** | **99.602%** | | **1000 nodes - Central India ACR** 1 | **40 GiB** | **97.115 TB** | **803.184 GB** | **99.173%** | **2287 / 6** | **99.738%** | @@ -56,6 +57,7 @@ aggregates below. | **1000 nodes - Canada Central ACR** | **40 GiB** | **1862.580s** | **774.028s** | **2030.636s** | **817.139s** | **2149.535s** | **891.766s** | **59.759% faster** | | **1000 nodes - Canada Central ACR rerun** 3 | **40 GiB** | **939.368s** | **1105.766s** | **1091.173s** | **1180.970s** | **1180.111s** | **1239.429s** | **8.229% slower** | | **1000 nodes - Canada Central ACR, 6 downloads** 4 | **40 GiB** | **759.746s** | **923.473s** | **867.850s** | **994.468s** | **956.492s** | **1058.555s** | **14.590% slower** | +| **1000 nodes - Canada Central, bounded coordination** 5 | **40 GiB** | **759.746s** | **917.602s** | **867.850s** | **1001.609s** | **956.492s** | **1132.420s** | **15.413% slower** | | **1000 nodes - UK South ACR** | **40 GiB** | **3561.000s** | **1064.557s** | **3953.000s** | **1146.557s** | **5399.000s** | **1815.557s** | **70.995% faster** | | **1000 nodes - East US ACR** | **40 GiB** | **1401.026s** | **1065.950s** | **1655.894s** | **1144.771s** | **2351.081s** | **1831.832s** | **30.867% faster** | | **1000 nodes - Central India ACR** 2 | **40 GiB** | **3184.649s** | **1065.570s** | **4851.649s** | **1155.570s** | **5351.649s** | **1865.570s** | **76.182% faster** | @@ -64,12 +66,11 @@ aggregates below. | 2000 nodes - sample 3 | 40 GiB | 1241.331s | 1096.589s | 1472.041s | 1184.248s | 2131.053s | 1821.000s | 19.551% faster | Positive improvement means Gantry started pods faster. Most rows used a maximum -Gantry-to-baseline P95 ratio of 3.0, so all six audit-complete runs and the -cross-region performance-only samples passed even when an unusually fast -baseline made Gantry slower. The Canada Central rerun is the exception: it ran -with the gate tightened to 1.0 and did not meet it. The UK South and Central -India rows use retained Kubernetes pod status timestamps; every other row, -including East US, uses AKS audit timestamps. +Gantry-to-baseline P95 ratio of 3.0. The three Canada Central follow-ups marked +3 through 5 used a tightened ratio of 1.0 and reported FAIL on latency while +passing their traffic and integrity gates. The UK South and Central India rows +use retained Kubernetes pod status timestamps; every other row, including +East US, uses AKS audit timestamps. 2 Central India latency uses retained Kubernetes pod status because the telemetry timeout occurred before the runner wrote its audit measurement. @@ -96,6 +97,41 @@ the 1.0 gate this run used. All 2000 pods succeeded with no image-pull backoff and no origin fallbacks. See [PULL-LATENCY-ANALYSIS.md](PULL-LATENCY-ANALYSIS.md). +5 Gantry-only run `run-20260807-035224-b5d10f22`, reported +**FAIL** against the retained baseline from footnote 4 because its P95 ratio +was 1.154, above the strict 1.0 gate. All 1000 pods succeeded, no Gantry pod +restarted, peer delivery reached 100% in minute 11, and no direct-origin +fallback occurred. This run deployed the 15-minute absolute peer-fetch +ceiling, three deterministic remote-prefetch coordinators, bounded 8/4/32 +bootstrap dialing with peer-ID address deduplication, authoritative Pod-IP +replacement before coordination RPCs, and context-aware containerd storage +error classification. + +The fixes removed the targeted failure signatures. Peer stalls fell from +30,829 to zero; remote prefetch groups fell from 575,623 to 1,138 (880 +successful and 258 failed); bootstrap dial failures fell from 1,099,355 to +2,088; and the 1,614 former `storage unavailable` warnings fell to zero. The +configured 2% fanout remained intact: 801 layer origin pulls completed for 40 +layers and 20 selected seeds per layer, plus one extra completed layer copy. +The higher 945.400 GB ACR figure is therefore expected seed traffic, not peer +fallback. Compared with the immediately preceding Gantry-only run using the +60-second ceiling, P50 increased 4.566%, P95 increased 5.794%, and ACR traffic +changed by only +0.011%. The 15-minute ceiling removed false stall +classification, but did not improve startup latency in this sample. + +Two issues remain visible in the complete all-pod log scan. One remote +prefetch coordinator hit libp2p's default connection resource limit for 117 +groups, while another had 141 transient refusal/backoff failures; redundant +dispatch still started every intended seed. Also, one canceled transfer-store +open was logged at WARN and returned HTTP 500, so the transfer endpoint still +needs the same cancellation handling already applied to the mirror endpoint. +The 6,311 peer `notfound` outcomes were confined to the first 44 seconds and +came from current healthy pods whose stale provider records preceded +containerd inventory reconciliation. None of these observations caused a +failed workload pod, origin fallback, digest mismatch, protocol error, or +server-error peer outcome. This Gantry-only validation reuses the footnote 4 +baseline and is excluded from the aggregate independent-run statistics below. + #### Latency excluding image-pull backoff AKS audit logs retained the pod status patches containing `ErrImagePull` and @@ -125,6 +161,7 @@ The unfiltered table remains the primary end-to-end result because | **1000 nodes - Canada Central ACR** | **40 GiB** | **42.735 TB** | **223.358 GB** | **214** | **212** | **41,791** | **0** | | **1000 nodes - Canada Central ACR rerun** 3 | **40 GiB** | **42.817 TB** | **140.672 GB** | **134** | **132** | **41,870** | **0** | | **1000 nodes - Canada Central ACR, 6 downloads** 4 | **40 GiB** | **42.734 TB** | **233.022 GB** | **223** | **219** | **41,789** | **0** | +| **1000 nodes - Canada Central, bounded coordination** 5 | **40 GiB** | **42.153 TB** | **859.069 GB** | **806** | **801** | **41,250** | **0** | | **1000 nodes - East US ACR** | **40 GiB** | **43.137 TB** | **161.075 GB** | **159** | **155** | **42,179** | **0** | | **1000 nodes - Central India ACR** | **40 GiB** | **43.070 TB** | **709.766 GB** | **682** | **662** | **42,129** | **0** | | 2000 nodes - sample 1 | 40 GiB | 86.971 TB | 221.210 GB | 213 | 210 | 85,044 | 0 | diff --git a/hack/gantry-benchmark/manifests/monitoring.yaml.tmpl b/hack/gantry-benchmark/manifests/monitoring.yaml.tmpl index 2df9acd53..6b7fe6ded 100644 --- a/hack/gantry-benchmark/manifests/monitoring.yaml.tmpl +++ b/hack/gantry-benchmark/manifests/monitoring.yaml.tmpl @@ -29,7 +29,7 @@ spec: - action: keep sourceLabels: - __name__ - regex: gantry_storage_mode_info|p2p_dht_health_score|p2p_dht_lookup_total|p2p_dht_lookup_duration_seconds_(bucket|sum|count)|gantry_peer_serve_bytes_total|gantry_origin_bytes_total|p2p_origin_pull_total|p2p_peer_fetch_total|gantry_peer_fetch_last_timestamp_seconds|p2p_peer_fetch_duration_seconds_(bucket|sum|count)|p2p_origin_pull_success_total|p2p_origin_fallback_total|p2p_in_flight_pulls|gantry_peer_fetch_bytes_total|gantry_mirror_bytes_served_total|gantry_mirror_response_completed_timestamp_seconds|gantry_containerd_commit_observed_total|gantry_containerd_commit_observed_timestamp_seconds|gantry_containerd_commit_observation_duration_seconds_(bucket|sum|count)|gantry_containerd_commit_latest_observation_duration_seconds|gantry_containerd_commit_missing_after_stream_total|p2p_coord_please_pull_(served|started|declined)_total|p2p_coord_pull_intent_served_total|p2p_prefetch_(batches|digests|groups)_total|p2p_prefetch_pullers_per_manifest_(bucket|sum|count) + regex: gantry_storage_mode_info|p2p_dht_health_score|p2p_dht_lookup_total|p2p_dht_lookup_duration_seconds_(bucket|sum|count)|gantry_peer_serve_bytes_total|gantry_origin_bytes_total|p2p_origin_pull_total|p2p_peer_fetch_total|gantry_peer_fetch_last_timestamp_seconds|p2p_peer_fetch_duration_seconds_(bucket|sum|count)|p2p_origin_pull_success_total|p2p_origin_fallback_total|p2p_in_flight_pulls|gantry_peer_fetch_bytes_total|gantry_mirror_bytes_served_total|gantry_mirror_response_completed_timestamp_seconds|gantry_layer_download_completed_timestamp_seconds|gantry_containerd_commit_observed_total|gantry_containerd_commit_observed_timestamp_seconds|gantry_containerd_commit_observation_duration_seconds_(bucket|sum|count)|gantry_containerd_commit_latest_observation_duration_seconds|gantry_containerd_commit_missing_after_stream_total|p2p_coord_please_pull_(served|started|declined)_total|p2p_coord_pull_intent_served_total|p2p_prefetch_(batches|digests|groups)_total|p2p_prefetch_pullers_per_manifest_(bucket|sum|count) - action: replace targetLabel: gantry_benchmark replacement: "true" @@ -69,6 +69,7 @@ spec: - --path.rootfs=/host/root - --web.listen-address=:29100 - --collector.filesystem.mount-points-exclude=^/(dev|proc|sys|var/lib/kubelet/pods/.+)($|/) + - --collector.textfile.directory=/textfile ports: - name: node-metrics containerPort: 29100 @@ -96,6 +97,9 @@ spec: - name: root mountPath: /host/root readOnly: true + - name: progress + mountPath: /textfile + readOnly: true - name: containerd-metrics-target image: mcr.microsoft.com/cbl-mariner/busybox:2.0 command: ["sh", "-c", "exec sleep 2147483647"] @@ -119,19 +123,79 @@ spec: - name: containerd-journal image: mcr.microsoft.com/cbl-mariner/busybox:2.0 command: - - chroot - - /host - sh - -c - | - containerd_pid="$(systemctl show --property MainPID --value containerd)" - containerd_bin="$(readlink -f "/proc/${containerd_pid}/exe")" - if [ -z "${containerd_bin}" ] || ! "${containerd_bin}" config dump 2>/dev/null | grep -Eq "^[[:space:]]*level = ['\"]debug['\"]$"; then + set -eu + + textfile="${TEXTFILE_DIR:-/textfile}" + current_image_file="${textfile}/current-image" + mkdir -p "${textfile}" + rm -f "${textfile}"/*.tmp + + containerd_pid="$(chroot /host systemctl show --property MainPID --value containerd)" + containerd_bin="$(chroot /host readlink -f "/proc/${containerd_pid}/exe")" + if [ -z "${containerd_bin}" ] || ! chroot /host "${containerd_bin}" config dump 2>/dev/null | grep -Eq "^[[:space:]]*level = ['\"]debug['\"]$"; then echo "containerd debug logging is required for unpack timing capture" >&2 exit 1 fi - exec journalctl -f -n 0 -u containerd -o cat | \ - grep --line-buffered -E 'PullImage |Pulled image |cancel pulling image |layer unpacked|image unpacked' + + chroot /host journalctl -f -n 0 -u containerd -o cat | while IFS= read -r line; do + case "${line}" in + *PullImage*gantry-benchmark-pull*) + image="$(printf '%s\n' "${line}" | sed -n 's/.*image="\([^"]*\)".*/\1/p')" + if [ -n "${image}" ]; then + previous="$(cat "${current_image_file}" 2>/dev/null || true)" + if [ "${previous}" != "${image}" ]; then + rm -f "${textfile}"/layer-*.prom "${textfile}"/image-*.prom + printf '%s\n' "${image}" >"${current_image_file}.tmp" + mv "${current_image_file}.tmp" "${current_image_file}" + fi + + output="${textfile}/image-start.prom" + if [ ! -f "${output}" ]; then + timestamp="$(date +%s)" + { + echo '# TYPE gantry_benchmark_image_unpack_started_timestamp_seconds gauge' + printf 'gantry_benchmark_image_unpack_started_timestamp_seconds{node="%s",image="%s"} %s\n' "${NODE_NAME}" "${image}" "${timestamp}" + } >"${output}.tmp" + mv "${output}.tmp" "${output}" + fi + fi + printf '%s\n' "${line}" + ;; + *'layer unpacked'*) + digest="$(printf '%s\n' "${line}" | sed -n 's/.*layer=\(sha256:[0-9a-f]*\).*/\1/p')" + image="$(cat "${current_image_file}" 2>/dev/null || true)" + if [ -n "${digest}" ] && [ -n "${image}" ]; then + timestamp="$(date +%s)" + output="${textfile}/layer-${digest#sha256:}.prom" + { + echo '# TYPE gantry_benchmark_layer_unpacked_timestamp_seconds gauge' + printf 'gantry_benchmark_layer_unpacked_timestamp_seconds{node="%s",image="%s",layer_digest="%s"} %s\n' "${NODE_NAME}" "${image}" "${digest}" "${timestamp}" + } >"${output}.tmp" + mv "${output}.tmp" "${output}" + fi + printf '%s\n' "${line}" + ;; + *'image unpacked'*) + image="$(cat "${current_image_file}" 2>/dev/null || true)" + if [ -n "${image}" ]; then + timestamp="$(date +%s)" + output="${textfile}/image-complete.prom" + { + echo '# TYPE gantry_benchmark_image_unpacked_timestamp_seconds gauge' + printf 'gantry_benchmark_image_unpacked_timestamp_seconds{node="%s",image="%s"} %s\n' "${NODE_NAME}" "${image}" "${timestamp}" + } >"${output}.tmp" + mv "${output}.tmp" "${output}" + fi + printf '%s\n' "${line}" + ;; + *'Pulled image'*|*'cancel pulling image'*) + printf '%s\n' "${line}" + ;; + esac + done resources: requests: cpu: 2m @@ -142,10 +206,17 @@ spec: securityContext: privileged: true runAsUser: 0 + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName volumeMounts: - name: host mountPath: /host readOnly: true + - name: progress + mountPath: /textfile volumes: - name: proc hostPath: @@ -163,6 +234,8 @@ spec: hostPath: path: / type: Directory + - name: progress + emptyDir: {} --- apiVersion: monitoring.coreos.com/v1 kind: PodMonitor @@ -184,7 +257,7 @@ spec: metricRelabelings: - action: keep sourceLabels: [__name__] - regex: node_uname_info|node_cpu_seconds_total|node_memory_(MemAvailable|MemTotal)_bytes|node_disk_(read|written)_bytes_total|node_disk_io_time_seconds_total|node_disk_io_time_weighted_seconds_total|node_disk_reads_completed_total|node_disk_writes_completed_total|node_filesystem_(avail|size)_bytes|node_network_speed_bytes|node_network_(receive|transmit)_(bytes|drop|errs)_total + regex: node_uname_info|node_cpu_seconds_total|node_memory_(MemAvailable|MemTotal)_bytes|node_disk_(read|written)_bytes_total|node_disk_io_time_seconds_total|node_disk_io_time_weighted_seconds_total|node_disk_reads_completed_total|node_disk_writes_completed_total|node_filesystem_(avail|size)_bytes|node_network_speed_bytes|node_network_(receive|transmit)_(bytes|drop|errs)_total|gantry_benchmark_(image_unpack_started|image_unpacked|layer_unpacked)_timestamp_seconds - action: replace targetLabel: gantry_benchmark replacement: "true" From 864a4c4b481b6164d403eb00bb5d9ff73774d81e Mon Sep 17 00:00:00 2001 From: Vaibhav Patel Date: Fri, 7 Aug 2026 08:23:08 -0400 Subject: [PATCH 49/60] feat(benchmark): prebuild reusable Gantry images --- hack/cmd/gantry-benchmark/config.go | 8 +- hack/cmd/gantry-benchmark/direct_mode_test.go | 14 + hack/cmd/gantry-benchmark/enable_test.go | 150 ++++++ hack/cmd/gantry-benchmark/image_pool.go | 502 ++++++++++++++++++ hack/cmd/gantry-benchmark/image_pool_test.go | 229 ++++++++ hack/cmd/gantry-benchmark/main.go | 25 + hack/gantry-benchmark/Makefile | 44 +- hack/gantry-benchmark/README.md | 83 ++- .../gantry-benchmark/operator-vm-bootstrap.sh | 39 ++ .../operator-vm-image-pool.sh | 200 +++++++ .../operator-vm-prebuild-images.sh | 109 ++++ .../gantry-benchmark/operator-vm-provision.sh | 7 + hack/gantry-benchmark/operator-vm-run.sh | 119 ++++- 13 files changed, 1497 insertions(+), 32 deletions(-) create mode 100644 hack/cmd/gantry-benchmark/image_pool.go create mode 100644 hack/cmd/gantry-benchmark/image_pool_test.go create mode 100755 hack/gantry-benchmark/operator-vm-image-pool.sh create mode 100755 hack/gantry-benchmark/operator-vm-prebuild-images.sh diff --git a/hack/cmd/gantry-benchmark/config.go b/hack/cmd/gantry-benchmark/config.go index f2ccd8e94..2b34210cf 100644 --- a/hack/cmd/gantry-benchmark/config.go +++ b/hack/cmd/gantry-benchmark/config.go @@ -72,6 +72,8 @@ type benchmarkConfig struct { TelemetryPollInterval time.Duration JobProgressInterval time.Duration StateRoot string + ImagePoolRoot string + ImagePoolBuildRoot string } type phaseRegistry struct { @@ -146,6 +148,8 @@ func loadBenchmarkConfig(getenv func(string) string) (benchmarkConfig, error) { ) } + stateRoot := filepath.Join(repoRoot, "tmp", "gantry-benchmark") + config := benchmarkConfig{ RepoRoot: repoRoot, Mode: mode, @@ -189,7 +193,9 @@ func loadBenchmarkConfig(getenv func(string) string) (benchmarkConfig, error) { TelemetryTimeout: telemetryTimeout, TelemetryPollInterval: telemetryPollInterval, JobProgressInterval: jobProgressInterval, - StateRoot: filepath.Join(repoRoot, "tmp", "gantry-benchmark"), + StateRoot: stateRoot, + ImagePoolRoot: envDefault(getenv, "BENCHMARK_IMAGE_POOL_ROOT", filepath.Join(stateRoot, "image-pool")), + ImagePoolBuildRoot: envDefault(getenv, "BENCHMARK_IMAGE_POOL_BUILD_ROOT", filepath.Join(stateRoot, "image-pool-build")), } if config.NodeCount <= 0 { diff --git a/hack/cmd/gantry-benchmark/direct_mode_test.go b/hack/cmd/gantry-benchmark/direct_mode_test.go index 2c51bcf19..ce568db50 100644 --- a/hack/cmd/gantry-benchmark/direct_mode_test.go +++ b/hack/cmd/gantry-benchmark/direct_mode_test.go @@ -457,6 +457,20 @@ func TestLoadBenchmarkConfigJobTimeoutIsFourHours(t *testing.T) { } } +func TestLoadBenchmarkConfigImagePoolRoots(t *testing.T) { + config, err := loadBenchmarkConfig(envFromMap(map[string]string{ + "BENCHMARK_IMAGE_POOL_ROOT": "/durable/pool", + "BENCHMARK_IMAGE_POOL_BUILD_ROOT": "/build/pool", + })) + if err != nil { + t.Fatalf("loadBenchmarkConfig: %v", err) + } + + if config.ImagePoolRoot != "/durable/pool" || config.ImagePoolBuildRoot != "/build/pool" { + t.Fatalf("image pool roots = %q and %q", config.ImagePoolRoot, config.ImagePoolBuildRoot) + } +} + func TestLoadBenchmarkConfigRejectsUnknownMode(t *testing.T) { _, err := loadBenchmarkConfig(envFromMap(map[string]string{"BENCHMARK_MODE": "proxyless"})) if err == nil { diff --git a/hack/cmd/gantry-benchmark/enable_test.go b/hack/cmd/gantry-benchmark/enable_test.go index 800bebc2a..b30453d51 100644 --- a/hack/cmd/gantry-benchmark/enable_test.go +++ b/hack/cmd/gantry-benchmark/enable_test.go @@ -7,11 +7,13 @@ import ( "bytes" "io" "os" + "os/exec" "path/filepath" "slices" "strings" "testing" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" utilyaml "k8s.io/apimachinery/pkg/util/yaml" ) @@ -113,6 +115,8 @@ func TestRenderMonitoringManifest(t *testing.T) { "p2p_dht_lookup_duration_seconds_(bucket|sum|count)", "gantry_peer_fetch_last_timestamp_seconds", "gantry_mirror_response_completed_timestamp_seconds", + "gantry_layer_download_completed_timestamp_seconds", + "gantry_benchmark_(image_unpack_started|image_unpacked|layer_unpacked)_timestamp_seconds", "gantry_containerd_commit_observation_duration_seconds_(bucket|sum|count)", "gantry_containerd_commit_latest_observation_duration_seconds", "node_uname_info", @@ -131,6 +135,14 @@ func TestRenderMonitoringManifest(t *testing.T) { t.Fatalf("monitoring manifest must not reference the proxy") } + journalScript := renderedContainerScript(t, rendered, "gantry-benchmark-node-observer", "containerd-journal") + command := exec.Command("sh", "-n") + + command.Stdin = strings.NewReader(journalScript) + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("containerd-journal script syntax: %v: %s", err, output) + } + kinds := decodeManifestKinds(t, rendered) wantKinds := []string{"PodMonitor", "DaemonSet", "PodMonitor"} @@ -139,6 +151,144 @@ func TestRenderMonitoringManifest(t *testing.T) { } } +func TestContainerdJournalProgressScript(t *testing.T) { + repoRoot, err := findRepoRoot() + if err != nil { + t.Fatalf("findRepoRoot: %v", err) + } + + benchmark := &benchmark{config: benchmarkConfig{RepoRoot: repoRoot}} + + rendered, err := benchmark.renderManifest(monitoringManifestPath, proxyManifestData{ + Namespace: "gantry-benchmark", + GantryNamespace: "gantry-system", + MonitoringLabel: "kps", + NodeOS: "linux", + NodeArch: "amd64", + RunID: "run-1", + }) + if err != nil { + t.Fatalf("renderManifest: %v", err) + } + + script := renderedContainerScript(t, rendered, "gantry-benchmark-node-observer", "containerd-journal") + tempDir := t.TempDir() + progressDir := filepath.Join(tempDir, "textfile") + + if err := os.Mkdir(progressDir, 0o750); err != nil { + t.Fatalf("mkdir textfile: %v", err) + } + + fakeChroot := `#!/bin/sh +shift +case "$1" in + systemctl) echo 123 ;; + readlink) echo /usr/bin/containerd ;; + /usr/bin/containerd) echo 'level = "debug"' ;; + journalctl) + cat <<'EOF' +level=info msg="PullImage request" image="registry.example/gantry-benchmark-pull@sha256:image" +level=debug msg="layer unpacked" duration=1s layer=sha256:aaaaaaaa +level=debug msg="image unpacked" duration=2s +EOF + ;; + *) echo "unexpected chroot command: $*" >&2; exit 1 ;; +esac +` + + fakePath := filepath.Join(tempDir, "chroot") + if err := os.WriteFile(fakePath, []byte(fakeChroot), 0o750); err != nil { + t.Fatalf("write fake chroot: %v", err) + } + + command := exec.Command("sh", "-c", script) + + command.Env = append(os.Environ(), + "PATH="+tempDir+":"+os.Getenv("PATH"), + "NODE_NAME=node-a", + "TEXTFILE_DIR="+progressDir, + ) + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("run containerd-journal script: %v: %s", err, output) + } + + wantFiles := map[string][]string{ + "image-start.prom": { + `gantry_benchmark_image_unpack_started_timestamp_seconds`, + `node="node-a"`, + `image="registry.example/gantry-benchmark-pull@sha256:image"`, + }, + "layer-aaaaaaaa.prom": { + `gantry_benchmark_layer_unpacked_timestamp_seconds`, + `layer_digest="sha256:aaaaaaaa"`, + }, + "image-complete.prom": { + `gantry_benchmark_image_unpacked_timestamp_seconds`, + }, + } + for name, fragments := range wantFiles { + content, err := os.ReadFile(filepath.Join(progressDir, name)) + if err != nil { + t.Fatalf("read %s: %v", name, err) + } + + for _, fragment := range fragments { + if !bytes.Contains(content, []byte(fragment)) { + t.Errorf("%s missing %q: %s", name, fragment, content) + } + } + } +} + +func renderedContainerScript(t *testing.T, rendered []byte, objectName, containerName string) string { + t.Helper() + + decoder := utilyaml.NewYAMLOrJSONDecoder(bytes.NewReader(rendered), 4096) + + for { + object := &unstructured.Unstructured{} + if err := decoder.Decode(object); err != nil { + if err == io.EOF { + break + } + + t.Fatalf("decode rendered manifest: %v", err) + } + + if object.GetName() != objectName { + continue + } + + containers, found, err := unstructured.NestedSlice(object.Object, "spec", "template", "spec", "containers") + if err != nil || !found { + t.Fatalf("containers for %s: found=%t err=%v", objectName, found, err) + } + + for _, raw := range containers { + container, ok := raw.(map[string]any) + if !ok || container["name"] != containerName { + continue + } + + command, ok := container["command"].([]any) + if !ok || len(command) < 3 { + t.Fatalf("container %s command = %#v", containerName, container["command"]) + } + + script, ok := command[len(command)-1].(string) + if !ok { + t.Fatalf("container %s script = %#v", containerName, command[len(command)-1]) + } + + return script + } + } + + t.Fatalf("container %s in %s not found", containerName, objectName) + + return "" +} + func TestContainerdBenchmarkManifest(t *testing.T) { repoRoot, err := findRepoRoot() if err != nil { diff --git a/hack/cmd/gantry-benchmark/image_pool.go b/hack/cmd/gantry-benchmark/image_pool.go new file mode 100644 index 000000000..e83f4c8d7 --- /dev/null +++ b/hack/cmd/gantry-benchmark/image_pool.go @@ -0,0 +1,502 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "sort" + "strconv" + "time" +) + +const ( + gantryImagePoolSchemaVersion = 1 + maxGantryImagePrebuildCount = 100 +) + +type gantryImagePoolEntry struct { + SchemaVersion int `json:"schema_version"` + ID string `json:"id"` + CreatedAt time.Time `json:"created_at"` + Image string `json:"image"` + PayloadSHA256 string `json:"payload_sha256"` + ImageSizeMiB int `json:"image_size_mib"` + ImageLayers int `json:"image_layers"` + ImagePlatform string `json:"image_platform"` + WorkloadRepository string `json:"workload_repository"` + GantryACRLoginServer string `json:"gantry_acr_login_server"` + ClaimedByRunID string `json:"claimed_by_run_id,omitempty"` + ClaimedAt *time.Time `json:"claimed_at,omitempty"` +} + +func (b *benchmark) prebuildGantryImages(ctx context.Context, count int) error { + if count < 1 || count > maxGantryImagePrebuildCount { + return fmt.Errorf("prebuild count must be between 1 and %d, got %d", maxGantryImagePrebuildCount, count) + } + + if b.config.usesProxy() { + return errors.New("prebuild-gantry requires direct dual-ACR mode") + } + + if b.config.GantryACRLoginServer == "" || b.config.GantryACRUsername == "" || b.config.GantryACRPassword == "" { + return errors.New("prebuild-gantry requires GANTRY_ACR_LOGIN_SERVER, GANTRY_ACR_USERNAME, and GANTRY_ACR_PASSWORD") + } + + if err := b.ensureImagePoolDirectories(); err != nil { + return err + } + + if err := b.loginRegistry(ctx, b.config.GantryACRLoginServer, b.config.GantryACRUsername, b.config.GantryACRPassword); err != nil { + return fmt.Errorf("log in to Gantry ACR: %w", err) + } + + writeAll(b.stdout, fmt.Sprintf( + "prebuilding %d Gantry images (%s, %d layers) into %s\n", + count, + formatMiB(b.config.ImageSizeMiB), + b.config.ImageLayers, + b.config.ImagePoolRoot, + )) + + for index := range count { + entryID, err := newImagePoolEntryID() + if err != nil { + return err + } + + writeAll(b.stdout, fmt.Sprintf("pool image %d/%d: %s\n", index+1, count, entryID)) + + entry, taggedImage, err := b.buildGantryPoolImage(ctx, entryID) + if taggedImage != "" { + b.removeLocalImage(taggedImage) + } + + if err != nil { + return fmt.Errorf("prebuild pool image %s: %w", entryID, err) + } + + if err := writeJSONAtomic(b.imagePoolReadyPath(entry.ID), entry); err != nil { + return fmt.Errorf("record pool image %s: %w", entry.ID, err) + } + + writeAll(b.stdout, fmt.Sprintf( + "pool image ready: id=%s image=%s payload=%s\n", + entry.ID, + entry.Image, + entry.PayloadSHA256, + )) + } + + return b.printImagePoolStatus() +} + +func (b *benchmark) buildGantryPoolImage(ctx context.Context, entryID string) (gantryImagePoolEntry, string, error) { + buildDirectory := filepath.Join(b.imagePoolBuildDirectory(), entryID) + if err := os.RemoveAll(buildDirectory); err != nil { + return gantryImagePoolEntry{}, "", fmt.Errorf("clear image pool build directory: %w", err) + } + + defer func() { _ = os.RemoveAll(buildDirectory) }() //nolint:errcheck // Pool metadata and the pushed image are authoritative. + + if err := os.MkdirAll(buildDirectory, 0o750); err != nil { + return gantryImagePoolEntry{}, "", fmt.Errorf("create image pool build directory: %w", err) + } + + payloadPaths, err := b.writeImagePayloads(buildDirectory) + if err != nil { + return gantryImagePoolEntry{}, "", err + } + defer removePayloads(payloadPaths) + + payloadSHA, err := payloadSHA256(payloadPaths) + if err != nil { + return gantryImagePoolEntry{}, "", err + } + + dockerfile := dualACRDockerfile(proxyPhase("gantry-pool-"+entryID), payloadPaths, payloadSHA) + if err := os.WriteFile(filepath.Join(buildDirectory, "Dockerfile."+string(proxyPhaseGantryCold)), []byte(dockerfile), 0o640); err != nil { + return gantryImagePoolEntry{}, "", fmt.Errorf("write image pool Dockerfile: %w", err) + } + + taggedImage := fmt.Sprintf("%s/%s:%s", b.config.GantryACRLoginServer, b.config.WorkloadRepository, entryID) + + imageDigest, err := b.buildAndPushPreparedImage(ctx, buildDirectory, proxyPhaseGantryCold, taggedImage) + if err != nil { + return gantryImagePoolEntry{}, taggedImage, fmt.Errorf("build and push image pool entry: %w", err) + } + + return gantryImagePoolEntry{ + SchemaVersion: gantryImagePoolSchemaVersion, + ID: entryID, + CreatedAt: time.Now().UTC(), + Image: fmt.Sprintf("%s/%s@%s", b.config.GantryACRLoginServer, b.config.WorkloadRepository, imageDigest), + PayloadSHA256: payloadSHA, + ImageSizeMiB: b.config.ImageSizeMiB, + ImageLayers: b.config.ImageLayers, + ImagePlatform: b.config.ImagePlatform, + WorkloadRepository: b.config.WorkloadRepository, + GantryACRLoginServer: b.config.GantryACRLoginServer, + }, taggedImage, nil +} + +func (b *benchmark) removeLocalImage(taggedImage string) { + var args []string + + switch b.config.ContainerEngine { + case "podman", "docker": + args = []string{"image", "rm", "-f", taggedImage} + default: + return + } + + cleanupContext, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + if _, err := b.commands.Run(cleanupContext, nil, b.config.ContainerEngine, args...); err != nil { + writeAll(b.stderr, fmt.Sprintf("warning: remove local pool image %s: %v\n", taggedImage, err)) + } +} + +func (b *benchmark) prepareGantryOnlyFromPool(ctx context.Context, baselineRunID string) error { + state, err := b.loadState(ctx) + if err != nil { + return err + } + + if state.Status != "enabled" { + return fmt.Errorf("benchmark state is %q, run enable before prepare-gantry-pool", state.Status) + } + + if state.usesProxy() { + return errors.New("prepare-gantry-pool requires direct dual-ACR mode") + } + + if filepath.Base(baselineRunID) != baselineRunID || baselineRunID == "." || baselineRunID == "" { + return fmt.Errorf("invalid baseline run ID %q", baselineRunID) + } + + if err := b.requireLock(ctx, state.RunID); err != nil { + return err + } + + if err := b.validateContext(ctx); err != nil { + return err + } + + baselineState, err := b.readLocalState(baselineRunID) + if err != nil { + return fmt.Errorf("read baseline run state: %w", err) + } + + baselineResult, err := b.readPhaseResult(baselineRunID, "baseline.json") + if err != nil { + return fmt.Errorf("read retained baseline result: %w", err) + } + + if err := validateGantryOnlySource(state, baselineState, baselineResult); err != nil { + return err + } + + entry, readyPath, claimedPath, err := b.claimImagePoolEntry(state, baselineState) + if err != nil { + return err + } + + adopted := false + + defer func() { + if !adopted { + entry.ClaimedByRunID = "" + entry.ClaimedAt = nil + + if err := writeJSONAtomic(claimedPath, entry); err == nil { + _ = os.Rename(claimedPath, readyPath) //nolint:errcheck // Preserve the primary adoption error. + } + } + }() + + baselineResult.RunID = state.RunID + baselineResult.ImageLayers = state.ImageLayers + baselineResult.WorkloadComparisonMode = workloadComparisonRandomShape + state.BaselineImage = baselineState.BaselineImage + state.GantryColdImage = entry.Image + state.WorkloadPayloadSHA256 = entry.PayloadSHA256 + state.WorkloadComparisonMode = workloadComparisonRandomShape + state.Status = "images-prepared" + + if err := b.writeJSONArtifact(state.RunID, "baseline.json", baselineResult); err != nil { + return err + } + + if err := b.saveState(ctx, state); err != nil { + return err + } + + adopted = true + + writeAll(b.stdout, fmt.Sprintf( + "claimed prebuilt Gantry image %s for %s from pool entry %s\n", + entry.Image, + state.RunID, + entry.ID, + )) + + return nil +} + +func (b *benchmark) claimImagePoolEntry(current, baseline benchmarkState) (gantryImagePoolEntry, string, string, error) { + if err := b.ensureImagePoolDirectories(); err != nil { + return gantryImagePoolEntry{}, "", "", err + } + + entries, err := os.ReadDir(b.imagePoolReadyDirectory()) + if err != nil { + return gantryImagePoolEntry{}, "", "", fmt.Errorf("list ready image pool entries: %w", err) + } + + sort.Slice(entries, func(i, j int) bool { return entries[i].Name() < entries[j].Name() }) + + compatible := 0 + + for _, file := range entries { + if file.IsDir() || filepath.Ext(file.Name()) != ".json" { + continue + } + + readyPath := filepath.Join(b.imagePoolReadyDirectory(), file.Name()) + + entry, err := readImagePoolEntry(readyPath) + if err != nil || file.Name() != entry.ID+".json" || validateImagePoolEntry(entry, b.config, current, baseline) != nil { + continue + } + + compatible++ + + claimedName := current.RunID + "--" + file.Name() + + claimedPath := filepath.Join(b.imagePoolClaimedDirectory(), claimedName) + if err := os.Rename(readyPath, claimedPath); err != nil { + if errors.Is(err, fs.ErrNotExist) { + continue + } + + return gantryImagePoolEntry{}, "", "", fmt.Errorf("claim image pool entry %s: %w", entry.ID, err) + } + + entry.ClaimedByRunID = current.RunID + claimedAt := time.Now().UTC() + + entry.ClaimedAt = &claimedAt + if err := writeJSONAtomic(claimedPath, entry); err != nil { + _ = os.Rename(claimedPath, readyPath) //nolint:errcheck // Preserve metadata write error. + + return gantryImagePoolEntry{}, "", "", fmt.Errorf("record image pool claim %s: %w", entry.ID, err) + } + + return entry, readyPath, claimedPath, nil + } + + return gantryImagePoolEntry{}, "", "", fmt.Errorf( + "no compatible ready Gantry image in %s (examined %d entries, %d compatible before claim races)", + b.imagePoolReadyDirectory(), + len(entries), + compatible, + ) +} + +func validateImagePoolEntry(entry gantryImagePoolEntry, config benchmarkConfig, current, baseline benchmarkState) error { + if entry.SchemaVersion != gantryImagePoolSchemaVersion { + return fmt.Errorf("pool entry schema %d, want %d", entry.SchemaVersion, gantryImagePoolSchemaVersion) + } + + if entry.ID == "" || entry.ID == "." || entry.ID == ".." || filepath.Base(entry.ID) != entry.ID { + return fmt.Errorf("invalid pool entry ID %q", entry.ID) + } + + if entry.ImageSizeMiB != current.ImageSizeMiB || entry.ImageLayers != current.ImageLayers || + entry.ImagePlatform != current.ImagePlatform || entry.WorkloadRepository != current.WorkloadRepository || + entry.GantryACRLoginServer != current.GantryACRLoginServer { + return errors.New("pool entry shape does not match current benchmark") + } + + if entry.ImageSizeMiB != config.ImageSizeMiB || entry.ImageLayers != config.ImageLayers || + entry.ImagePlatform != config.ImagePlatform || entry.WorkloadRepository != config.WorkloadRepository || + entry.GantryACRLoginServer != config.GantryACRLoginServer { + return errors.New("pool entry shape does not match configured builder") + } + + return validateAdoptedFreshGantryImage(current, baseline, entry.Image, entry.PayloadSHA256) +} + +func (b *benchmark) printImagePoolStatus() error { + if err := b.ensureImagePoolDirectories(); err != nil { + return err + } + + ready, err := readImagePoolDirectory(b.imagePoolReadyDirectory()) + if err != nil { + return err + } + + claimed, err := readImagePoolDirectory(b.imagePoolClaimedDirectory()) + if err != nil { + return err + } + + writeAll(b.stdout, fmt.Sprintf("Gantry image pool: root=%s ready=%d claimed=%d\n", b.config.ImagePoolRoot, len(ready), len(claimed))) + + for _, entry := range ready { + writeAll(b.stdout, fmt.Sprintf( + "READY %s %s %s %s/%d\n", + entry.ID, + entry.CreatedAt.Format(time.RFC3339), + entry.Image, + formatMiB(entry.ImageSizeMiB), + entry.ImageLayers, + )) + } + + for _, entry := range claimed { + claimedAt := "unknown" + if entry.ClaimedAt != nil { + claimedAt = entry.ClaimedAt.Format(time.RFC3339) + } + + writeAll(b.stdout, fmt.Sprintf( + "CLAIMED %s run=%s at=%s image=%s\n", + entry.ID, + entry.ClaimedByRunID, + claimedAt, + entry.Image, + )) + } + + return nil +} + +func readImagePoolDirectory(directory string) ([]gantryImagePoolEntry, error) { + files, err := os.ReadDir(directory) + if err != nil { + return nil, fmt.Errorf("read image pool directory %s: %w", directory, err) + } + + entries := make([]gantryImagePoolEntry, 0, len(files)) + for _, file := range files { + if file.IsDir() || filepath.Ext(file.Name()) != ".json" { + continue + } + + entry, err := readImagePoolEntry(filepath.Join(directory, file.Name())) + if err != nil { + return nil, err + } + + entries = append(entries, entry) + } + + sort.Slice(entries, func(i, j int) bool { return entries[i].ID < entries[j].ID }) + + return entries, nil +} + +func readImagePoolEntry(path string) (gantryImagePoolEntry, error) { + var entry gantryImagePoolEntry + if err := readJSONFile(path, &entry); err != nil { + return gantryImagePoolEntry{}, fmt.Errorf("read image pool entry %s: %w", path, err) + } + + return entry, nil +} + +func writeJSONAtomic(path string, value any) error { + encoded, err := json.MarshalIndent(value, "", " ") + if err != nil { + return err + } + + if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil { + return err + } + + temporary, err := os.CreateTemp(filepath.Dir(path), ".pool-entry-*.tmp") + if err != nil { + return err + } + + temporaryPath := temporary.Name() + + defer func() { _ = os.Remove(temporaryPath) }() //nolint:errcheck // Rename removes successful temporary files. + + if err := temporary.Chmod(0o640); err != nil { + return errors.Join(err, temporary.Close()) + } + + if _, err := temporary.Write(append(encoded, '\n')); err != nil { + return errors.Join(err, temporary.Close()) + } + + if err := temporary.Sync(); err != nil { + return errors.Join(err, temporary.Close()) + } + + if err := temporary.Close(); err != nil { + return err + } + + return os.Rename(temporaryPath, path) +} + +func newImagePoolEntryID() (string, error) { + suffix, err := randomHex(4) + if err != nil { + return "", err + } + + return "pool-" + time.Now().UTC().Format("20060102-150405.000000000") + "-" + suffix, nil +} + +func (b *benchmark) ensureImagePoolDirectories() error { + for _, directory := range []string{b.imagePoolReadyDirectory(), b.imagePoolClaimedDirectory(), b.imagePoolBuildDirectory()} { + if err := os.MkdirAll(directory, 0o750); err != nil { + return fmt.Errorf("create image pool directory %s: %w", directory, err) + } + } + + return nil +} + +func (b *benchmark) imagePoolReadyDirectory() string { + return filepath.Join(b.config.ImagePoolRoot, "ready") +} + +func (b *benchmark) imagePoolClaimedDirectory() string { + return filepath.Join(b.config.ImagePoolRoot, "claimed") +} + +func (b *benchmark) imagePoolBuildDirectory() string { + if b.config.ImagePoolBuildRoot != "" { + return b.config.ImagePoolBuildRoot + } + + return filepath.Join(b.config.ImagePoolRoot, "build") +} + +func (b *benchmark) imagePoolReadyPath(entryID string) string { + return filepath.Join(b.imagePoolReadyDirectory(), entryID+".json") +} + +func parsePrebuildCount(value string) (int, error) { + count, err := strconv.Atoi(value) + if err != nil { + return 0, fmt.Errorf("parse prebuild count %q: %w", value, err) + } + + return count, nil +} diff --git a/hack/cmd/gantry-benchmark/image_pool_test.go b/hack/cmd/gantry-benchmark/image_pool_test.go new file mode 100644 index 000000000..670e7492c --- /dev/null +++ b/hack/cmd/gantry-benchmark/image_pool_test.go @@ -0,0 +1,229 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package main + +import ( + "bytes" + "context" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func poolTestStates() (benchmarkState, benchmarkState) { + current := benchmarkState{ + RunID: "run-new", Mode: benchmarkModeDirect, NodeCount: 1000, + ImagePlatform: "linux/amd64", ImageSizeMiB: 40960, ImageLayers: 40, + WorkloadRepository: "gantry-benchmark-pull", + BaselineACRLoginServer: "baseline.example", + GantryACRLoginServer: "gantry.example", + } + baseline := current + baseline.RunID = "run-baseline" + baseline.BaselineImage = "baseline.example/gantry-benchmark-pull@sha256:" + repeatHex("a") + baseline.WorkloadPayloadSHA256 = "sha256:" + repeatHex("b") + + return current, baseline +} + +func poolTestEntry(id, imageHex, payloadHex string) gantryImagePoolEntry { + return gantryImagePoolEntry{ + SchemaVersion: gantryImagePoolSchemaVersion, + ID: id, + CreatedAt: time.Date(2026, 8, 7, 1, 2, 3, 0, time.UTC), + Image: "gantry.example/gantry-benchmark-pull@sha256:" + repeatHex(imageHex), + PayloadSHA256: "sha256:" + repeatHex(payloadHex), + ImageSizeMiB: 40960, + ImageLayers: 40, + ImagePlatform: "linux/amd64", + WorkloadRepository: "gantry-benchmark-pull", + GantryACRLoginServer: "gantry.example", + } +} + +func TestValidateImagePoolEntry(t *testing.T) { + current, baseline := poolTestStates() + config := benchmarkConfig{ + Mode: benchmarkModeDirect, ImageSizeMiB: 40960, ImageLayers: 40, + ImagePlatform: "linux/amd64", WorkloadRepository: "gantry-benchmark-pull", + GantryACRLoginServer: "gantry.example", + } + entry := poolTestEntry("pool-a", "c", "d") + + if err := validateImagePoolEntry(entry, config, current, baseline); err != nil { + t.Fatalf("validate matching entry: %v", err) + } + + entry.ImageLayers++ + if err := validateImagePoolEntry(entry, config, current, baseline); err == nil { + t.Fatal("expected mismatched layer count to fail") + } + + entry = poolTestEntry("..", "c", "d") + if err := validateImagePoolEntry(entry, config, current, baseline); err == nil { + t.Fatal("expected unsafe entry ID to fail") + } +} + +func TestClaimImagePoolEntryMovesReadyEntryOnce(t *testing.T) { + root := t.TempDir() + current, baseline := poolTestStates() + + benchmark := &benchmark{config: benchmarkConfig{ + Mode: benchmarkModeDirect, ImagePoolRoot: root, + ImageSizeMiB: 40960, ImageLayers: 40, ImagePlatform: "linux/amd64", + WorkloadRepository: "gantry-benchmark-pull", GantryACRLoginServer: "gantry.example", + }} + if err := benchmark.ensureImagePoolDirectories(); err != nil { + t.Fatal(err) + } + + entry := poolTestEntry("pool-a", "c", "d") + + readyPath := benchmark.imagePoolReadyPath(entry.ID) + if err := writeJSONAtomic(readyPath, entry); err != nil { + t.Fatal(err) + } + + claimed, gotReadyPath, claimedPath, err := benchmark.claimImagePoolEntry(current, baseline) + if err != nil { + t.Fatalf("claimImagePoolEntry: %v", err) + } + + if claimed.ID != entry.ID || gotReadyPath != readyPath || claimed.ClaimedByRunID != current.RunID || claimed.ClaimedAt == nil { + t.Fatalf("claimed entry = %#v, readyPath=%q", claimed, gotReadyPath) + } + + if _, err := os.Stat(readyPath); !os.IsNotExist(err) { + t.Fatalf("ready entry still exists: %v", err) + } + + if _, err := os.Stat(claimedPath); err != nil { + t.Fatalf("claimed entry missing: %v", err) + } + + if _, _, _, err := benchmark.claimImagePoolEntry(current, baseline); err == nil { + t.Fatal("second claim unexpectedly succeeded") + } +} + +func TestReadImagePoolDirectorySortsEntries(t *testing.T) { + directory := t.TempDir() + for _, id := range []string{"pool-z", "pool-a"} { + if err := writeJSONAtomic(filepath.Join(directory, id+".json"), poolTestEntry(id, "c", "d")); err != nil { + t.Fatal(err) + } + } + + entries, err := readImagePoolDirectory(directory) + if err != nil { + t.Fatal(err) + } + + if len(entries) != 2 || entries[0].ID != "pool-a" || entries[1].ID != "pool-z" { + t.Fatalf("entries = %#v", entries) + } +} + +func TestPoolEntryClaimFieldsOmitWhenCleared(t *testing.T) { + path := filepath.Join(t.TempDir(), "pool-a.json") + entry := poolTestEntry("pool-a", "c", "d") + entry.ClaimedByRunID = "" + + entry.ClaimedAt = nil + if err := writeJSONAtomic(path, entry); err != nil { + t.Fatal(err) + } + + content, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + + if strings.Contains(string(content), "claimed_by_run_id") || strings.Contains(string(content), "claimed_at") { + t.Fatalf("cleared claim fields persisted: %s", content) + } +} + +func TestParsePrebuildCount(t *testing.T) { + if count, err := parsePrebuildCount("10"); err != nil || count != 10 { + t.Fatalf("parsePrebuildCount = %d, %v", count, err) + } + + if _, err := parsePrebuildCount("ten"); err == nil { + t.Fatal("expected invalid count to fail") + } +} + +func TestPrebuildGantryImagesCreatesReadyEntriesAndCleansBuilds(t *testing.T) { + poolRoot := filepath.Join(t.TempDir(), "pool") + buildRoot := filepath.Join(t.TempDir(), "build") + runner := &dualACRImageRunner{} + + var progress bytes.Buffer + + benchmark := &benchmark{ + config: benchmarkConfig{ + Mode: benchmarkModeDirect, + ImagePoolRoot: poolRoot, + ImagePoolBuildRoot: buildRoot, + ContainerEngine: "podman", + ImagePlatform: "linux/amd64", + ImageSizeMiB: 2, + ImageLayers: 2, + WorkloadRepository: "gantry-benchmark-pull", + GantryACRLoginServer: "gantry.azurecr.io", + GantryACRUsername: "user", + GantryACRPassword: "password", + }, + commands: runner, + stdout: &progress, + stderr: &progress, + } + + if err := benchmark.prebuildGantryImages(context.Background(), 2); err != nil { + t.Fatalf("prebuildGantryImages: %v", err) + } + + entries, err := readImagePoolDirectory(benchmark.imagePoolReadyDirectory()) + if err != nil { + t.Fatal(err) + } + + if len(entries) != 2 { + t.Fatalf("ready entries = %d, want 2", len(entries)) + } + + if entries[0].PayloadSHA256 == entries[1].PayloadSHA256 { + t.Fatal("pool entries unexpectedly reused random payload bytes") + } + + buildFiles, err := os.ReadDir(buildRoot) + if err != nil { + t.Fatal(err) + } + + if len(buildFiles) != 0 { + t.Fatalf("pool build root contains %d entries after cleanup", len(buildFiles)) + } + + commands := strings.Join(runner.commands, "\n") + if got := strings.Count(commands, "podman login "); got != 1 { + t.Fatalf("registry login count = %d, want 1:\n%s", got, commands) + } + + if got := strings.Count(commands, "podman push "); got != 2 { + t.Fatalf("push count = %d, want 2:\n%s", got, commands) + } + + if got := strings.Count(commands, "podman image rm -f "); got != 2 { + t.Fatalf("local image removal count = %d, want 2:\n%s", got, commands) + } + + if strings.Contains(commands, "kubectl") { + t.Fatalf("standalone prebuild unexpectedly invoked Kubernetes:\n%s", commands) + } +} diff --git a/hack/cmd/gantry-benchmark/main.go b/hack/cmd/gantry-benchmark/main.go index e8dcc38fe..7e73faaff 100644 --- a/hack/cmd/gantry-benchmark/main.go +++ b/hack/cmd/gantry-benchmark/main.go @@ -59,6 +59,19 @@ func runCLI(ctx context.Context, args []string, stdout, stderr io.Writer) error } switch args[0] { + case "image-pool-status": + return benchmark.printImagePoolStatus() + case "prebuild-gantry": + if len(args) != 2 { + return fmt.Errorf("usage: gantry-benchmark prebuild-gantry ") + } + + count, err := parsePrebuildCount(args[1]) + if err != nil { + return err + } + + return benchmark.prebuildGantryImages(ctx, count) case "disable": return benchmark.disable(ctx) case "enable": @@ -94,6 +107,12 @@ func runCLI(ctx context.Context, args []string, stdout, stderr io.Writer) error } return benchmark.prepareAdoptedFreshGantryOnly(ctx, args[1], args[2], args[3]) + case "prepare-gantry-pool": + if len(args) != 2 { + return fmt.Errorf("usage: gantry-benchmark prepare-gantry-pool ") + } + + return benchmark.prepareGantryOnlyFromPool(ctx, args[1]) case "preflight": return benchmark.preflight(ctx) case "run": @@ -113,6 +132,10 @@ func printUsage(writer io.Writer) { writeAll(writer, `Usage: gantry-benchmark Subcommands: + image-pool-status + list ready and claimed prebuilt Gantry images + prebuild-gantry + build and push reusable Gantry images without enabling a benchmark disable restore the cluster and remove benchmark instrumentation enable install benchmark instrumentation after safety checks prepare build and push both digest-pinned images before ACR goes private @@ -124,6 +147,8 @@ Subcommands: generate new random bytes and build only a fresh Gantry image prepare-gantry-adopt adopt an already-pushed fresh Gantry image by immutable digest + prepare-gantry-pool + atomically claim and adopt one compatible prebuilt Gantry image preflight validate Azure sources, monitoring, Gantry, and all target nodes run execute baseline and Gantry cold phases, then restore routing run-gantry execute only Gantry cold against the retained baseline diff --git a/hack/gantry-benchmark/Makefile b/hack/gantry-benchmark/Makefile index a029b025e..5d8444cdf 100644 --- a/hack/gantry-benchmark/Makefile +++ b/hack/gantry-benchmark/Makefile @@ -2,7 +2,7 @@ REPO_ROOT := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))/../..) ENV_FILE ?= $(CURDIR)/env.local DEPLOY_CONFIG ?= $(CURDIR)/deploy.env -.PHONY: help test monitor deploy deploy-plan deploy-status operator-vm-check operator-vm-provision operator-vm-status operator-vm-watch proxy-image proxy-push enable prepare prepare-adopt prepare-gantry prepare-gantry-fresh prepare-gantry-adopt preflight run run-gantry status disable +.PHONY: help test monitor deploy deploy-plan deploy-status operator-vm-check operator-vm-provision operator-vm-status operator-vm-watch operator-vm-prebuild operator-vm-run-pool operator-vm-image-pool-status proxy-image proxy-push image-pool-status prebuild-gantry enable prepare prepare-adopt prepare-gantry prepare-gantry-fresh prepare-gantry-adopt prepare-gantry-pool preflight run run-gantry status disable help: ## Show benchmark targets @echo "" @@ -17,14 +17,20 @@ help: ## Show benchmark targets @echo " operator-vm-provision Provision/bootstrap the private operator VM" @echo " operator-vm-status Print one live operator VM progress snapshot" @echo " operator-vm-watch Follow operator VM progress until completion" + @echo " operator-vm-prebuild Start an asynchronous operator-VM pool build" + @echo " operator-vm-run-pool Start Gantry-only using the next compatible pool image" + @echo " operator-vm-image-pool-status Show operator-VM pool builder and entries" @echo " proxy-image Build BENCHMARK_PROXY_IMAGE" @echo " proxy-push Push BENCHMARK_PROXY_IMAGE" + @echo " image-pool-status List ready and claimed prebuilt Gantry images" + @echo " prebuild-gantry Build GANTRY_IMAGE_POOL_COUNT reusable Gantry images" @echo " enable Install benchmark instrumentation" @echo " prepare Build and push both digest-pinned workload images" @echo " prepare-adopt Adopt already-pushed direct-mode workload images" @echo " prepare-gantry Build only a cache-cold Gantry image from a retained baseline" @echo " prepare-gantry-fresh Build only Gantry with a brand-new random payload" @echo " prepare-gantry-adopt Adopt an already-pushed fresh Gantry image digest" + @echo " prepare-gantry-pool Claim one compatible prebuilt Gantry image" @echo " preflight Validate ACR, monitoring, Gantry, and all target nodes" @echo " run Execute baseline and Gantry cold phases" @echo " run-gantry Execute only Gantry cold against the retained baseline" @@ -43,7 +49,7 @@ monitor: cd "$(REPO_ROOT)" && GOTOOLCHAIN=auto go run ./hack/cmd/gantry-benchmark-monitor $(MONITOR_ARGS) operator-vm-check: - bash -n deploy.sh operator-vm-provision.sh operator-vm-bootstrap.sh operator-vm-build-images.sh operator-vm-run.sh operator-vm-status.sh operator-vm-watch.sh + bash -n deploy.sh operator-vm-provision.sh operator-vm-bootstrap.sh operator-vm-build-images.sh operator-vm-run.sh operator-vm-prebuild-images.sh operator-vm-image-pool.sh operator-vm-status.sh operator-vm-watch.sh ./deploy.sh plan deploy.env.example >/dev/null ! grep -Eq 'az acr login|podman (build|push|login|pull|tag)' deploy.sh ! grep -Eq '^[[:space:]]*az login([[:space:]]|$$)' deploy.sh @@ -67,6 +73,17 @@ operator-vm-status: operator-vm-check operator-vm-watch: operator-vm-check cd "$(REPO_ROOT)" && hack/gantry-benchmark/operator-vm-watch.sh --follow +operator-vm-prebuild: operator-vm-check + @test -n "$(GANTRY_IMAGE_POOL_COUNT)" || { echo "GANTRY_IMAGE_POOL_COUNT is required" >&2; exit 2; } + cd "$(REPO_ROOT)" && hack/gantry-benchmark/operator-vm-image-pool.sh start "$(GANTRY_IMAGE_POOL_COUNT)" + +operator-vm-run-pool: operator-vm-check + @test -n "$(GANTRY_ONLY_BASELINE_RUN_ID)" || { echo "GANTRY_ONLY_BASELINE_RUN_ID is required" >&2; exit 2; } + cd "$(REPO_ROOT)" && hack/gantry-benchmark/operator-vm-image-pool.sh run "$(GANTRY_ONLY_BASELINE_RUN_ID)" + +operator-vm-image-pool-status: operator-vm-check + cd "$(REPO_ROOT)" && hack/gantry-benchmark/operator-vm-image-pool.sh status + proxy-image: set -a; \ [ ! -f "$(ENV_FILE)" ] || . "$(ENV_FILE)"; \ @@ -91,6 +108,19 @@ proxy-push: --username "$${ACR_USERNAME}" --password-stdin "$${ACR_LOGIN_SERVER}"; \ "$${CONTAINER_ENGINE:-podman}" push "$${BENCHMARK_PROXY_IMAGE}" +image-pool-status: + set -a; \ + [ ! -f "$(ENV_FILE)" ] || . "$(ENV_FILE)"; \ + set +a; \ + cd "$(REPO_ROOT)" && GOTOOLCHAIN=auto go run ./hack/cmd/gantry-benchmark image-pool-status + +prebuild-gantry: + @test -n "$(GANTRY_IMAGE_POOL_COUNT)" || { echo "GANTRY_IMAGE_POOL_COUNT is required" >&2; exit 2; } + set -a; \ + [ ! -f "$(ENV_FILE)" ] || . "$(ENV_FILE)"; \ + set +a; \ + cd "$(REPO_ROOT)" && GOTOOLCHAIN=auto go run ./hack/cmd/gantry-benchmark prebuild-gantry "$(GANTRY_IMAGE_POOL_COUNT)" + enable prepare preflight run run-gantry status disable: set -a; \ [ ! -f "$(ENV_FILE)" ] || . "$(ENV_FILE)"; \ @@ -131,4 +161,12 @@ prepare-gantry-adopt: [ ! -f "$(ENV_FILE)" ] || . "$(ENV_FILE)"; \ set +a; \ cd "$(REPO_ROOT)" && GOTOOLCHAIN=auto go run ./hack/cmd/gantry-benchmark prepare-gantry-adopt \ - "$(GANTRY_ONLY_BASELINE_RUN_ID)" "$(GANTRY_ONLY_ADOPT_IMAGE)" "$(GANTRY_ONLY_ADOPT_PAYLOAD_SHA256)" \ No newline at end of file + "$(GANTRY_ONLY_BASELINE_RUN_ID)" "$(GANTRY_ONLY_ADOPT_IMAGE)" "$(GANTRY_ONLY_ADOPT_PAYLOAD_SHA256)" + +prepare-gantry-pool: + @test -n "$(GANTRY_ONLY_BASELINE_RUN_ID)" || { echo "GANTRY_ONLY_BASELINE_RUN_ID is required" >&2; exit 2; } + set -a; \ + [ ! -f "$(ENV_FILE)" ] || . "$(ENV_FILE)"; \ + set +a; \ + cd "$(REPO_ROOT)" && GOTOOLCHAIN=auto go run ./hack/cmd/gantry-benchmark prepare-gantry-pool \ + "$(GANTRY_ONLY_BASELINE_RUN_ID)" diff --git a/hack/gantry-benchmark/README.md b/hack/gantry-benchmark/README.md index e4c740024..13d7aaca5 100644 --- a/hack/gantry-benchmark/README.md +++ b/hack/gantry-benchmark/README.md @@ -226,13 +226,86 @@ It redraws every second and shows per-phase-minute peer outcomes (`busy`, and per-node throughput, cumulative payload percentage, and live Pod counts for completed, running, creating, image-pull failures, and failed Pods. Pod counts come from one Kubernetes list followed by watch events rather than polling all -1000 Pod objects. The header uses ANSI emphasis on a TTY and remains plain when -redirected or piped. The monitor uses one server-side aggregated Prometheus -range query per refresh; it does not download per-pod metric series. Prometheus -scrapes Gantry every 10 seconds, so the screen updates each second while counter -values advance at scrape cadence. +1000 Pod objects. + +Two node-paged grids show the current image in detail. The layer-by-node grid +uses `.` for pending and `0-9/A-Z` for the phase minute when Gantry finished +writing that layer to the node's containerd client (`Z` means minute 35 or +later). The image-by-node grid uses `.` for not started, `0` for started, +`1-9` for unpacked-layer deciles, and `#` after containerd reports the image +unpacked. Node columns are sorted and identified by their three-character +suffix. The default page width follows the terminal, bounded to 16-96 nodes. +Select another page or a fixed width with, for example: + +```bash +make -C hack/gantry-benchmark monitor \ + MONITOR_ARGS="--node-page 2 --nodes-per-page 64" +``` + +The header uses ANSI emphasis on a TTY and remains plain when redirected or +piped. The monitor uses one server-side aggregated Prometheus range query per +display refresh and one exact-image progress query every 10 seconds. Prometheus +scrapes at 10-second cadence, so the screen updates each second while grid and +counter values advance at scrape cadence. Use `MONITOR_ARGS="--once --no-clear"` for a single non-interactive snapshot. +## Reusable Gantry image pool + +Gantry-only benchmarks can consume prebuilt images instead of generating, +building, and pushing 40 GiB during every benchmark lifecycle. Start a batch +on the operator VM from the workstation: + +```bash +AZURE_RESOURCE_GROUP=vapa-gantry-benchmark1 \ +OPERATOR_VM_NAME=gantry-benchmark-operator \ +GANTRY_IMAGE_POOL_COUNT=10 \ +make -C hack/gantry-benchmark operator-vm-prebuild +``` + +The command starts `gantry-benchmark-image-builder.service` asynchronously. +The builder authenticates its managed identity once, generates each random +payload sequentially, pushes each image to the Gantry ACR, records its +immutable digest and payload SHA-256, removes the local image, and deletes the +40 GiB build context before starting the next image. Durable pool metadata +lives under `/var/lib/gantry-benchmark/image-pool`; transient build data lives +on the `/opt/gantry-benchmark` build disk. + +Use the bounded status view while it runs: + +```bash +AZURE_RESOURCE_GROUP=vapa-gantry-benchmark1 \ +OPERATOR_VM_NAME=gantry-benchmark-operator \ +make -C hack/gantry-benchmark operator-vm-image-pool-status +``` + +Start a Gantry-only benchmark with the oldest compatible ready image: + +```bash +AZURE_RESOURCE_GROUP=vapa-gantry-benchmark1 \ +OPERATOR_VM_NAME=gantry-benchmark-operator \ +GANTRY_ONLY_BASELINE_RUN_ID=run-20260806-205719-51c38730 \ +make -C hack/gantry-benchmark operator-vm-run-pool +``` + +The benchmark restores the retained baseline metadata automatically, atomically +moves one image from `ready/` to `claimed/`, and adopts its immutable digest. +Claimed images are never offered to a later run, preserving the cache-cold +contract. Pool adoption performs no image build, push, or registry credential +exchange inside the benchmark lifecycle. + +Pool building and benchmark execution are mutually exclusive on one operator +VM. Both services hold the same lifecycle lock, and the builder also refuses +to run while a benchmark state exists. This is required for measurement +correctness: pushing another image to the Gantry ACR during a measured phase +would produce unrelated repository events and invalidate the Azure telemetry +completeness gate. Build the 5-10 image batch before starting benchmark runs, +not concurrently with them. + +For direct CLI use with an already-authenticated container engine, the +equivalent targets are `prebuild-gantry`, `image-pool-status`, and +`prepare-gantry-pool`. Configure metadata and scratch locations with +`BENCHMARK_IMAGE_POOL_ROOT` and `BENCHMARK_IMAGE_POOL_BUILD_ROOT`. + Artifacts persist on the VM under `/var/lib/gantry-benchmark/artifacts//`; `latest` points at the newest run. By default the operator is a `Standard_D32ds_v5` with a dedicated 512 GiB diff --git a/hack/gantry-benchmark/operator-vm-bootstrap.sh b/hack/gantry-benchmark/operator-vm-bootstrap.sh index a84df5267..829e1fed1 100755 --- a/hack/gantry-benchmark/operator-vm-bootstrap.sh +++ b/hack/gantry-benchmark/operator-vm-bootstrap.sh @@ -193,6 +193,12 @@ require_private_resolution "$gantry_acr_name.$gantry_location.data.azurecr.io" " repo_root="$build_mount/unbounded" source_description="$repo_url ($repo_branch)" +for lifecycle_service in gantry-benchmark-operator.service gantry-benchmark-image-builder.service; do + if systemctl is-active --quiet "$lifecycle_service"; then + echo "$lifecycle_service is active; finish it before refreshing the operator checkout" >&2 + exit 1 + fi +done if [[ -n "$source_image" ]]; then gantry_login_server=$(az acr show -g "$resource_group" -n "$gantry_acr_name" --query loginServer -o tsv) source_token=$(acr_access_token "$gantry_acr_name") @@ -271,6 +277,11 @@ BENCHMARK_REPO_ROOT="$repo_root" BENCHMARK_BUILD_MOUNT="$build_mount" BENCHMARK_ARTIFACT_ROOT="/var/lib/gantry-benchmark/artifacts" BENCHMARK_OPERATOR_HOME="/var/lib/gantry-benchmark" +BENCHMARK_IMAGE_POOL_ROOT="/var/lib/gantry-benchmark/image-pool" +BENCHMARK_IMAGE_POOL_BUILD_ROOT="$repo_root/tmp/gantry-benchmark/image-pool-build" +BENCHMARK_IMAGE_POOL_PROGRESS="/var/lib/gantry-benchmark/image-pool-progress.json" +BENCHMARK_IMAGE_POOL_LOG="/var/lib/gantry-benchmark/image-pool-builder.log" +BENCHMARK_LIFECYCLE_LOCK="/var/lib/gantry-benchmark/benchmark-lifecycle.lock" BENCHMARK_CONFIRM_CONTEXT="$aks_cluster" BENCHMARK_MODE="direct" @@ -335,6 +346,34 @@ TimeoutStopSec=45min WantedBy=multi-user.target UNIT +cat >/etc/systemd/system/gantry-benchmark-image-builder.service </etc/gantry-benchmark/image-pool.env <<'ENV' +GANTRY_IMAGE_POOL_COUNT="1" +ENV +fi + systemctl daemon-reload retry az aks get-credentials \ diff --git a/hack/gantry-benchmark/operator-vm-image-pool.sh b/hack/gantry-benchmark/operator-vm-image-pool.sh new file mode 100755 index 000000000..1cf327199 --- /dev/null +++ b/hack/gantry-benchmark/operator-vm-image-pool.sh @@ -0,0 +1,200 @@ +#!/usr/bin/env bash +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +set -Eeuo pipefail + +AZURE_RESOURCE_GROUP="${AZURE_RESOURCE_GROUP:-}" +OPERATOR_VM_NAME="${OPERATOR_VM_NAME:-gantry-benchmark-operator}" +OPERATOR_RUN_COMMAND_LOCK="${OPERATOR_RUN_COMMAND_LOCK:-${TMPDIR:-/tmp}/gantry-benchmark-${AZURE_RESOURCE_GROUP}-${OPERATOR_VM_NAME}.run-command.lock}" + +usage() { + cat <<'USAGE' +Usage: operator-vm-image-pool.sh start COUNT + operator-vm-image-pool.sh run BASELINE_RUN_ID + operator-vm-image-pool.sh status + +Starts an asynchronous operator-VM image-pool build, or prints bounded status. +Pool builds and benchmark runs are mutually exclusive because pushes to the +Gantry ACR during a measured phase invalidate Azure telemetry. +USAGE +} + +remote_status_marker="GANTRY_BENCHMARK_REMOTE_STATUS=" + +invoke_remote() { + local body=$1 + local output + local remote_status + local transport_status + local wrapped_script + + wrapped_script="#!/usr/bin/env bash +set +e +( +$body +) +gantry_benchmark_remote_status=\$? +printf '${remote_status_marker}%s\\n' \"\$gantry_benchmark_remote_status\" +exit 0" + + if output=$(az vm run-command invoke \ + -g "$AZURE_RESOURCE_GROUP" \ + -n "$OPERATOR_VM_NAME" \ + --command-id RunShellScript \ + --scripts "$wrapped_script" \ + --only-show-errors \ + --query 'value[0].message' \ + -o tsv); then + : + else + transport_status=$? + return "$transport_status" + fi + + output=${output//$'\r'/} + remote_status=$(sed -n "s/^${remote_status_marker}\\([0-9][0-9]*\\)$/\\1/p" <<<"$output" | tail -1) + printf '%s\n' "$output" | sed \ + -e '/^Enable succeeded: *$/d' \ + -e '/^\[stdout\]$/d' \ + -e '/^\[stderr\]$/d' \ + -e "/^${remote_status_marker}[0-9][0-9]*$/d" + + if [[ ! "$remote_status" =~ ^[0-9]+$ ]]; then + echo "operator VM command did not return a remote exit status" >&2 + return 1 + fi + + if ((remote_status != 0)); then + echo "operator VM command failed with exit code $remote_status" >&2 + return "$remote_status" + fi +} + +(($# >= 1)) || { usage >&2; exit 2; } +action=$1 +shift + +: "${AZURE_RESOURCE_GROUP:?Set AZURE_RESOURCE_GROUP}" + +exec {run_command_lock_fd}>"$OPERATOR_RUN_COMMAND_LOCK" +flock "$run_command_lock_fd" +trap 'flock -u "$run_command_lock_fd"' EXIT + +case "$action" in + start) + (($# == 1)) || { usage >&2; exit 2; } + count=$1 + [[ "$count" =~ ^[1-9][0-9]*$ ]] || { echo "COUNT must be a positive integer" >&2; exit 2; } + ((count <= 100)) || { echo "COUNT must not exceed 100" >&2; exit 2; } + + script=$(cat <