diff --git a/.dockerignore b/.dockerignore index 48b3a483b..2a3dae4c3 100644 --- a/.dockerignore +++ b/.dockerignore @@ -46,6 +46,22 @@ README.md # Local env files .envrc +.env +.env.* +**/.env +**/.env.* +!**/.env.example +*.kubeconfig +**/*kubeconfig* +*.pem +*.key +**/id_rsa +**/id_ed25519 +**/.azure/ +**/.aws/ +**/.config/gcloud/ +hack/gantry-benchmark/env.local +hack/gantry-benchmark/deploy.env # OS files .DS_Store diff --git a/.gitignore b/.gitignore index 141bb1c8f..3e355bf52 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ bin/ !cmd/unbounded-storage/src/bin/ /gantry +/gantry-benchmark /dist/ build/ inventory.db diff --git a/cmd/gantry/agent_byte_metrics_test.go b/cmd/gantry/agent_byte_metrics_test.go index e700efdb5..565c3a51c 100644 --- a/cmd/gantry/agent_byte_metrics_test.go +++ b/cmd/gantry/agent_byte_metrics_test.go @@ -13,6 +13,7 @@ func TestByteMetricFamiliesMaterializeBoundedLabelsAtStartup(t *testing.T) { reg := metrics.New() _ = newPhase1Metrics(reg) _ = newPhase2Metrics(reg) + _ = newPhase9Metrics(reg) families, err := reg.PrometheusRegistry().Gather() if err != nil { @@ -25,10 +26,19 @@ func TestByteMetricFamiliesMaterializeBoundedLabelsAtStartup(t *testing.T) { } want := map[string]int{ - "gantry_origin_bytes_total": 3, - "gantry_peer_fetch_bytes_total": 3, - "gantry_peer_serve_bytes_total": 3, - "gantry_mirror_bytes_served_total": 9, + "gantry_origin_bytes_total": 3, + "gantry_peer_fetch_bytes_total": 3, + "gantry_peer_serve_bytes_total": 3, + "gantry_mirror_bytes_served_total": 9, + "gantry_mirror_response_completed_timestamp_seconds": 9, + "p2p_peer_fetch_total": 10, + "gantry_peer_fetch_last_timestamp_seconds": 2, + "p2p_peer_fetch_duration_seconds": 10, + "p2p_dht_lookup_total": 4, + "p2p_dht_lookup_duration_seconds": 4, + "gantry_containerd_commit_observed_timestamp_seconds": 1, + "gantry_containerd_commit_observation_duration_seconds": 1, + "gantry_containerd_commit_latest_observation_duration_seconds": 1, } for name, wantSeries := range want { diff --git a/cmd/gantry/agent_metrics.go b/cmd/gantry/agent_metrics.go index 5c1412dcb..2af0ff2ee 100644 --- a/cmd/gantry/agent_metrics.go +++ b/cmd/gantry/agent_metrics.go @@ -12,12 +12,17 @@ package main // dependencies between subsystems instead of declaring metric metadata. import ( + "strconv" + "sync" "sync/atomic" + "time" "github.com/prometheus/client_golang/prometheus" + "github.com/Azure/unbounded/internal/gantry/digest" "github.com/Azure/unbounded/internal/gantry/ifaces" "github.com/Azure/unbounded/internal/gantry/inflight" + "github.com/Azure/unbounded/internal/gantry/manifest" "github.com/Azure/unbounded/internal/gantry/metrics" ) @@ -76,22 +81,25 @@ func newPhase1Metrics(reg *metrics.Registry) *phase1Metrics { // phase2Metrics groups metrics for peer fallback, DHT advertise, // and transfer endpoint (the design doc). type phase2Metrics struct { - peerServe prometheus.Counter - peerServeBytes *prometheus.CounterVec - peerMiss prometheus.Counter - peerFetch *prometheus.CounterVec - peerFetchBytes *prometheus.CounterVec - mirrorServeBytes *prometheus.CounterVec - peerFetchDur *prometheus.HistogramVec - peerDialSuccess prometheus.Counter - peerDialFailure prometheus.Counter - dhtProvide prometheus.Counter - dhtProvideErr *prometheus.CounterVec - dhtReconcile prometheus.Counter - dhtLookup *prometheus.CounterVec - dhtLookupDur *prometheus.HistogramVec - dhtAdvertise prometheus.Counter - cdsubReconnect prometheus.Counter + peerServe prometheus.Counter + peerServeBytes *prometheus.CounterVec + peerMiss prometheus.Counter + peerFetch *prometheus.CounterVec + peerFetchLastAt *prometheus.GaugeVec + peerFetchBytes *prometheus.CounterVec + mirrorServeBytes *prometheus.CounterVec + mirrorCompletedAt *prometheus.GaugeVec + layerCompletedAt *prometheus.GaugeVec + peerFetchDur *prometheus.HistogramVec + peerDialSuccess prometheus.Counter + peerDialFailure prometheus.Counter + dhtProvide prometheus.Counter + dhtProvideErr *prometheus.CounterVec + dhtReconcile prometheus.Counter + dhtLookup *prometheus.CounterVec + dhtLookupDur *prometheus.HistogramVec + dhtAdvertise prometheus.Counter + cdsubReconnect prometheus.Counter } func newPhase2Metrics(reg *metrics.Registry) *phase2Metrics { @@ -112,6 +120,10 @@ func newPhase2Metrics(reg *metrics.Registry) *phase2Metrics { Name: "p2p_peer_fetch_total", Help: "Peer fetches initiated by the mirror miss path.", }, []string{"outcome"}), + peerFetchLastAt: reg.NewGaugeVec("mirror", prometheus.GaugeOpts{ + Name: "gantry_peer_fetch_last_timestamp_seconds", + Help: "Unix timestamp of the most recent peer fetch event, retained for busy and stall outcomes.", + }, []string{"outcome"}), peerFetchBytes: reg.NewCounterVec("mirror", prometheus.CounterOpts{ Name: "gantry_peer_fetch_bytes_total", Help: "Bytes received from peer Gantry agents, including partial failed transfers and retries, labeled by OCI content kind.", @@ -120,6 +132,14 @@ func newPhase2Metrics(reg *metrics.Registry) *phase2Metrics { Name: "gantry_mirror_bytes_served_total", Help: "Bytes written by the containerd-facing mirror, labeled by OCI content kind and source path (cache, peer, or origin).", }, []string{"kind", "source"}), + mirrorCompletedAt: reg.NewGaugeVec("mirror", prometheus.GaugeOpts{ + Name: "gantry_mirror_response_completed_timestamp_seconds", + Help: "Unix timestamp when a complete response body was most recently written to the local containerd client, labeled by content kind and source path.", + }, []string{"kind", "source"}), + layerCompletedAt: reg.NewGaugeVec("mirror", prometheus.GaugeOpts{ + Name: "gantry_layer_download_completed_timestamp_seconds", + Help: "Unix timestamp when a current-image layer response completed to the local containerd client. Zero means pending. Labels are bounded to the current manifest and deleted when the manifest changes.", + }, []string{"node", "image_digest", "layer_digest", "layer_index"}), peerFetchDur: reg.NewHistogramVec("mirror", prometheus.HistogramOpts{ Name: "p2p_peer_fetch_duration_seconds", Help: "End-to-end peer-fetch latency from FetchFromPeer dial to terminal outcome (hit = cache commit, error/stall/notfound = first failing branch). Together with p2p_peer_fetch_total{outcome} this isolates dial vs. body vs. commit-time-digest-verification slowness.", @@ -170,12 +190,129 @@ func newPhase2Metrics(reg *metrics.Registry) *phase2Metrics { for _, source := range []string{"cache", "peer", "origin"} { p.mirrorServeBytes.WithLabelValues(kind, source).Add(0) + p.mirrorCompletedAt.WithLabelValues(kind, source).Set(0) } } + for _, outcome := range []string{ + "hit", + "notfound", + "unavailable", + "digest_mismatch", + "auth_or_config", + "server_error", + "protocol_error", + "stall", + "local_error", + "busy", + } { + p.peerFetch.WithLabelValues(outcome).Add(0) + p.peerFetchDur.WithLabelValues(outcome) + } + + for _, outcome := range []string{"busy", "stall"} { + p.peerFetchLastAt.WithLabelValues(outcome).Set(0) + } + + for _, outcome := range []string{"hit", "miss", "error", "timeout"} { + p.dhtLookup.WithLabelValues(outcome).Add(0) + p.dhtLookupDur.WithLabelValues(outcome) + } + return p } +type layerProgressTracker struct { + mu sync.Mutex + gauge *prometheus.GaugeVec + node string + now func() time.Time + manifest digest.Digest + layers map[digest.Digest]string + completedLayers map[digest.Digest]struct{} + earlyCompleted map[digest.Digest]time.Time + oldLabels [][]string +} + +const maxEarlyLayerCompletions = 256 + +func newLayerProgressTracker(gauge *prometheus.GaugeVec, node string, now func() time.Time) *layerProgressTracker { + return &layerProgressTracker{ + gauge: gauge, + node: node, + now: now, + layers: map[digest.Digest]string{}, + completedLayers: map[digest.Digest]struct{}{}, + earlyCompleted: map[digest.Digest]time.Time{}, + } +} + +func (t *layerProgressTracker) observeManifest(manifestDigest digest.Digest, children []manifest.TypedChild) { + t.mu.Lock() + defer t.mu.Unlock() + + if t.manifest == manifestDigest { + return + } + + for _, labels := range t.oldLabels { + t.gauge.DeleteLabelValues(labels...) + } + + t.manifest = manifestDigest + t.layers = make(map[digest.Digest]string, len(children)) + t.completedLayers = make(map[digest.Digest]struct{}, len(children)) + t.oldLabels = t.oldLabels[:0] + + layerIndex := 0 + + for _, child := range children { + if child.Kind != ifaces.KindBlob { + continue + } + + index := strconv.Itoa(layerIndex) + labels := []string{t.node, manifestDigest.String(), child.Digest.String(), index} + t.layers[child.Digest] = index + t.oldLabels = append(t.oldLabels, labels) + + completedAt, completed := t.earlyCompleted[child.Digest] + if completed { + t.gauge.WithLabelValues(labels...).Set(float64(completedAt.UnixNano()) / float64(time.Second)) + t.completedLayers[child.Digest] = struct{}{} + } else { + t.gauge.WithLabelValues(labels...).Set(0) + } + + layerIndex++ + } + + t.earlyCompleted = map[digest.Digest]time.Time{} +} + +func (t *layerProgressTracker) completed(d digest.Digest) { + t.mu.Lock() + defer t.mu.Unlock() + + index, ok := t.layers[d] + if !ok { + if _, exists := t.earlyCompleted[d]; !exists && len(t.earlyCompleted) < maxEarlyLayerCompletions { + t.earlyCompleted[d] = t.now() + } + + return + } + + if _, ok := t.completedLayers[d]; ok { + return + } + + t.gauge.WithLabelValues(t.node, t.manifest.String(), d.String(), index). + Set(float64(t.now().UnixNano()) / float64(time.Second)) + + t.completedLayers[d] = struct{}{} +} + // phase3Metrics groups the instruments owned by // HRW-rank-mismatch detection, DHT-false-empty observability, top-K // probe hit rate, in-flight pull gauge, cold-start latency, and coord // stream counters. @@ -194,6 +331,7 @@ type phase3Metrics struct { prefetchBatchesTotal prometheus.Counter prefetchDigestsTotal prometheus.Counter prefetchPullersPerBatch prometheus.Histogram + prefetchGroupsTotal *prometheus.CounterVec } func newPhase3Metrics(reg *metrics.Registry, infl *inflight.Map) *phase3Metrics { @@ -204,6 +342,17 @@ func newPhase3Metrics(reg *metrics.Registry, infl *inflight.Map) *phase3Metrics Help: "Current count of in-flight digest pulls on this node.", }, func() float64 { return float64(infl.Len()) }) + prefetchGroupsTotal := reg.NewCounterVec("coord", prometheus.CounterOpts{ + Name: "p2p_prefetch_groups_total", + Help: "Prefetch dispatch groups by local or remote target and success or error outcome.", + }, []string{"target", "outcome"}) + + for _, target := range []string{"local", "remote"} { + for _, outcome := range []string{"success", "error"} { + prefetchGroupsTotal.WithLabelValues(target, outcome).Add(0) + } + } + return &phase3Metrics{ hrwRankMismatch: reg.NewCounterVec("coord", prometheus.CounterOpts{ Name: "p2p_hrw_rank_mismatch_total", @@ -261,8 +410,9 @@ func newPhase3Metrics(reg *metrics.Registry, infl *inflight.Map) *phase3Metrics prefetchPullersPerBatch: reg.NewHistogram("coord", prometheus.HistogramOpts{ Name: "p2p_prefetch_pullers_per_manifest", Help: "Distribution of distinct HRW rank-0 pullers contacted per manifest pre-fan call.", - Buckets: prometheus.LinearBuckets(1, 1, 10), + Buckets: prometheus.ExponentialBuckets(1, 2, 11), }), + prefetchGroupsTotal: prefetchGroupsTotal, } } @@ -383,35 +533,38 @@ func newPhase5Metrics(reg *metrics.Registry, healthScore func() float64) *phase5 // in its content inventory" so the agent does not pretend a local // commit happened just because the HTTP stream completed. type phase9Metrics struct { - storageMode *prometheus.GaugeVec - advReconcileTotal prometheus.Counter - advReconcileError prometheus.Counter - advReconcileUnavailable prometheus.Counter - advReconcileDur prometheus.Histogram - advReconcileDigestCount prometheus.Gauge - advReconcileAdded prometheus.Counter - advReconcileRemoved prometheus.Counter - withdrawTotal prometheus.Counter - withdrawError prometheus.Counter - containerdLeaseCreated prometheus.Counter - containerdLeaseReleased prometheus.Counter - containerdLeaseActive prometheus.Gauge - containerdLeaseCleanupErr prometheus.Counter - containerdIngestTotal prometheus.Counter - containerdIngestFailure prometheus.Counter - containerdHit prometheus.Counter - containerdMiss prometheus.Counter - containerdUnavailable prometheus.Counter - containerdOpenError prometheus.Counter - originStreamStarted *prometheus.CounterVec - originStreamCompleted *prometheus.CounterVec - originStreamFailed *prometheus.CounterVec - containerdCommitObserved prometheus.Counter - dhtStaleOnly prometheus.Counter - staleProviderFiltered prometheus.Counter - commitMissingAfterStream prometheus.Counter - advertiseTotal prometheus.Counter - advertiseError prometheus.Counter + storageMode *prometheus.GaugeVec + advReconcileTotal prometheus.Counter + advReconcileError prometheus.Counter + advReconcileUnavailable prometheus.Counter + advReconcileDur prometheus.Histogram + advReconcileDigestCount prometheus.Gauge + advReconcileAdded prometheus.Counter + advReconcileRemoved prometheus.Counter + withdrawTotal prometheus.Counter + withdrawError prometheus.Counter + containerdLeaseCreated prometheus.Counter + containerdLeaseReleased prometheus.Counter + containerdLeaseActive prometheus.Gauge + containerdLeaseCleanupErr prometheus.Counter + containerdIngestTotal prometheus.Counter + containerdIngestFailure prometheus.Counter + containerdHit prometheus.Counter + containerdMiss prometheus.Counter + containerdUnavailable prometheus.Counter + containerdOpenError prometheus.Counter + originStreamStarted *prometheus.CounterVec + originStreamCompleted *prometheus.CounterVec + originStreamFailed *prometheus.CounterVec + containerdCommitObserved prometheus.Counter + containerdCommitObservedAt prometheus.Gauge + containerdCommitObserveDur prometheus.Histogram + containerdCommitLatestDur prometheus.Gauge + dhtStaleOnly prometheus.Counter + staleProviderFiltered prometheus.Counter + commitMissingAfterStream prometheus.Counter + advertiseTotal prometheus.Counter + advertiseError prometheus.Counter } func newPhase9Metrics(reg *metrics.Registry) *phase9Metrics { @@ -513,6 +666,19 @@ func newPhase9Metrics(reg *metrics.Registry) *phase9Metrics { Name: "gantry_containerd_commit_observed_total", Help: "Completed live stream-through responses whose digest later appeared in the local containerd inventory within the verification window. This is the truthful post-stream commit signal for live mirror traffic.", }), + containerdCommitObservedAt: reg.NewGauge("storage", prometheus.GaugeOpts{ + Name: "gantry_containerd_commit_observed_timestamp_seconds", + Help: "Unix timestamp when containerd inventory most recently showed a digest from a completed live stream-through response.", + }), + containerdCommitObserveDur: reg.NewHistogram("storage", prometheus.HistogramOpts{ + Name: "gantry_containerd_commit_observation_duration_seconds", + Help: "Time from a digest-verified live stream-through response completing to the digest appearing in containerd inventory. Resolution is bounded by the inventory probe interval.", + Buckets: prometheus.ExponentialBuckets(0.25, 2, 9), + }), + containerdCommitLatestDur: reg.NewGauge("storage", prometheus.GaugeOpts{ + Name: "gantry_containerd_commit_latest_observation_duration_seconds", + Help: "Most recent measured time from a digest-verified live stream-through response completing to the digest appearing in containerd inventory. Resolution is bounded by the inventory probe interval.", + }), dhtStaleOnly: reg.NewCounter("discovery", prometheus.CounterOpts{ Name: "gantry_dht_stale_only_total", Help: "Mirror cache-miss requests where the DHT returned candidate providers but every candidate was filtered (stale, suspicious, self, unavailable) before any peer fetch was attempted. Treated as if DHT returned empty - falls through to cold-start. Distinct from gantry_dht_lookup_total{outcome=\"miss\"} which counts true empty results.", diff --git a/cmd/gantry/agent_pprof.go b/cmd/gantry/agent_pprof.go new file mode 100644 index 000000000..1599bcf8a --- /dev/null +++ b/cmd/gantry/agent_pprof.go @@ -0,0 +1,50 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package main + +import ( + "errors" + "log/slog" + "net" + "net/http" + "net/http/pprof" + "time" +) + +func startPprofEndpoint(addr string, logger *slog.Logger) (*http.Server, <-chan error, error) { + listener, err := net.Listen("tcp", addr) + if err != nil { + return nil, nil, err + } + + mux := http.NewServeMux() + mux.HandleFunc("/debug/pprof/", pprof.Index) + mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) + mux.HandleFunc("/debug/pprof/profile", pprof.Profile) + mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) + mux.HandleFunc("/debug/pprof/trace", pprof.Trace) + + server := &http.Server{ + Addr: listener.Addr().String(), + Handler: mux, + ReadHeaderTimeout: 5 * time.Second, + } + errorsChannel := make(chan error, 1) + + go func() { + err := server.Serve(listener) + if !errors.Is(err, http.ErrServerClosed) { + errorsChannel <- err + } + + close(errorsChannel) + }() + + logger.Info("pprof endpoint listening", + slog.String("addr", listener.Addr().String()), + slog.String("path", "/debug/pprof/"), + ) + + return server, errorsChannel, nil +} diff --git a/cmd/gantry/agent_pprof_test.go b/cmd/gantry/agent_pprof_test.go new file mode 100644 index 000000000..dd2a88924 --- /dev/null +++ b/cmd/gantry/agent_pprof_test.go @@ -0,0 +1,75 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package main + +import ( + "context" + "errors" + "io" + "log/slog" + "net" + "net/http" + "strings" + "testing" + "time" +) + +func TestStartPprofEndpointServesRuntimeProfiles(t *testing.T) { + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + + server, errorsChannel, err := startPprofEndpoint("127.0.0.1:0", logger) + if err != nil { + t.Fatalf("startPprofEndpoint: %v", err) + } + + client := &http.Client{Timeout: 2 * time.Second} + + response, err := client.Get("http://" + server.Addr + "/debug/pprof/goroutine?debug=1") + if err != nil { + t.Fatalf("GET goroutine profile: %v", err) + } + + body, readErr := io.ReadAll(response.Body) + + closeErr := response.Body.Close() + if err := errors.Join(readErr, closeErr); err != nil { + t.Fatalf("read response: %v", err) + } + + if response.StatusCode != http.StatusOK || !strings.Contains(string(body), "goroutine profile") { + t.Fatalf("profile response: status=%d body=%q", response.StatusCode, body) + } + + shutdownContext, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + + if err := server.Shutdown(shutdownContext); err != nil { + t.Fatalf("Shutdown: %v", err) + } + + select { + case serveErr, ok := <-errorsChannel: + if ok && serveErr != nil { + t.Fatalf("Serve: %v", serveErr) + } + case <-shutdownContext.Done(): + t.Fatal("pprof server did not stop") + } +} + +func TestStartPprofEndpointReportsBindError(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("Listen: %v", err) + } + + defer func() { _ = listener.Close() }() + + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + + server, errorsChannel, err := startPprofEndpoint(listener.Addr().String(), logger) + if err == nil || server != nil || errorsChannel != nil { + t.Fatalf("startPprofEndpoint = %#v, %#v, %v; want bind error", server, errorsChannel, err) + } +} diff --git a/cmd/gantry/agent_shutdown.go b/cmd/gantry/agent_shutdown.go index 1093b7cc5..6140ed66b 100644 --- a/cmd/gantry/agent_shutdown.go +++ b/cmd/gantry/agent_shutdown.go @@ -27,6 +27,7 @@ type shutdownDeps struct { coordStop func() pullerPumpGate *pullerPumpGate metricsHTTP *http.Server + pprofHTTP *http.Server shutdownBudget time.Duration } @@ -42,8 +43,8 @@ type shutdownDeps struct { // handlers up to the shutdown deadline. // 4. Wait for cdsub.Run + outstanding pull-pump advertise calls to // flush before libp2p is closed by the runAgent defer chain. -// 5. Ops endpoint (Shutdown) - last so /readyz can keep reporting -// NotReady while we drain. +// 5. Profiling endpoint, then ops endpoint (Shutdown) - ops stays last so +// /readyz can keep reporting NotReady while we drain. // // discovery.Close + members.Stop run from the runAgent defer chain // after this returns. @@ -99,6 +100,12 @@ func gracefulShutdown(d shutdownDeps) { d.logger.Warn("puller-pump did not drain within shutdown budget") } + if d.pprofHTTP != nil { + if err := d.pprofHTTP.Shutdown(shutdownCtx); err != nil { + d.logger.Warn("pprof shutdown error", slog.Any("err", err)) + } + } + if err := d.metricsHTTP.Shutdown(shutdownCtx); err != nil { d.logger.Warn("metrics shutdown error", slog.Any("err", err)) } diff --git a/cmd/gantry/layer_progress_test.go b/cmd/gantry/layer_progress_test.go new file mode 100644 index 000000000..e4cc5b477 --- /dev/null +++ b/cmd/gantry/layer_progress_test.go @@ -0,0 +1,97 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package main + +import ( + "strings" + "testing" + "time" + + "github.com/Azure/unbounded/internal/gantry/digest" + "github.com/Azure/unbounded/internal/gantry/ifaces" + "github.com/Azure/unbounded/internal/gantry/manifest" + "github.com/Azure/unbounded/internal/gantry/metrics" +) + +func TestLayerProgressTrackerBoundsSeriesToCurrentManifest(t *testing.T) { + registry := metrics.New() + phase := newPhase2Metrics(registry) + now := time.Unix(123, 500_000_000) + tracker := newLayerProgressTracker(phase.layerCompletedAt, "node-a", func() time.Time { return now }) + + manifestA := digest.MustParse("sha256:" + strings.Repeat("a", 64)) + configA := digest.MustParse("sha256:" + strings.Repeat("b", 64)) + layerA0 := digest.MustParse("sha256:" + strings.Repeat("c", 64)) + layerA1 := digest.MustParse("sha256:" + strings.Repeat("d", 64)) + + tracker.completed(layerA0) + + tracker.observeManifest(manifestA, []manifest.TypedChild{ + {Digest: configA, Kind: ifaces.KindConfig}, + {Digest: layerA0, Kind: ifaces.KindBlob}, + {Digest: layerA1, Kind: ifaces.KindBlob}, + }) + tracker.completed(layerA1) + + now = time.Unix(200, 0) + + tracker.completed(layerA1) + + series := layerProgressSeries(t, registry) + if len(series) != 2 { + t.Fatalf("series after first manifest = %d, want 2", len(series)) + } + + if got := series[layerA0.String()]; got != 123.5 { + t.Fatalf("early completed layer value = %v, want 123.5", got) + } + + if got := series[layerA1.String()]; got != 123.5 { + t.Fatalf("completed layer value = %v, want 123.5", got) + } + + manifestB := digest.MustParse("sha256:" + strings.Repeat("e", 64)) + layerB0 := digest.MustParse("sha256:" + strings.Repeat("f", 64)) + tracker.observeManifest(manifestB, []manifest.TypedChild{{Digest: layerB0, Kind: ifaces.KindBlob}}) + + series = layerProgressSeries(t, registry) + if len(series) != 1 { + t.Fatalf("series after manifest replacement = %d, want 1", len(series)) + } + + if _, ok := series[layerB0.String()]; !ok { + t.Fatalf("current layer %s missing from series %v", layerB0, series) + } +} + +func layerProgressSeries(t *testing.T, registry *metrics.Registry) map[string]float64 { + t.Helper() + + families, err := registry.PrometheusRegistry().Gather() + if err != nil { + t.Fatalf("Gather: %v", err) + } + + result := map[string]float64{} + + for _, family := range families { + if family.GetName() != "gantry_layer_download_completed_timestamp_seconds" { + continue + } + + for _, metric := range family.Metric { + digestLabel := "" + + for _, label := range metric.Label { + if label.GetName() == "layer_digest" { + digestLabel = label.GetValue() + } + } + + result[digestLabel] = metric.GetGauge().GetValue() + } + } + + return result +} diff --git a/cmd/gantry/main.go b/cmd/gantry/main.go index 71f3ab5b6..ca6b440cb 100644 --- a/cmd/gantry/main.go +++ b/cmd/gantry/main.go @@ -17,6 +17,7 @@ import ( "io" "log/slog" "net" + "net/http" "os" "os/signal" "runtime" @@ -28,6 +29,7 @@ import ( "time" "github.com/libp2p/go-libp2p/core/peer" + "github.com/libp2p/go-libp2p/core/peerstore" "github.com/multiformats/go-multiaddr" "github.com/Azure/unbounded/internal/gantry/advertise" @@ -148,6 +150,7 @@ func runAgent(args []string) error { reg.RegisterDefaultCollectors() inst := newPhase1Metrics(reg) p2 := newPhase2Metrics(reg) + layerProgress := newLayerProgressTracker(p2.layerCompletedAt, c.NodeName, time.Now) p9 := newPhase9Metrics(reg) // Storage mode info: emit a single time-series at 1 for the // active backend so dashboards can filter by it. @@ -384,7 +387,7 @@ func runAgent(args []string) error { // annotation (the design doc) which Members reads in Snapshot. This // lets the cluster use stable K8s node names as NodeIDs // while still dialing libp2p RPCs to the right peer. - coord.WithPeerIDResolver(membershipPeerIDResolver(memberView, logger)), + coord.WithPeerIDResolver(membershipPeerIDResolver(memberView, disco.LibP2P().Peerstore(), logger)), ) // pullerPump bridges inbound please_pull RPCs to the local origin // puller (the step 7). The pump itself MUST NOT block the coord @@ -453,21 +456,24 @@ func runAgent(args []string) error { if hasMultiNodeMembership(memberView) { selfZone := lookupSelfZone(memberView) realResolver := coldstart.New(coldstart.Options{ - Members: memberView, - Discovery: disco, - Coord: coordClient, - Inflight: inflightMap, - Logger: logger, - HrwK: c.HRWK, - HrwScope: hrw.ParseScope(c.HRWTopologyScope), - SelfZone: selfZone, - LocalIntent: coordServer, - LocalPull: coordServer, - PrefetchPullerReplicas: c.PrefetchPullerReplicas, - PrefetchPullerFraction: c.PrefetchPullerFraction, - TransientCooldownCap: c.OriginFailureHonorWindowCap, - TopKExpansionFactor: c.TopKExpansionFactorDegraded, - TrustedFailureClasses: parseTrustedFailureClasses(c.OriginFailureClassesTrustedClusterWide, logger), + Members: memberView, + Discovery: disco, + Coord: coordClient, + Inflight: inflightMap, + Logger: logger, + HrwK: c.HRWK, + HrwScope: hrw.ParseScope(c.HRWTopologyScope), + SelfZone: selfZone, + LocalIntent: coordServer, + LocalPull: coordServer, + PrefetchPullerReplicas: c.PrefetchPullerReplicas, + PrefetchPullerFraction: c.PrefetchPullerFraction, + PrefetchCoordinatorReplicas: c.PrefetchCoordinatorReplicas, + PrefetchMaxConcurrentGroups: c.PrefetchMaxConcurrentGroups, + PrefetchDispatchJitter: c.PrefetchDispatchJitter, + TransientCooldownCap: c.OriginFailureHonorWindowCap, + TopKExpansionFactor: c.TopKExpansionFactorDegraded, + TrustedFailureClasses: parseTrustedFailureClasses(c.OriginFailureClassesTrustedClusterWide, logger), Metrics: coldstart.MetricsHooks{ OnRankMismatch: func(kindLabel string, _ ifaces.NodeID) { p3.hrwRankMismatch.WithLabelValues(kindLabel).Inc() @@ -488,10 +494,13 @@ func runAgent(args []string) error { p3.prefetchDigestsTotal.Add(float64(digests)) p3.prefetchPullersPerBatch.Observe(float64(pullers)) }, + OnPrefetchGroup: func(target, outcome string) { + p3.prefetchGroupsTotal.WithLabelValues(target, outcome).Inc() + }, }, }) coldStartResolver = coldStartAdapter{r: realResolver} - layerPrefetcher = newLayerPrefetcher(realResolver, cstore, logger) + layerPrefetcher = newLayerPrefetcher(realResolver, cstore, logger, layerProgress.observeManifest) logger.Info("cold-start orchestrator wired", slog.Int("hrw_k", c.HRWK), slog.String("hrw_scope", c.HRWTopologyScope), @@ -500,6 +509,10 @@ func runAgent(args []string) error { logger.Info("cold-start orchestrator disabled (single-self membership; no Kubernetes informer)") } + if layerPrefetcher == nil { + layerPrefetcher = newLayerPrefetcher(nil, cstore, logger, layerProgress.observeManifest) + } + // - direct-origin-fallback direct-origin fallback controller (the design doc). Wired // only when the cold-start resolver is also wired; without // orchestration there is no `ErrColdStartExhausted` path to gate. @@ -563,6 +576,11 @@ func runAgent(args []string) error { streamCommitTracker := newStreamCommitTracker(containerdInv, logger, func(n int) { p9.containerdCommitObserved.Add(float64(n)) }, + func(duration time.Duration) { + p9.containerdCommitObserveDur.Observe(duration.Seconds()) + p9.containerdCommitLatestDur.Set(duration.Seconds()) + p9.containerdCommitObservedAt.SetToCurrentTime() + }, func(n int) { p9.commitMissingAfterStream.Add(float64(n)) }, ) @@ -588,6 +606,10 @@ func runAgent(args []string) error { p2.mirrorServeBytes.WithLabelValues(kind, source).Add(float64(bytes)) }, ), + mirror.WithMirrorResponseCompletedHook(func(d digest.Digest, kind, source string) { + p2.mirrorCompletedAt.WithLabelValues(kind, source).SetToCurrentTime() + layerProgress.completed(d) + }), mirror.WithOriginStreamMetrics( func(kind string) { p9.originStreamStarted.WithLabelValues(kind).Inc() }, func(kind string) { p9.originStreamCompleted.WithLabelValues(kind).Inc() }, @@ -609,7 +631,13 @@ func runAgent(args []string) error { mirror.WithSelfNodeID(memberView.Self()), mirror.WithSelfPeerID(ifaces.NodeID(disco.PeerID().String())), mirror.WithPeerMetrics( - func(outcome string) { p2.peerFetch.WithLabelValues(outcome).Inc() }, + func(outcome string) { + p2.peerFetch.WithLabelValues(outcome).Inc() + + if outcome == "busy" || outcome == "stall" { + p2.peerFetchLastAt.WithLabelValues(outcome).SetToCurrentTime() + } + }, func(success bool) { if success { p2.peerDialSuccess.Inc() @@ -949,7 +977,25 @@ func runAgent(args []string) error { // agent_readiness.go for the full handler wiring. metricsHTTP, metricsErr := startOpsEndpoint(c.MetricsListen, reg, readyCheck, logger) - // Block until signal or metrics-server crash. + var pprofHTTP *http.Server + + if c.PprofListen != "" { + startedPprofHTTP, pprofErr, pprofListenErr := startPprofEndpoint(c.PprofListen, logger) + if pprofListenErr != nil { + logger.Warn("pprof endpoint unavailable", slog.Any("err", pprofListenErr)) + } else { + pprofHTTP = startedPprofHTTP + + go func() { + if pprofServeErr, ok := <-pprofErr; ok && pprofServeErr != nil { + logger.Warn("pprof endpoint died", slog.Any("err", pprofServeErr)) + } + }() + } + } + + // Block until signal or an essential background server crashes. Profiling + // is diagnostic and never owns data-plane availability. select { case <-ctx.Done(): logger.Info("shutdown signal received") @@ -971,6 +1017,7 @@ func runAgent(args []string) error { coordStop: func() { coordServer.Unbind(disco.LibP2P()) }, pullerPumpGate: pullerPumpGate, metricsHTTP: metricsHTTP, + pprofHTTP: pprofHTTP, shutdownBudget: 10 * time.Second, }) logger.Info("gantry stopped") @@ -1274,13 +1321,9 @@ func transferPortFromListen(listen string) int { return n } -// membershipPeerIDResolver returns a coord.WithPeerIDResolver callback -// that consults the live members snapshot. NodeID -> Node.PeerID is the -// fast path; on miss the resolver returns (_, false) so coord.Client -// falls through to its static teach-cache and finally to -// peer.Decode(NodeID). The membership view is read on every call (cheap -// in-memory copy) so newly-joined peers are picked up without restart. -func membershipPeerIDResolver(mv ifaces.Members, logger *slog.Logger) func(ifaces.NodeID) (peer.ID, bool) { +// membershipPeerIDResolver also installs the target's Pod-IP addresses before +// direct coordination RPCs dial it; DHT bootstrap does not populate every peer. +func membershipPeerIDResolver(mv ifaces.Members, ps peerstore.Peerstore, logger *slog.Logger) func(ifaces.NodeID) (peer.ID, bool) { return func(id ifaces.NodeID) (peer.ID, bool) { for _, n := range mv.Snapshot() { if n.ID != id || n.PeerID == "" { @@ -1300,6 +1343,42 @@ func membershipPeerIDResolver(mv ifaces.Members, logger *slog.Logger) func(iface return "", false } + var addrs []multiaddr.Multiaddr + + for _, raw := range n.P2PAddrs { + info, err := peer.AddrInfoFromString(raw) + if err != nil { + if logger != nil { + logger.Debug("membership peer address decode failed", + slog.String("node_id", string(id)), + slog.String("address", raw), + slog.Any("err", err), + ) + } + + continue + } + + if info.ID != pid { + if logger != nil { + logger.Warn("membership peer address identity mismatch", + slog.String("node_id", string(id)), + slog.String("peer_id", pid.String()), + slog.String("address_peer_id", info.ID.String()), + ) + } + + continue + } + + addrs = append(addrs, info.Addrs...) + } + + if ps != nil && len(addrs) > 0 { + ps.ClearAddrs(pid) + ps.AddAddrs(pid, addrs, peerstore.AddressTTL) + } + return pid, true } @@ -1850,9 +1929,10 @@ func (a coldStartAdapter) Resolve(ctx context.Context, d digest.Digest, kind ifa // The implementation runs in a goroutine spawned by the mirror; it // MUST NOT panic. All errors are logged at DEBUG. type layerPrefetchAdapter struct { - resolver *coldstart.Resolver - cache ifaces.LocalContentStore - logger *slog.Logger + resolver *coldstart.Resolver + cache ifaces.LocalContentStore + logger *slog.Logger + onManifest func(digest.Digest, []manifest.TypedChild) } // maxManifestBytes caps the size of a manifest body the prefetcher @@ -1911,16 +1991,59 @@ func advertiseOnCommit(ctx context.Context, adv *advertise.Advertiser, store ifa } } -func newLayerPrefetcher(r *coldstart.Resolver, cache ifaces.LocalContentStore, logger *slog.Logger) mirror.LayerPrefetcher { +func newLayerPrefetcher( + r *coldstart.Resolver, + cache ifaces.LocalContentStore, + logger *slog.Logger, + onManifest func(digest.Digest, []manifest.TypedChild), +) mirror.LayerPrefetcher { return &layerPrefetchAdapter{ - resolver: r, - cache: cache, - logger: logger.With(slog.String("subsystem", "prefetch")), + resolver: r, + cache: cache, + logger: logger.With(slog.String("subsystem", "prefetch")), + onManifest: onManifest, } } +// openManifest reads the manifest body from the shared content store. Under +// live stream-through the mirror proxies the body straight to containerd, so +// it only appears once containerd commits it; retry briefly rather than +// dropping the prefetch and losing cold-start seeding entirely. +func (p *layerPrefetchAdapter) openManifest(ctx context.Context, d digest.Digest) (io.ReadCloser, error) { + const attempts = 8 + + delay := 100 * time.Millisecond + + var lastErr error + + for attempt := range attempts { + rc, _, err := p.cache.Open(ctx, d) + if err == nil { + return rc, nil + } + + lastErr = err + + if attempt == attempts-1 { + break + } + + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(delay): + } + + if delay < time.Second { + delay *= 2 + } + } + + return nil, lastErr +} + func (p *layerPrefetchAdapter) OnManifestServed(ctx context.Context, registry, repository string, manifestDigest digest.Digest) { - if p.resolver == nil { + if p.resolver == nil && p.onManifest == nil { return } // Use a fresh deadline so the prefetch survives the request @@ -1929,7 +2052,7 @@ func (p *layerPrefetchAdapter) OnManifestServed(ctx context.Context, registry, r ctx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() - rc, _, err := p.cache.Open(ctx, manifestDigest) + rc, err := p.openManifest(ctx, manifestDigest) if err != nil { p.logger.Debug("prefetch: manifest not in cache", slog.String("digest", manifestDigest.String()), @@ -1971,7 +2094,11 @@ func (p *layerPrefetchAdapter) OnManifestServed(ctx context.Context, registry, r return } - if len(children) == 0 { + if p.onManifest != nil { + p.onManifest(manifestDigest, children) + } + + if len(children) == 0 || p.resolver == nil { // Image index or no children - nothing to fan out. return } @@ -2001,7 +2128,7 @@ func (p *layerPrefetchAdapter) OnManifestServed(ctx context.Context, registry, r return } - if err := p.resolver.PrefetchChildren(ctx, pending, registry, repository); err != nil { + if err := p.resolver.PrefetchManifestChildren(ctx, manifestDigest, pending, registry, repository); err != nil { p.logger.Debug("prefetch: PrefetchChildren reported errors", slog.String("manifest", manifestDigest.String()), slog.Int("children", len(pending)), diff --git a/cmd/gantry/membership_peer_resolver_test.go b/cmd/gantry/membership_peer_resolver_test.go new file mode 100644 index 000000000..df8756ef0 --- /dev/null +++ b/cmd/gantry/membership_peer_resolver_test.go @@ -0,0 +1,114 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package main + +import ( + "log/slog" + "testing" + + libp2p "github.com/libp2p/go-libp2p" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/libp2p/go-libp2p/core/peerstore" + "github.com/multiformats/go-multiaddr" + + "github.com/Azure/unbounded/internal/gantry/ifaces" + "github.com/Azure/unbounded/internal/gantry/ifaces/fakes" +) + +func TestMembershipPeerIDResolverInstallsPodAddresses(t *testing.T) { + t.Parallel() + + target, err := libp2p.New(libp2p.NoListenAddrs) + if err != nil { + t.Fatalf("create target host: %v", err) + } + + defer func() { _ = target.Close() }() //nolint:errcheck // best-effort test cleanup + + caller, err := libp2p.New(libp2p.NoListenAddrs) + if err != nil { + t.Fatalf("create caller host: %v", err) + } + + defer func() { _ = caller.Close() }() //nolint:errcheck // best-effort test cleanup + + announced := "/ip4/10.64.1.23/tcp/4001/p2p/" + target.ID().String() + members := fakes.NewMembers("self", ifaces.Node{ + ID: "target-node", + PeerID: target.ID().String(), + P2PAddrs: []string{announced}, + }) + caller.Peerstore().AddAddr( + target.ID(), + multiaddr.StringCast("/ip4/127.0.0.1/tcp/4001"), + peerstore.PermanentAddrTTL, + ) + + resolve := membershipPeerIDResolver(members, caller.Peerstore(), slog.Default()) + + got, ok := resolve("target-node") + if !ok || got != target.ID() { + t.Fatalf("resolved peer = %q, %v; want %q, true", got, ok, target.ID()) + } + + want := multiaddr.StringCast("/ip4/10.64.1.23/tcp/4001") + if addrs := caller.Peerstore().Addrs(target.ID()); len(addrs) != 1 || !addrs[0].Equal(want) { + t.Fatalf("peerstore addresses = %v, want [%s]", addrs, want) + } +} + +func TestMembershipPeerIDResolverRejectsMismatchedAddressIdentity(t *testing.T) { + t.Parallel() + + target, err := libp2p.New(libp2p.NoListenAddrs) + if err != nil { + t.Fatalf("create target host: %v", err) + } + + defer func() { _ = target.Close() }() //nolint:errcheck // best-effort test cleanup + + other, err := libp2p.New(libp2p.NoListenAddrs) + if err != nil { + t.Fatalf("create other host: %v", err) + } + + defer func() { _ = other.Close() }() //nolint:errcheck // best-effort test cleanup + + caller, err := libp2p.New(libp2p.NoListenAddrs) + if err != nil { + t.Fatalf("create caller host: %v", err) + } + + defer func() { _ = caller.Close() }() //nolint:errcheck // best-effort test cleanup + + members := fakes.NewMembers("self", ifaces.Node{ + ID: "target-node", + PeerID: target.ID().String(), + P2PAddrs: []string{"/ip4/10.64.1.23/tcp/4001/p2p/" + other.ID().String()}, + }) + known := multiaddr.StringCast("/ip4/127.0.0.1/tcp/4001") + caller.Peerstore().AddAddr(target.ID(), known, peerstore.PermanentAddrTTL) + + resolve := membershipPeerIDResolver(members, caller.Peerstore(), slog.Default()) + + got, ok := resolve("target-node") + if !ok || got != target.ID() { + t.Fatalf("resolved peer = %q, %v; want %q, true", got, ok, target.ID()) + } + + if addrs := caller.Peerstore().Addrs(target.ID()); len(addrs) != 1 || !addrs[0].Equal(known) { + t.Fatalf("peerstore addresses = %v, want existing address [%s]", addrs, known) + } +} + +func TestMembershipPeerIDResolverMiss(t *testing.T) { + t.Parallel() + + members := fakes.NewMembers("self") + resolve := membershipPeerIDResolver(members, nil, slog.Default()) + + if got, ok := resolve(ifaces.NodeID(peer.ID("missing"))); ok || got != "" { + t.Fatalf("resolved missing peer = %q, %v; want empty, false", got, ok) + } +} diff --git a/cmd/gantry/prefetch_manifest_test.go b/cmd/gantry/prefetch_manifest_test.go new file mode 100644 index 000000000..1b0246e58 --- /dev/null +++ b/cmd/gantry/prefetch_manifest_test.go @@ -0,0 +1,148 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package main + +import ( + "context" + "errors" + "io" + "log/slog" + "strings" + "sync" + "testing" + "time" + + "github.com/Azure/unbounded/internal/gantry/digest" + "github.com/Azure/unbounded/internal/gantry/ifaces" + "github.com/Azure/unbounded/internal/gantry/manifest" +) + +// delayedManifestStore returns ErrNotFound until availableAfter opens, which +// models containerd committing the streamed manifest a moment after the mirror +// finishes serving it. +type delayedManifestStore struct { + mu sync.Mutex + opens int + body string + ready bool +} + +func (s *delayedManifestStore) markReady() { + s.mu.Lock() + defer s.mu.Unlock() + + s.ready = true +} + +func (s *delayedManifestStore) Open(_ context.Context, _ digest.Digest) (io.ReadCloser, int64, error) { + s.mu.Lock() + defer s.mu.Unlock() + + s.opens++ + + if !s.ready { + return nil, 0, errors.New("content not found") + } + + return io.NopCloser(strings.NewReader(s.body)), int64(len(s.body)), nil +} + +func (s *delayedManifestStore) openCount() int { + s.mu.Lock() + defer s.mu.Unlock() + + return s.opens +} + +func (s *delayedManifestStore) Has(context.Context, digest.Digest) (bool, error) { + return false, nil +} + +func (s *delayedManifestStore) Writer(context.Context, digest.Digest) (ifaces.ContentWriter, error) { + return nil, errors.New("not implemented") +} + +func (s *delayedManifestStore) Delete(context.Context, digest.Digest) error { + return errors.New("not implemented") +} + +func testDigest(t *testing.T, fill string) digest.Digest { + t.Helper() + + d, err := digest.Parse("sha256:" + strings.Repeat(fill, 64)) + if err != nil { + t.Fatalf("digest.Parse: %v", err) + } + + return d +} + +func TestOpenManifestRetriesUntilContainerdCommits(t *testing.T) { + store := &delayedManifestStore{body: `{"schemaVersion":2}`} + adapter := &layerPrefetchAdapter{cache: store, logger: slog.Default()} + + go func() { + time.Sleep(250 * time.Millisecond) + store.markReady() + }() + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + rc, err := adapter.openManifest(ctx, testDigest(t, "a")) + if err != nil { + t.Fatalf("openManifest: %v", err) + } + + defer func() { _ = rc.Close() }() //nolint:errcheck // best-effort close + + if store.openCount() < 2 { + t.Fatalf("open attempts = %d, want the retry path exercised", store.openCount()) + } +} + +func TestOpenManifestGivesUpWhenNeverCommitted(t *testing.T) { + store := &delayedManifestStore{body: "unused"} + adapter := &layerPrefetchAdapter{cache: store, logger: slog.Default()} + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + + if _, err := adapter.openManifest(ctx, testDigest(t, "b")); err == nil { + t.Fatal("openManifest succeeded, want failure when the manifest never lands") + } + + if store.openCount() < 2 { + t.Fatalf("open attempts = %d, want more than one before giving up", store.openCount()) + } +} + +func TestLayerPrefetchAdapterReportsManifestChildrenWithoutResolver(t *testing.T) { + manifestDigest := testDigest(t, "a") + configDigest := testDigest(t, "b") + layer0 := testDigest(t, "c") + layer1 := testDigest(t, "d") + body := `{"schemaVersion":2,"config":{"digest":"` + configDigest.String() + `"},"layers":[{"digest":"` + layer0.String() + `"},{"digest":"` + layer1.String() + `"}]}` + store := &delayedManifestStore{body: body, ready: true} + + var ( + gotManifest digest.Digest + gotChildren int + ) + + adapter := &layerPrefetchAdapter{ + cache: store, + logger: slog.Default(), + onManifest: func(observed digest.Digest, children []manifest.TypedChild) { + gotManifest = observed + gotChildren = len(children) + }, + } + + adapter.OnManifestServed(context.Background(), "registry.example", "pull", manifestDigest) + + if gotManifest != manifestDigest || gotChildren != 3 { + t.Fatalf("manifest callback = %s with %d children, want %s with 3", gotManifest, gotChildren, manifestDigest) + } +} diff --git a/cmd/gantry/stream_commit_tracker.go b/cmd/gantry/stream_commit_tracker.go index 3e0abdb99..88dc9c40b 100644 --- a/cmd/gantry/stream_commit_tracker.go +++ b/cmd/gantry/stream_commit_tracker.go @@ -7,6 +7,7 @@ import ( "context" "errors" "log/slog" + "sort" "sync" "time" @@ -37,27 +38,45 @@ type streamCommitTracker struct { verifyWindow time.Duration inventoryBudget time.Duration - onObserved func(n int) - onMissing func(n int) + onObserved func(n int) + onObservedDuration func(time.Duration) + onMissing func(n int) mu sync.Mutex - pending map[string][]time.Time + pending map[string][]pendingStreamCommit } -func newStreamCommitTracker(inv inventorySource, logger *slog.Logger, onObserved, onMissing func(n int)) *streamCommitTracker { +type pendingStreamCommit struct { + completedAt time.Time + deadline time.Time +} + +type observedStreamCommit struct { + completedAt time.Time + duration time.Duration +} + +func newStreamCommitTracker( + inv inventorySource, + logger *slog.Logger, + onObserved func(n int), + onObservedDuration func(time.Duration), + onMissing func(n int), +) *streamCommitTracker { if logger == nil { logger = slog.Default() } return &streamCommitTracker{ - inv: inv, - logger: logger.With(slog.String("subsystem", "stream_commit_tracker")), - probeInterval: defaultStreamCommitProbeInterval, - verifyWindow: defaultStreamCommitVerifyWindow, - inventoryBudget: defaultStreamCommitInventoryBudget, - onObserved: onObserved, - onMissing: onMissing, - pending: map[string][]time.Time{}, + inv: inv, + logger: logger.With(slog.String("subsystem", "stream_commit_tracker")), + probeInterval: defaultStreamCommitProbeInterval, + verifyWindow: defaultStreamCommitVerifyWindow, + inventoryBudget: defaultStreamCommitInventoryBudget, + onObserved: onObserved, + onObservedDuration: onObservedDuration, + onMissing: onMissing, + pending: map[string][]pendingStreamCommit{}, } } @@ -67,7 +86,11 @@ func (t *streamCommitTracker) RecordCompleted(d digest.Digest) { t.mu.Lock() defer t.mu.Unlock() - t.pending[d.String()] = append(t.pending[d.String()], time.Now().Add(t.verifyWindow)) + completedAt := time.Now() + t.pending[d.String()] = append(t.pending[d.String()], pendingStreamCommit{ + completedAt: completedAt, + deadline: completedAt.Add(t.verifyWindow), + }) } func (t *streamCommitTracker) Run(ctx context.Context) error { @@ -123,25 +146,32 @@ func (t *streamCommitTracker) probe(parent context.Context) { now := time.Now() observed := 0 missing := 0 + observedCommits := make([]observedStreamCommit, 0) t.mu.Lock() - for ds, deadlines := range t.pending { + for ds, commits := range t.pending { if _, ok := present[ds]; ok { - observed += len(deadlines) + observed += len(commits) + for _, commit := range commits { + observedCommits = append(observedCommits, observedStreamCommit{ + completedAt: commit.completedAt, + duration: now.Sub(commit.completedAt), + }) + } delete(t.pending, ds) continue } - kept := make([]time.Time, 0, len(deadlines)) - for _, deadline := range deadlines { - if now.After(deadline) || now.Equal(deadline) { + kept := make([]pendingStreamCommit, 0, len(commits)) + for _, commit := range commits { + if now.After(commit.deadline) || now.Equal(commit.deadline) { missing++ continue } - kept = append(kept, deadline) + kept = append(kept, commit) } if len(kept) == 0 { @@ -157,6 +187,16 @@ func (t *streamCommitTracker) probe(parent context.Context) { t.onObserved(observed) } + sort.Slice(observedCommits, func(i, j int) bool { + return observedCommits[i].completedAt.Before(observedCommits[j].completedAt) + }) + + if t.onObservedDuration != nil { + for _, commit := range observedCommits { + t.onObservedDuration(commit.duration) + } + } + if missing > 0 && t.onMissing != nil { t.onMissing(missing) } diff --git a/cmd/gantry/stream_commit_tracker_test.go b/cmd/gantry/stream_commit_tracker_test.go index fc058c70a..97952ca85 100644 --- a/cmd/gantry/stream_commit_tracker_test.go +++ b/cmd/gantry/stream_commit_tracker_test.go @@ -91,10 +91,17 @@ func TestStreamCommitTracker_ObservedAfterInventoryAppears(t *testing.T) { d := trackerDigestOf([]byte("observed-after-stream")) inv := &fakeInventorySource{} - var observed, missing int32 + var observed, missing, durations int32 tracker := newStreamCommitTracker(inv, nil, func(n int) { atomic.AddInt32(&observed, int32(n)) }, + func(duration time.Duration) { + if duration <= 0 { + t.Errorf("observed duration = %s, want positive", duration) + } + + atomic.AddInt32(&durations, 1) + }, func(n int) { atomic.AddInt32(&missing, int32(n)) }, ) tracker.probeInterval = 5 * time.Millisecond @@ -111,6 +118,7 @@ func TestStreamCommitTracker_ObservedAfterInventoryAppears(t *testing.T) { inv.SetCurrent(d) waitForAtomic(t, &observed, 1) + waitForAtomic(t, &durations, 1) if got := atomic.LoadInt32(&missing); got != 0 { t.Fatalf("missing = %d, want 0", got) @@ -125,6 +133,7 @@ func TestStreamCommitTracker_MissingAfterDeadline(t *testing.T) { tracker := newStreamCommitTracker(inv, nil, func(n int) { atomic.AddInt32(&observed, int32(n)) }, + nil, func(n int) { atomic.AddInt32(&missing, int32(n)) }, ) tracker.probeInterval = 5 * time.Millisecond @@ -153,6 +162,7 @@ func TestStreamCommitTracker_RetriesAfterUnavailableInventory(t *testing.T) { tracker := newStreamCommitTracker(inv, nil, func(n int) { atomic.AddInt32(&observed, int32(n)) }, + nil, func(n int) { atomic.AddInt32(&missing, int32(n)) }, ) tracker.probeInterval = 5 * time.Millisecond @@ -174,3 +184,30 @@ func TestStreamCommitTracker_RetriesAfterUnavailableInventory(t *testing.T) { t.Fatalf("missing = %d, want 0", got) } } + +func TestStreamCommitTracker_ReportsLatestCompletedStreamLast(t *testing.T) { + earlier := trackerDigestOf([]byte("earlier")) + later := trackerDigestOf([]byte("later")) + inv := &fakeInventorySource{current: []digest.Digest{earlier, later}} + + var durations []time.Duration + + tracker := newStreamCommitTracker(inv, nil, nil, func(duration time.Duration) { + durations = append(durations, duration) + }, nil) + now := time.Now() + tracker.pending[earlier.String()] = []pendingStreamCommit{{ + completedAt: now.Add(-2 * time.Second), + deadline: now.Add(time.Minute), + }} + tracker.pending[later.String()] = []pendingStreamCommit{{ + completedAt: now.Add(-time.Second), + deadline: now.Add(time.Minute), + }} + + tracker.probe(context.Background()) + + if len(durations) != 2 || durations[0] <= durations[1] { + t.Fatalf("durations = %v, want earlier completion before latest completion", durations) + } +} diff --git a/deploy/gantry/configmap.yaml.tmpl b/deploy/gantry/configmap.yaml.tmpl index e8dcc2781..293b35647 100644 --- a/deploy/gantry/configmap.yaml.tmpl +++ b/deploy/gantry/configmap.yaml.tmpl @@ -43,6 +43,9 @@ data: mirror_bind_allow_non_loopback: true transfer_listen: "0.0.0.0:5001" metrics_listen: "0.0.0.0:9095" + # Disabled by default. When enabled, Config.Validate requires a loopback + # bind; use kubectl port-forward rather than exposing this as a pod port. + pprof_listen: {{ default "" .PprofListen | quote }} # ---------- libp2p ---------- # IPv4 wildcard listeners. The pods/patch self-announce only @@ -140,6 +143,12 @@ data: # A positive fraction overrides prefetch_puller_replicas. There is no # maximum cap other than the eligible cluster/zone node count. prefetch_puller_fraction: 0.02 + # Only three deterministic manifest consumers dispatch remote groups; all + # consumers still start any self-selected local seed. This bounds redundant + # fan-out without reducing the configured pullers per layer. + prefetch_coordinator_replicas: 3 + prefetch_max_concurrent_groups: 64 + prefetch_dispatch_jitter: "1s" # Coord peer authorization. Default false ships observe-only: an # inbound coord request from a peer not in the membership view only # increments p2p_coord_unauthorized_peer_total and is still served. @@ -155,13 +164,10 @@ data: # one node. Each pull holds an HTTP body, containerd writer, # goroutine, and lease. coord_max_concurrent_pulls: 16 - # Deliberately SHORT. A requester stuck on a lockstep-saturated seed must - # bail at this deadline and re-select a fresher finisher-seed rather than - # ride the slow stream to completion; that bail-and-re-select (with the - # strict containerd hosts.toml) drives the cold-start cascade. Too high - # (e.g. 1h) removes re-selection and collapses to single-seed bandwidth - # (the ~12min lockstep). Matches upstream gantry's hardcoded 60s. - peer_fetch_timeout: "60s" + # Safety ceiling for a complete peer request. Progressing large layers must + # not switch providers only because their total transfer exceeds 60s. The + # HTTP/2 transport separately probes connections after 10s without reads. + peer_fetch_timeout: "15m" # Peer re-discovery loop. On a cache miss the mirror repeatedly re-runs DHT # FindProviders and retries peer fetches for up to peer_rediscover_budget to # pick up finisher-seeds that advertise mid-swarm (the cold-start cascade), diff --git a/hack/cmd/gantry-benchmark-monitor/grid.go b/hack/cmd/gantry-benchmark-monitor/grid.go new file mode 100644 index 000000000..6cdf9c924 --- /dev/null +++ b/hack/cmd/gantry-benchmark-monitor/grid.go @@ -0,0 +1,367 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package main + +import ( + "encoding/json" + "fmt" + "math" + "sort" + "strconv" + "strings" + "time" +) + +const defaultNodesPerPage = 64 + +type instantSeries struct { + Metric map[string]string + Timestamp time.Time + Value float64 +} + +type instantResponse struct { + Series []instantSeries +} + +type progressLayer struct { + Index int + Digest string +} + +type progressGrid struct { + Image string + ImageDigest string + Nodes []string + Layers []progressLayer + Downloaded map[string]map[int]time.Time + Unpacked map[string]map[string]time.Time + ImageStart map[string]time.Time + ImageDone map[string]time.Time + Latest time.Time +} + +func parseInstantResponse(raw []byte) (instantResponse, error) { + var envelope struct { + Status string `json:"status"` + Data struct { + Result []struct { + Metric map[string]string `json:"metric"` + Value [2]json.RawMessage `json:"value"` + } `json:"result"` + } `json:"data"` + } + if err := json.Unmarshal(raw, &envelope); err != nil { + return instantResponse{}, fmt.Errorf("decode Prometheus instant response: %w", err) + } + + if envelope.Status != "success" { + return instantResponse{}, fmt.Errorf("prometheus instant query status is %q", envelope.Status) + } + + response := instantResponse{Series: make([]instantSeries, 0, len(envelope.Data.Result))} + for _, rawSeries := range envelope.Data.Result { + var timestamp float64 + if err := json.Unmarshal(rawSeries.Value[0], ×tamp); err != nil { + return instantResponse{}, fmt.Errorf("decode Prometheus instant timestamp: %w", err) + } + + var text string + if err := json.Unmarshal(rawSeries.Value[1], &text); err != nil { + return instantResponse{}, fmt.Errorf("decode Prometheus instant value: %w", err) + } + + value, err := strconv.ParseFloat(text, 64) + if err != nil { + return instantResponse{}, fmt.Errorf("parse Prometheus instant value %q: %w", text, err) + } + + if math.IsNaN(value) || math.IsInf(value, 0) { + continue + } + + seconds, fraction := math.Modf(timestamp) + response.Series = append(response.Series, instantSeries{ + Metric: rawSeries.Metric, + Timestamp: time.Unix(int64(seconds), int64(fraction*float64(time.Second))).UTC(), + Value: value, + }) + } + + return response, nil +} + +func imageDigest(reference string) string { + _, digest, ok := strings.Cut(reference, "@") + if ok { + return digest + } + + return "" +} + +func imageMatches(reference, target, targetDigest string) bool { + if reference == target { + return true + } + + return targetDigest != "" && imageDigest(reference) == targetDigest +} + +func aggregateProgressGrid(response instantResponse, nodes []string, image string) progressGrid { + grid := progressGrid{ + Image: image, + ImageDigest: imageDigest(image), + Nodes: append([]string(nil), nodes...), + Downloaded: map[string]map[int]time.Time{}, + Unpacked: map[string]map[string]time.Time{}, + ImageStart: map[string]time.Time{}, + ImageDone: map[string]time.Time{}, + } + sort.Strings(grid.Nodes) + + layers := map[int]string{} + + for _, series := range response.Series { + if series.Timestamp.After(grid.Latest) { + grid.Latest = series.Timestamp + } + + name, node := series.Metric["__name__"], series.Metric["node"] + + if node == "" { + continue + } + + switch name { + case "gantry_layer_download_completed_timestamp_seconds": + if grid.ImageDigest != "" && series.Metric["image_digest"] != grid.ImageDigest { + continue + } + + index, err := strconv.Atoi(series.Metric["layer_index"]) + if err != nil || index < 0 { + continue + } + + digest := series.Metric["layer_digest"] + if digest == "" { + continue + } + + layers[index] = digest + + if series.Value > 0 { + if grid.Downloaded[node] == nil { + grid.Downloaded[node] = map[int]time.Time{} + } + + grid.Downloaded[node][index] = time.Unix(0, int64(series.Value*float64(time.Second))).UTC() + } + case "gantry_benchmark_layer_unpacked_timestamp_seconds": + if !imageMatches(series.Metric["image"], image, grid.ImageDigest) || series.Value <= 0 { + continue + } + + digest := series.Metric["layer_digest"] + if digest == "" { + continue + } + + if grid.Unpacked[node] == nil { + grid.Unpacked[node] = map[string]time.Time{} + } + + grid.Unpacked[node][digest] = time.Unix(seriesValueSeconds(series.Value), 0).UTC() + case "gantry_benchmark_image_unpack_started_timestamp_seconds": + if imageMatches(series.Metric["image"], image, grid.ImageDigest) && series.Value > 0 { + grid.ImageStart[node] = time.Unix(seriesValueSeconds(series.Value), 0).UTC() + } + case "gantry_benchmark_image_unpacked_timestamp_seconds": + if imageMatches(series.Metric["image"], image, grid.ImageDigest) && series.Value > 0 { + grid.ImageDone[node] = time.Unix(seriesValueSeconds(series.Value), 0).UTC() + } + } + } + + indexes := make([]int, 0, len(layers)) + for index := range layers { + indexes = append(indexes, index) + } + + sort.Ints(indexes) + + grid.Layers = make([]progressLayer, 0, len(indexes)) + for _, index := range indexes { + grid.Layers = append(grid.Layers, progressLayer{Index: index, Digest: layers[index]}) + } + + return grid +} + +func seriesValueSeconds(value float64) int64 { + return int64(math.Floor(value)) +} + +func phaseMinuteCell(completedAt, phaseStart time.Time) byte { + if completedAt.IsZero() { + return '.' + } + + minute := int(completedAt.Sub(phaseStart) / time.Minute) + if minute < 0 { + minute = 0 + } + + if minute > 35 { + minute = 35 + } + + const digits = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" + + return digits[minute] +} + +func unpackCell(grid progressGrid, node string) byte { + if !grid.ImageDone[node].IsZero() { + return '#' + } + + if len(grid.Layers) == 0 { + return '.' + } + + completed := 0 + + for _, layer := range grid.Layers { + if !grid.Unpacked[node][layer.Digest].IsZero() { + completed++ + } + } + + if completed == 0 { + if !grid.ImageStart[node].IsZero() { + return '0' + } + + return '.' + } + + level := int(math.Ceil(float64(completed) / float64(len(grid.Layers)) * 9)) + if level < 1 { + level = 1 + } + + if level > 9 { + level = 9 + } + + return byte('0' + level) +} + +func pageNodes(nodes []string, page, perPage int) ([]string, int, int) { + if perPage <= 0 { + perPage = defaultNodesPerPage + } + + totalPages := max(1, (len(nodes)+perPage-1)/perPage) + + if page < 1 { + page = 1 + } + + if page > totalPages { + page = totalPages + } + + start := (page - 1) * perPage + end := min(start+perPage, len(nodes)) + + return nodes[start:end], page, totalPages +} + +func renderProgressGrids(builder *strings.Builder, snapshot monitorSnapshot) { + if snapshot.GridError != "" { + fmt.Fprintf(builder, "\nprogress grids unavailable: %s\n", snapshot.GridError) + return + } + + grid := snapshot.Progress + if len(grid.Nodes) == 0 || len(grid.Layers) == 0 { + fmt.Fprintln(builder, "\nprogress grids: waiting for per-layer samples") + return + } + + nodes, page, pages := pageNodes(grid.Nodes, snapshot.NodePage, snapshot.NodesPerPage) + + fmt.Fprintln(builder, "\n=== Layer downloads x nodes ===") + fmt.Fprintf(builder, "page %d/%d; nodes %d; showing %s .. %s\n", page, pages, len(grid.Nodes), nodes[0], nodes[len(nodes)-1]) + renderNodeHeader(builder, nodes) + + downloaded := 0 + + for _, layer := range grid.Layers { + fmt.Fprintf(builder, "L%02d %s ", layer.Index, shortDigest(layer.Digest)) + + for _, node := range nodes { + cell := phaseMinuteCell(grid.Downloaded[node][layer.Index], snapshot.PhaseStart) + builder.WriteByte(cell) + + if cell != '.' { + downloaded++ + } + } + + builder.WriteByte('\n') + } + + fmt.Fprintf(builder, "shown cells downloaded: %d/%d; legend: .=pending, 0-9/A-Z=completion phase minute (Z=35+)\n", downloaded, len(nodes)*len(grid.Layers)) + + fmt.Fprintln(builder, "\n=== Image unpack x nodes ===") + fmt.Fprintf(builder, "image %s\n", shortDigest(grid.ImageDigest)) + renderNodeHeader(builder, nodes) + fmt.Fprintf(builder, "image ") + + for _, node := range nodes { + builder.WriteByte(unpackCell(grid, node)) + } + + builder.WriteByte('\n') + fmt.Fprintln(builder, "legend: .=not started, 0=started, 1-9=unpacked layer decile, #=image unpacked") +} + +func renderNodeHeader(builder *strings.Builder, nodes []string) { + for position := 0; position < 3; position++ { + if position == 0 { + fmt.Fprint(builder, "node[-3:] ") + } else { + fmt.Fprint(builder, " ") + } + + for _, node := range nodes { + suffix := node + + if len(suffix) > 3 { + suffix = suffix[len(suffix)-3:] + } + + for len(suffix) <= position { + suffix = " " + suffix + } + + builder.WriteByte(suffix[position]) + } + + builder.WriteByte('\n') + } +} + +func shortDigest(value string) string { + value = strings.TrimPrefix(value, "sha256:") + if len(value) > 8 { + return value[:8] + } + + return value +} diff --git a/hack/cmd/gantry-benchmark-monitor/grid_test.go b/hack/cmd/gantry-benchmark-monitor/grid_test.go new file mode 100644 index 000000000..e86af5694 --- /dev/null +++ b/hack/cmd/gantry-benchmark-monitor/grid_test.go @@ -0,0 +1,105 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package main + +import ( + "fmt" + "strings" + "testing" + "time" +) + +func TestParseAndAggregateProgressGrid(t *testing.T) { + phaseStart := time.Unix(1_000, 0).UTC() + image := "registry.example/pull@sha256:image" + raw := fmt.Sprintf(`{"status":"success","data":{"result":[ + {"metric":{"__name__":"gantry_layer_download_completed_timestamp_seconds","node":"node-a","image_digest":"sha256:image","layer_digest":"sha256:aaa","layer_index":"0"},"value":[%d,"%d"]}, + {"metric":{"__name__":"gantry_layer_download_completed_timestamp_seconds","node":"node-b","image_digest":"sha256:image","layer_digest":"sha256:aaa","layer_index":"0"},"value":[%d,"0"]}, + {"metric":{"__name__":"gantry_layer_download_completed_timestamp_seconds","node":"node-a","image_digest":"sha256:image","layer_digest":"sha256:bbb","layer_index":"1"},"value":[%d,"%d"]}, + {"metric":{"__name__":"gantry_benchmark_image_unpack_started_timestamp_seconds","node":"node-a","image":"%s"},"value":[%d,"%d"]}, + {"metric":{"__name__":"gantry_benchmark_layer_unpacked_timestamp_seconds","node":"node-a","image":"%s","layer_digest":"sha256:aaa"},"value":[%d,"%d"]}, + {"metric":{"__name__":"gantry_benchmark_image_unpacked_timestamp_seconds","node":"node-b","image":"%s"},"value":[%d,"%d"]} + ]}}`, + phaseStart.Unix()+100, phaseStart.Unix()+30, + phaseStart.Unix()+100, + phaseStart.Unix()+100, phaseStart.Unix()+90, + image, phaseStart.Unix()+100, phaseStart.Unix()+5, + image, phaseStart.Unix()+100, phaseStart.Unix()+80, + image, phaseStart.Unix()+100, phaseStart.Unix()+95, + ) + + response, err := parseInstantResponse([]byte(raw)) + if err != nil { + t.Fatalf("parseInstantResponse: %v", err) + } + + grid := aggregateProgressGrid(response, []string{"node-b", "node-a", "node-c"}, image) + if len(grid.Layers) != 2 || grid.Layers[0].Digest != "sha256:aaa" || grid.Layers[1].Digest != "sha256:bbb" { + t.Fatalf("layers = %#v", grid.Layers) + } + + if got := phaseMinuteCell(grid.Downloaded["node-a"][0], phaseStart); got != '0' { + t.Fatalf("layer 0 cell = %q, want 0", got) + } + + if got := phaseMinuteCell(grid.Downloaded["node-a"][1], phaseStart); got != '1' { + t.Fatalf("layer 1 cell = %q, want 1", got) + } + + if got := unpackCell(grid, "node-a"); got != '5' { + t.Fatalf("node-a unpack cell = %q, want 5", got) + } + + if got := unpackCell(grid, "node-b"); got != '#' { + t.Fatalf("node-b unpack cell = %q, want #", got) + } + + if got := unpackCell(grid, "node-c"); got != '.' { + t.Fatalf("node-c unpack cell = %q, want .", got) + } +} + +func TestRenderProgressGridsPagesNodes(t *testing.T) { + phaseStart := time.Unix(1_000, 0).UTC() + grid := progressGrid{ + Image: "registry.example/pull@sha256:image", + ImageDigest: "sha256:image", + Nodes: []string{"node-001", "node-002", "node-003"}, + Layers: []progressLayer{{Index: 0, Digest: "sha256:aaaaaaaa"}}, + Downloaded: map[string]map[int]time.Time{ + "node-001": {0: phaseStart.Add(2 * time.Minute)}, + }, + Unpacked: map[string]map[string]time.Time{}, + ImageStart: map[string]time.Time{}, + ImageDone: map[string]time.Time{}, + } + + var builder strings.Builder + renderProgressGrids(&builder, monitorSnapshot{ + PhaseStart: phaseStart, + Progress: grid, + NodePage: 1, + NodesPerPage: 2, + }) + + output := builder.String() + for _, want := range []string{ + "=== Layer downloads x nodes ===", + "page 1/2; nodes 3; showing node-001 .. node-002", + "L00 aaaaaaaa 2.", + "=== Image unpack x nodes ===", + "image ..", + } { + if !strings.Contains(output, want) { + t.Fatalf("rendered grid missing %q:\n%s", want, output) + } + } +} + +func TestPageNodesClampsPage(t *testing.T) { + nodes, page, pages := pageNodes([]string{"a", "b", "c"}, 99, 2) + if page != 2 || pages != 2 || len(nodes) != 1 || nodes[0] != "c" { + t.Fatalf("pageNodes = %v, page %d/%d", nodes, page, pages) + } +} diff --git a/hack/cmd/gantry-benchmark-monitor/main.go b/hack/cmd/gantry-benchmark-monitor/main.go new file mode 100644 index 000000000..4dfb88eb2 --- /dev/null +++ b/hack/cmd/gantry-benchmark-monitor/main.go @@ -0,0 +1,616 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "net/url" + "os" + "os/exec" + "os/signal" + "strconv" + "strings" + "syscall" + "time" + + "golang.org/x/term" +) + +const ( + defaultBenchmarkNamespace = "gantry-benchmark" + defaultGantryNamespace = "gantry-system" + defaultMonitoringNS = "monitoring" + defaultPrometheusService = "kps-kube-prometheus-stack-prometheus" + stateConfigMapName = "gantry-benchmark-state" + gantryPhaseLabel = "gantry-cold" + gridQueryInterval = 10 * time.Second +) + +type monitorConfig struct { + kubectl string + kubeconfig string + benchmarkNamespace string + gantryNamespace string + monitoringNamespace string + prometheusService string + runID string + refreshInterval time.Duration + nodePage int + nodesPerPage int + once bool + noClear bool +} + +type kubectlRunner struct { + binary string + kubeconfig string +} + +func (r kubectlRunner) run(ctx context.Context, args ...string) ([]byte, error) { + if r.kubeconfig != "" { + args = append([]string{"--kubeconfig", r.kubeconfig}, args...) + } + + command := exec.CommandContext(ctx, r.binary, args...) + + output, err := command.CombinedOutput() + if err != nil { + return nil, fmt.Errorf("kubectl %s: %w: %s", strings.Join(args, " "), err, strings.TrimSpace(string(output))) + } + + return output, nil +} + +type benchmarkState struct { + RunID string `json:"run_id"` + MonitoringNamespace string `json:"monitoring_namespace"` + PrometheusService string `json:"prometheus_service"` + NodeCount int `json:"node_count"` + ImageSizeMiB int `json:"image_size_mib"` +} + +type monitorSession struct { + runID string + jobName string + phaseStart time.Time + nodeCount int + expectedBytes float64 + monitoringNamespace string + prometheusService string + revision string + image string +} + +type jobStatus struct { + Succeeded int + Active int + Failed int + Complete bool +} + +func envDefault(name, fallback string) string { + if value := os.Getenv(name); value != "" { + return value + } + + return fallback +} + +func parseConfig(args []string) (monitorConfig, error) { + config := monitorConfig{} + flags := flag.NewFlagSet("gantry-benchmark-monitor", flag.ContinueOnError) + flags.SetOutput(os.Stderr) + flags.Usage = func() { + fmt.Fprintln(flags.Output(), "Usage: gantry-benchmark-monitor [options]") //nolint:errcheck // best-effort help output + fmt.Fprintln(flags.Output(), "Live peer traffic, layer downloads, image unpacking, and Pod state.") //nolint:errcheck // best-effort help output + flags.PrintDefaults() + } + + flags.StringVar(&config.kubectl, "kubectl", envDefault("KUBECTL", "kubectl"), "kubectl executable") + flags.StringVar(&config.kubeconfig, "kubeconfig", os.Getenv("KUBECONFIG"), "kubeconfig path") + flags.StringVar(&config.benchmarkNamespace, "namespace", envDefault("BENCHMARK_NAMESPACE", defaultBenchmarkNamespace), "benchmark namespace") + flags.StringVar(&config.gantryNamespace, "gantry-namespace", envDefault("GANTRY_NAMESPACE", defaultGantryNamespace), "Gantry namespace") + flags.StringVar(&config.monitoringNamespace, "monitoring-namespace", envDefault("MONITORING_NAMESPACE", defaultMonitoringNS), "monitoring namespace") + flags.StringVar(&config.prometheusService, "prometheus-service", envDefault("PROMETHEUS_SERVICE", defaultPrometheusService), "Prometheus service") + flags.StringVar(&config.runID, "run-id", "", "run ID (default: active benchmark state)") + flags.DurationVar(&config.refreshInterval, "refresh", time.Second, "display and query refresh interval") + flags.IntVar(&config.nodePage, "node-page", 1, "1-based node page shown in progress grids") + flags.IntVar(&config.nodesPerPage, "nodes-per-page", defaultGridColumns(), "node columns shown per progress-grid page") + flags.BoolVar(&config.once, "once", false, "print one snapshot and exit") + flags.BoolVar(&config.noClear, "no-clear", false, "do not redraw the terminal") + + if err := flags.Parse(args); err != nil { + return monitorConfig{}, err + } + + if config.refreshInterval <= 0 { + return monitorConfig{}, fmt.Errorf("refresh must be positive, got %s", config.refreshInterval) + } + + if config.nodePage < 1 { + return monitorConfig{}, fmt.Errorf("node-page must be at least 1, got %d", config.nodePage) + } + + if config.nodesPerPage < 1 { + return monitorConfig{}, fmt.Errorf("nodes-per-page must be at least 1, got %d", config.nodesPerPage) + } + + return config, nil +} + +func loadBenchmarkState(ctx context.Context, runner kubectlRunner, config monitorConfig) (benchmarkState, error) { + output, err := runner.run(ctx, "-n", config.benchmarkNamespace, "get", "configmap", stateConfigMapName, "-o", "json") + if err != nil { + return benchmarkState{}, err + } + + var configMap struct { + Data map[string]string `json:"data"` + } + if err := json.Unmarshal(output, &configMap); err != nil { + return benchmarkState{}, fmt.Errorf("decode benchmark state ConfigMap: %w", err) + } + + raw := configMap.Data["state.json"] + if raw == "" { + return benchmarkState{}, errors.New("benchmark state ConfigMap has no state.json") + } + + var state benchmarkState + if err := json.Unmarshal([]byte(raw), &state); err != nil { + return benchmarkState{}, fmt.Errorf("decode benchmark state: %w", err) + } + + return state, nil +} + +func loadGantryRevision(ctx context.Context, runner kubectlRunner, namespace string) (string, error) { + output, err := runner.run(ctx, "-n", namespace, "get", "pods", "-l", "app.kubernetes.io/name=gantry", "-o", "json") + if err != nil { + return "", err + } + + var pods struct { + Items []struct { + Metadata struct { + Labels map[string]string `json:"labels"` + } `json:"metadata"` + Status struct { + Conditions []struct { + Type string `json:"type"` + Status string `json:"status"` + } `json:"conditions"` + } `json:"status"` + } `json:"items"` + } + if err := json.Unmarshal(output, &pods); err != nil { + return "", fmt.Errorf("decode Gantry pods: %w", err) + } + + for _, pod := range pods.Items { + ready := false + + for _, condition := range pod.Status.Conditions { + if condition.Type == "Ready" && condition.Status == "True" { + ready = true + break + } + } + + if !ready { + continue + } + + if revision := pod.Metadata.Labels["controller-revision-hash"]; revision != "" { + return revision, nil + } + } + + return "", errors.New("no Ready Gantry pod has controller-revision-hash") +} + +func loadJob(ctx context.Context, runner kubectlRunner, namespace, runID string) (monitorSession, jobStatus, error) { + selector := "gantry.unbounded-cloud.io/run-id=" + runID + ",gantry.unbounded-cloud.io/phase=" + gantryPhaseLabel + + output, err := runner.run(ctx, "-n", namespace, "get", "jobs", "-l", selector, "-o", "json") + if err != nil { + return monitorSession{}, jobStatus{}, err + } + + var jobs struct { + Items []struct { + Metadata struct { + Name string `json:"name"` + } `json:"metadata"` + Spec struct { + Completions int `json:"completions"` + Template struct { + Spec struct { + Containers []struct { + Name string `json:"name"` + Image string `json:"image"` + } `json:"containers"` + } `json:"spec"` + } `json:"template"` + } `json:"spec"` + Status struct { + StartTime string `json:"startTime"` + Succeeded int `json:"succeeded"` + Active int `json:"active"` + Failed int `json:"failed"` + Conditions []struct { + Type string `json:"type"` + Status string `json:"status"` + } `json:"conditions"` + } `json:"status"` + } `json:"items"` + } + if err := json.Unmarshal(output, &jobs); err != nil { + return monitorSession{}, jobStatus{}, fmt.Errorf("decode Gantry pull Job: %w", err) + } + + if len(jobs.Items) == 0 { + return monitorSession{}, jobStatus{}, errors.New("gantry-cold pull Job has not started") + } + + job := jobs.Items[len(jobs.Items)-1] + if job.Status.StartTime == "" { + return monitorSession{}, jobStatus{}, errors.New("gantry-cold pull Job has no startTime") + } + + startedAt, err := time.Parse(time.RFC3339, job.Status.StartTime) + if err != nil { + return monitorSession{}, jobStatus{}, fmt.Errorf("parse Job startTime %q: %w", job.Status.StartTime, err) + } + + status := jobStatus{Succeeded: job.Status.Succeeded, Active: job.Status.Active, Failed: job.Status.Failed} + for _, condition := range job.Status.Conditions { + if condition.Type == "Complete" && condition.Status == "True" { + status.Complete = true + } + } + + image := "" + + for _, container := range job.Spec.Template.Spec.Containers { + if container.Name == "pull" { + image = container.Image + break + } + } + + return monitorSession{jobName: job.Metadata.Name, phaseStart: startedAt, nodeCount: job.Spec.Completions, image: image}, status, nil +} + +func defaultGridColumns() int { + width, _, err := term.GetSize(int(os.Stdout.Fd())) + if err != nil { + return defaultNodesPerPage + } + + return min(96, max(16, width-14)) +} + +func discoverSession(ctx context.Context, runner kubectlRunner, config monitorConfig) (monitorSession, jobStatus, error) { + state, err := loadBenchmarkState(ctx, runner, config) + if err != nil { + return monitorSession{}, jobStatus{}, err + } + + runID := config.runID + if runID == "" { + runID = state.RunID + } + + if runID == "" { + return monitorSession{}, jobStatus{}, errors.New("active benchmark state has no run ID") + } + + session, status, err := loadJob(ctx, runner, config.benchmarkNamespace, runID) + if err != nil { + return monitorSession{}, jobStatus{}, err + } + + revision, err := loadGantryRevision(ctx, runner, config.gantryNamespace) + if err != nil { + return monitorSession{}, jobStatus{}, err + } + + session.runID = runID + if session.nodeCount == 0 { + session.nodeCount = state.NodeCount + } + + session.expectedBytes = float64(state.NodeCount) * float64(state.ImageSizeMiB) * 1024 * 1024 + + session.monitoringNamespace = config.monitoringNamespace + if state.MonitoringNamespace != "" { + session.monitoringNamespace = state.MonitoringNamespace + } + + session.prometheusService = config.prometheusService + if state.PrometheusService != "" { + session.prometheusService = state.PrometheusService + } + + session.revision = revision + + return session, status, nil +} + +func refreshJobStatus(ctx context.Context, runner kubectlRunner, config monitorConfig, session monitorSession) (jobStatus, error) { + _, status, err := loadJob(ctx, runner, config.benchmarkNamespace, session.runID) + return status, err +} + +func prometheusExpression(session monitorSession, gantryNamespace string) string { + labels := fmt.Sprintf(`namespace=%s,gantry_benchmark="true",controller_revision_hash=%s`, strconv.Quote(gantryNamespace), strconv.Quote(session.revision)) + peer := fmt.Sprintf(`sum by (outcome) (p2p_peer_fetch_total{%s})`, labels) + bytes := fmt.Sprintf(`sum(gantry_mirror_bytes_served_total{%s,kind="layer"})`, labels) + + return peer + " or " + bytes +} + +func queryPrometheusRange(ctx context.Context, runner kubectlRunner, session monitorSession, gantryNamespace string, now time.Time) (rangeResponse, error) { + start := session.phaseStart.Add(-15 * time.Second) + rawPath := fmt.Sprintf( + "/api/v1/namespaces/%s/services/http:%s:9090/proxy/api/v1/query_range?query=%s&start=%s&end=%s&step=1", + session.monitoringNamespace, + session.prometheusService, + url.QueryEscape(prometheusExpression(session, gantryNamespace)), + url.QueryEscape(start.UTC().Format(time.RFC3339Nano)), + url.QueryEscape(now.UTC().Format(time.RFC3339Nano)), + ) + + output, err := runner.run(ctx, "get", "--raw", rawPath) + if err != nil { + return rangeResponse{}, err + } + + return parseRangeResponse(output) +} + +func progressExpression(session monitorSession, config monitorConfig) string { + downloadLabels := fmt.Sprintf( + `namespace=%s,gantry_benchmark="true",controller_revision_hash=%s`, + strconv.Quote(config.gantryNamespace), + strconv.Quote(session.revision), + ) + + observerLabels := fmt.Sprintf(`namespace=%s,gantry_benchmark="true"`, strconv.Quote(config.benchmarkNamespace)) + + if digest := imageDigest(session.image); digest != "" { + downloadLabels += `,image_digest=` + strconv.Quote(digest) + } + + if session.image != "" { + observerLabels += `,image=` + strconv.Quote(session.image) + } + + progress := fmt.Sprintf(`gantry_layer_download_completed_timestamp_seconds{%s} or {__name__=~"gantry_benchmark_(image_unpack_started|image_unpacked|layer_unpacked)_timestamp_seconds",%s}`, + downloadLabels, + observerLabels, + ) + + return progress + " or " + nodeResourceExpression(config) +} + +func queryProgress(ctx context.Context, runner kubectlRunner, session monitorSession, config monitorConfig, now time.Time) (instantResponse, error) { + rawPath := fmt.Sprintf( + "/api/v1/namespaces/%s/services/http:%s:9090/proxy/api/v1/query?query=%s&time=%s", + session.monitoringNamespace, + session.prometheusService, + url.QueryEscape(progressExpression(session, config)), + url.QueryEscape(now.UTC().Format(time.RFC3339Nano)), + ) + + output, err := runner.run(ctx, "get", "--raw", rawPath) + if err != nil { + return instantResponse{}, err + } + + return parseInstantResponse(output) +} + +func renderWaiting(config monitorConfig, err error) { + if !config.noClear && term.IsTerminal(int(os.Stdout.Fd())) { + fmt.Print("\033[H\033[2J") + } + + fmt.Printf("Gantry benchmark live monitor\n\ntime: %s\nstatus: waiting\nreason: %v\n", time.Now().UTC().Format(time.RFC3339), err) +} + +func runMonitor(ctx context.Context, config monitorConfig) error { + runner := kubectlRunner{binary: config.kubectl, kubeconfig: config.kubeconfig} + + var ( + session *monitorSession + podTracker *podStateTracker + lastSnapshot monitorSnapshot + lastProgress progressGrid + lastResources nodeResourceGrid + gridError string + nextGridAt time.Time + ) + + for { + now := time.Now().UTC() + + var status jobStatus + + if session == nil { + discovered, job, err := discoverSession(ctx, runner, config) + if err != nil { + renderWaiting(config, err) + + if config.once { + return err + } + + if err := waitForNext(ctx, config.refreshInterval); err != nil { + return err + } + + continue + } + + session = &discovered + status = job + + tracker, err := newPodStateTracker(ctx, config.kubeconfig, config.benchmarkNamespace, session.jobName) + if err != nil { + session = nil + + renderWaiting(config, err) + + if config.once { + return err + } + + if err := waitForNext(ctx, config.refreshInterval); err != nil { + return err + } + + continue + } + + podTracker = tracker + } else { + job, err := refreshJobStatus(ctx, runner, config, *session) + if err != nil { + renderWaiting(config, err) + + if err := waitForNext(ctx, config.refreshInterval); err != nil { + return err + } + + continue + } + + status = job + } + + queryCtx, cancel := context.WithTimeout(ctx, 15*time.Second) + response, err := queryPrometheusRange(queryCtx, runner, *session, config.gantryNamespace, now) + + cancel() + + if err != nil { + renderWaiting(config, err) + + if config.once { + return err + } + + if err := waitForNext(ctx, config.refreshInterval); err != nil { + return err + } + + continue + } + + lastSnapshot = aggregateRange(response, session.phaseStart, now, session.expectedBytes, session.nodeCount) + lastSnapshot.RunID = session.runID + lastSnapshot.JobName = session.jobName + lastSnapshot.PhaseStart = session.phaseStart + lastSnapshot.Now = now + lastSnapshot.RefreshInterval = config.refreshInterval + lastSnapshot.Job = status + lastSnapshot.Color = term.IsTerminal(int(os.Stdout.Fd())) && os.Getenv("NO_COLOR") == "" + lastSnapshot.NodePage = config.nodePage + lastSnapshot.NodesPerPage = config.nodesPerPage + + var nodes []string + + if podTracker != nil { + counts, podErr := podTracker.snapshot() + nodes = podTracker.snapshotNodes() + + lastSnapshot.PodStates = counts + if podErr != nil { + lastSnapshot.PodStateError = podErr.Error() + } + } + + if status.Complete || !now.Before(nextGridAt) { + gridCtx, gridCancel := context.WithTimeout(ctx, 15*time.Second) + gridResponse, gridErr := queryProgress(gridCtx, runner, *session, config, now) + + gridCancel() + + if gridErr != nil { + gridError = gridErr.Error() + } else { + lastProgress = aggregateProgressGrid(gridResponse, nodes, session.image) + lastResources = aggregateNodeResources(gridResponse, nodes) + gridError = "" + } + + nextGridAt = now.Add(gridQueryInterval) + } else if len(nodes) > 0 { + lastProgress.Nodes = append(lastProgress.Nodes[:0], nodes...) + lastResources.Nodes = append(lastResources.Nodes[:0], nodes...) + } + + lastSnapshot.Progress = lastProgress + lastSnapshot.Resources = lastResources + lastSnapshot.GridError = gridError + + if !config.noClear && term.IsTerminal(int(os.Stdout.Fd())) { + fmt.Print("\033[H\033[2J") + } + + fmt.Print(renderSnapshot(lastSnapshot)) + + if config.once || status.Complete { + return nil + } + + if err := waitForNext(ctx, config.refreshInterval); err != nil { + return err + } + } +} + +func waitForNext(ctx context.Context, interval time.Duration) error { + timer := time.NewTimer(interval) + defer timer.Stop() + + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + +func main() { + config, err := parseConfig(os.Args[1:]) + if err != nil { + if errors.Is(err, flag.ErrHelp) { + return + } + + fmt.Fprintln(os.Stderr, err) + os.Exit(2) + } + + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + + isTTY := term.IsTerminal(int(os.Stdout.Fd())) && !config.noClear + if isTTY { + fmt.Print("\033[?1049h") + defer fmt.Print("\033[?1049l") + } + + if err := runMonitor(ctx, config); err != nil && !errors.Is(err, context.Canceled) { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} diff --git a/hack/cmd/gantry-benchmark-monitor/monitor.go b/hack/cmd/gantry-benchmark-monitor/monitor.go new file mode 100644 index 000000000..ac4d2ee70 --- /dev/null +++ b/hack/cmd/gantry-benchmark-monitor/monitor.go @@ -0,0 +1,371 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package main + +import ( + "encoding/json" + "fmt" + "math" + "sort" + "strconv" + "strings" + "time" +) + +type rangeSample struct { + Timestamp time.Time + Value float64 +} + +type rangeSeries struct { + Metric map[string]string + Samples []rangeSample +} + +type rangeResponse struct { + Series []rangeSeries +} + +type minuteBin struct { + Minute int + PeerOutcomes map[string]float64 + Bytes float64 +} + +type monitorSnapshot struct { + RunID string + JobName string + PhaseStart time.Time + Now time.Time + LatestSample time.Time + RefreshInterval time.Duration + NodeCount int + ExpectedBytes float64 + Bins []minuteBin + PeerTotals map[string]float64 + TotalBytes float64 + Job jobStatus + PodStates podStateCounts + PodStateError string + Progress progressGrid + Resources nodeResourceGrid + GridError string + NodePage int + NodesPerPage int + Color bool +} + +func parseRangeResponse(raw []byte) (rangeResponse, error) { + var envelope struct { + Status string `json:"status"` + Data struct { + Result []struct { + Metric map[string]string `json:"metric"` + Values [][2]json.RawMessage `json:"values"` + } `json:"result"` + } `json:"data"` + } + if err := json.Unmarshal(raw, &envelope); err != nil { + return rangeResponse{}, fmt.Errorf("decode Prometheus range response: %w", err) + } + + if envelope.Status != "success" { + return rangeResponse{}, fmt.Errorf("prometheus range query status is %q", envelope.Status) + } + + response := rangeResponse{Series: make([]rangeSeries, 0, len(envelope.Data.Result))} + for _, rawSeries := range envelope.Data.Result { + series := rangeSeries{Metric: rawSeries.Metric, Samples: make([]rangeSample, 0, len(rawSeries.Values))} + for _, pair := range rawSeries.Values { + var timestamp float64 + if err := json.Unmarshal(pair[0], ×tamp); err != nil { + return rangeResponse{}, fmt.Errorf("decode Prometheus sample timestamp: %w", err) + } + + var text string + if err := json.Unmarshal(pair[1], &text); err != nil { + return rangeResponse{}, fmt.Errorf("decode Prometheus sample value: %w", err) + } + + value, err := strconv.ParseFloat(text, 64) + if err != nil { + return rangeResponse{}, fmt.Errorf("parse Prometheus sample value %q: %w", text, err) + } + + if math.IsNaN(value) || math.IsInf(value, 0) { + continue + } + + seconds, fraction := math.Modf(timestamp) + series.Samples = append(series.Samples, rangeSample{ + Timestamp: time.Unix(int64(seconds), int64(fraction*float64(time.Second))).UTC(), + Value: value, + }) + } + + response.Series = append(response.Series, series) + } + + return response, nil +} + +func aggregateRange(response rangeResponse, phaseStart, now time.Time, expectedBytes float64, nodeCount int) monitorSnapshot { + elapsed := now.Sub(phaseStart) + if elapsed < 0 { + elapsed = 0 + } + + binCount := int(elapsed/time.Minute) + 1 + + bins := make([]minuteBin, binCount) + for index := range bins { + bins[index] = minuteBin{Minute: index, PeerOutcomes: map[string]float64{}} + } + + totals := map[string]float64{} + latest := time.Time{} + + for _, series := range response.Series { + sort.Slice(series.Samples, func(i, j int) bool { + return series.Samples[i].Timestamp.Before(series.Samples[j].Timestamp) + }) + + if len(series.Samples) == 0 { + continue + } + + outcome := series.Metric["outcome"] + + previous := series.Samples[0].Value + if series.Samples[0].Timestamp.After(latest) { + latest = series.Samples[0].Timestamp + } + + for _, sample := range series.Samples[1:] { + if sample.Timestamp.After(latest) { + latest = sample.Timestamp + } + + delta := sample.Value - previous + + previous = sample.Value + if delta <= 0 || sample.Timestamp.Before(phaseStart) { + continue + } + + minute := int(sample.Timestamp.Sub(phaseStart) / time.Minute) + if minute < 0 || minute >= len(bins) { + continue + } + + if outcome == "" { + bins[minute].Bytes += delta + continue + } + + bins[minute].PeerOutcomes[outcome] += delta + totals[outcome] += delta + } + } + + totalBytes := 0.0 + for _, bin := range bins { + totalBytes += bin.Bytes + } + + return monitorSnapshot{ + LatestSample: latest, + NodeCount: nodeCount, + ExpectedBytes: expectedBytes, + Bins: bins, + PeerTotals: totals, + TotalBytes: totalBytes, + } +} + +func commaInteger(value float64) string { + text := strconv.FormatInt(int64(math.Round(value)), 10) + + start := 0 + if strings.HasPrefix(text, "-") { + start = 1 + } + + for index := len(text) - 3; index > start; index -= 3 { + text = text[:index] + "," + text[index:] + } + + return text +} + +func percentage(numerator, denominator float64) float64 { + if denominator <= 0 { + return 0 + } + + return numerator / denominator * 100 +} + +func binDuration(snapshot monitorSnapshot, minute int) float64 { + if minute < len(snapshot.Bins)-1 { + return 60 + } + + seconds := snapshot.Now.Sub(snapshot.PhaseStart.Add(time.Duration(minute) * time.Minute)).Seconds() + if seconds < 1 { + return 1 + } + + if seconds > 60 { + return 60 + } + + return seconds +} + +func renderPeerTable(builder *strings.Builder, snapshot monitorSnapshot) { + fmt.Fprintln(builder, "=== Peer fetch outcomes by phase minute ===") + fmt.Fprintf(builder, "%4s %12s %10s %10s %10s %12s\n", "min", "busy", "hit", "stall", "notfound", "unavailable") + + for _, bin := range snapshot.Bins { + minute := strconv.Itoa(bin.Minute) + if bin.Minute == len(snapshot.Bins)-1 { + minute += "*" + } + + fmt.Fprintf(builder, "%4s %12s %10s %10s %10s %12s\n", + minute, + commaInteger(bin.PeerOutcomes["busy"]), + commaInteger(bin.PeerOutcomes["hit"]), + commaInteger(bin.PeerOutcomes["stall"]), + commaInteger(bin.PeerOutcomes["notfound"]), + commaInteger(bin.PeerOutcomes["unavailable"]), + ) + } + + fmt.Fprintf(builder, "\nTOTAL %10s %10s %10s %10s %12s\n", + commaInteger(snapshot.PeerTotals["busy"]), + commaInteger(snapshot.PeerTotals["hit"]), + commaInteger(snapshot.PeerTotals["stall"]), + commaInteger(snapshot.PeerTotals["notfound"]), + commaInteger(snapshot.PeerTotals["unavailable"]), + ) + + firstSixBusy := 0.0 + firstSixHit := 0.0 + + for _, bin := range snapshot.Bins { + if bin.Minute >= 6 { + break + } + + firstSixBusy += bin.PeerOutcomes["busy"] + firstSixHit += bin.PeerOutcomes["hit"] + } + + if snapshot.PeerTotals["busy"] > 0 { + fmt.Fprintf(builder, "\nbusy in first 6 min: %s of %s = %.1f%%\n", + commaInteger(firstSixBusy), commaInteger(snapshot.PeerTotals["busy"]), percentage(firstSixBusy, snapshot.PeerTotals["busy"])) + } + + if snapshot.PeerTotals["hit"] > 0 { + fmt.Fprintf(builder, "hit in first 6 min: %s of %s = %.1f%%\n", + commaInteger(firstSixHit), commaInteger(snapshot.PeerTotals["hit"]), percentage(firstSixHit, snapshot.PeerTotals["hit"])) + } +} + +func renderByteTable(builder *strings.Builder, snapshot monitorSnapshot) { + fmt.Fprintln(builder, "=== Layer bytes delivered by phase minute ===") + fmt.Fprintf(builder, "%4s %12s %16s %14s %8s\n", "min", "GB moved", "GB/s all-nodes", "MB/s per node", "cum %") + + cumulative := 0.0 + for _, bin := range snapshot.Bins { + cumulative += bin.Bytes + seconds := binDuration(snapshot, bin.Minute) + allNodesGBs := bin.Bytes / seconds / 1e9 + + perNodeMBs := 0.0 + if snapshot.NodeCount > 0 { + perNodeMBs = bin.Bytes / seconds / float64(snapshot.NodeCount) / 1e6 + } + + minute := strconv.Itoa(bin.Minute) + if bin.Minute == len(snapshot.Bins)-1 { + minute += "*" + } + + fmt.Fprintf(builder, "%4s %12s %16.1f %14.1f %7.1f%%\n", + minute, + commaInteger(bin.Bytes/1e9), + allNodesGBs, + perNodeMBs, + percentage(cumulative, snapshot.ExpectedBytes), + ) + } + + fmt.Fprintf(builder, "\ntotal %.3f TB of %.3f TB (%.1f%%)\n", + snapshot.TotalBytes/1e12, + snapshot.ExpectedBytes/1e12, + percentage(snapshot.TotalBytes, snapshot.ExpectedBytes), + ) +} + +func renderSnapshot(snapshot monitorSnapshot) string { + var builder strings.Builder + + elapsed := snapshot.Now.Sub(snapshot.PhaseStart) + if elapsed < 0 { + elapsed = 0 + } + + titleStart, metaStart, statusStart, reset := "", "", "", "" + if snapshot.Color { + titleStart = "\033[1;36m" + metaStart = "\033[2m" + statusStart = "\033[1m" + reset = "\033[0m" + } + + fmt.Fprintf(&builder, "%sGantry benchmark live monitor%s\n", titleStart, reset) + fmt.Fprintf(&builder, "%stime: %s\n", metaStart, snapshot.Now.UTC().Format(time.RFC3339)) + fmt.Fprintf(&builder, "run: %s\n", snapshot.RunID) + fmt.Fprintf(&builder, "job: %s\n", snapshot.JobName) + fmt.Fprintf(&builder, "phase started: %s (elapsed %s)%s\n", snapshot.PhaseStart.UTC().Format(time.RFC3339), elapsed.Round(time.Second), reset) + fmt.Fprintf(&builder, "%spods: %d/%d completed | %d running | %d creating | %d image-pull | %d failed%s\n", + statusStart, + snapshot.PodStates.Completed, + snapshot.NodeCount, + snapshot.PodStates.Running, + snapshot.PodStates.Creating, + snapshot.PodStates.ImagePull, + snapshot.PodStates.Failed, + reset, + ) + + if snapshot.PodStates.Unscheduled > 0 || snapshot.PodStates.Other > 0 { + fmt.Fprintf(&builder, "%spod detail: %d unscheduled, %d other%s\n", metaStart, snapshot.PodStates.Unscheduled, snapshot.PodStates.Other, reset) + } + + if snapshot.PodStateError != "" { + fmt.Fprintf(&builder, "%spod watch: %s%s\n", metaStart, snapshot.PodStateError, reset) + } + + renderNodeResources(&builder, snapshot) + renderProgressGrids(&builder, snapshot) + + fmt.Fprintf(&builder, "%sdisplay refresh: %s; Prometheus scrape cadence: 10s (values repeat between scrapes)\n", metaStart, snapshot.RefreshInterval) + + if !snapshot.LatestSample.IsZero() { + fmt.Fprintf(&builder, "latest query sample: %s\n", snapshot.LatestSample.UTC().Format(time.RFC3339)) + } + + fmt.Fprintf(&builder, "*: current partial minute%s\n", reset) + fmt.Fprintln(&builder) + + renderPeerTable(&builder, snapshot) + fmt.Fprintln(&builder) + renderByteTable(&builder, snapshot) + + return builder.String() +} diff --git a/hack/cmd/gantry-benchmark-monitor/monitor_test.go b/hack/cmd/gantry-benchmark-monitor/monitor_test.go new file mode 100644 index 000000000..aedb5845d --- /dev/null +++ b/hack/cmd/gantry-benchmark-monitor/monitor_test.go @@ -0,0 +1,200 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package main + +import ( + "fmt" + "strings" + "testing" + "time" +) + +func prometheusFixture(start time.Time) []byte { + return []byte(fmt.Sprintf(`{ + "status":"success", + "data":{"resultType":"matrix","result":[ + {"metric":{"outcome":"busy"},"values":[[%f,"100"],[%f,"110"],[%f,"130"],[%f,"135"]]}, + {"metric":{"outcome":"hit"},"values":[[%f,"20"],[%f,"22"],[%f,"25"]]}, + {"metric":{},"values":[[%f,"1000"],[%f,"60000001000"],[%f,"120000001000"],[%f,"150000001000"]]} + ]} +}`, + float64(start.Add(-10*time.Second).Unix()), + float64(start.Add(10*time.Second).Unix()), + float64(start.Add(70*time.Second).Unix()), + float64(start.Add(130*time.Second).Unix()), + float64(start.Add(-10*time.Second).Unix()), + float64(start.Add(30*time.Second).Unix()), + float64(start.Add(90*time.Second).Unix()), + float64(start.Add(-10*time.Second).Unix()), + float64(start.Add(59*time.Second).Unix()), + float64(start.Add(119*time.Second).Unix()), + float64(start.Add(130*time.Second).Unix()), + )) +} + +func TestParseAndAggregateRange(t *testing.T) { + start := time.Date(2026, 8, 7, 1, 0, 0, 0, time.UTC) + + response, err := parseRangeResponse(prometheusFixture(start)) + if err != nil { + t.Fatalf("parseRangeResponse: %v", err) + } + + snapshot := aggregateRange(response, start, start.Add(150*time.Second), 200e9, 1000) + if len(snapshot.Bins) != 3 { + t.Fatalf("bins = %d, want 3", len(snapshot.Bins)) + } + + checks := []struct { + minute int + busy float64 + hit float64 + bytes float64 + }{ + {minute: 0, busy: 10, hit: 2, bytes: 60e9}, + {minute: 1, busy: 20, hit: 3, bytes: 60e9}, + {minute: 2, busy: 5, hit: 0, bytes: 30e9}, + } + for _, check := range checks { + bin := snapshot.Bins[check.minute] + if bin.PeerOutcomes["busy"] != check.busy || bin.PeerOutcomes["hit"] != check.hit || bin.Bytes != check.bytes { + t.Errorf("minute %d = busy %.0f hit %.0f bytes %.0f; want %.0f %.0f %.0f", + check.minute, bin.PeerOutcomes["busy"], bin.PeerOutcomes["hit"], bin.Bytes, check.busy, check.hit, check.bytes) + } + } + + if snapshot.PeerTotals["busy"] != 35 || snapshot.PeerTotals["hit"] != 5 { + t.Fatalf("peer totals = %#v", snapshot.PeerTotals) + } + + if snapshot.TotalBytes != 150e9 { + t.Fatalf("total bytes = %.0f, want %.0f", snapshot.TotalBytes, 150e9) + } +} + +func TestRenderSnapshotIncludesBothLiveTables(t *testing.T) { + start := time.Date(2026, 8, 7, 1, 0, 0, 0, time.UTC) + + response, err := parseRangeResponse(prometheusFixture(start)) + if err != nil { + t.Fatal(err) + } + + snapshot := aggregateRange(response, start, start.Add(150*time.Second), 200e9, 1000) + snapshot.RunID = "run-1" + snapshot.JobName = "gantry-benchmark-gantry-cold-run-1" + snapshot.PhaseStart = start + snapshot.Now = start.Add(150 * time.Second) + snapshot.RefreshInterval = time.Second + snapshot.Job = jobStatus{Succeeded: 12, Active: 988} + snapshot.PodStates = podStateCounts{Completed: 12, Running: 20, Creating: 960, ImagePull: 8} + + output := renderSnapshot(snapshot) + for _, want := range []string{ + "=== Peer fetch outcomes by phase minute ===", + "busy", + "notfound", + "TOTAL", + "busy in first 6 min: 35 of 35 = 100.0%", + "=== Layer bytes delivered by phase minute ===", + "MB/s per node", + "2*", + "total 0.150 TB of 0.200 TB (75.0%)", + "pods: 12/1000 completed | 20 running | 960 creating | 8 image-pull | 0 failed", + "Prometheus scrape cadence: 10s", + } { + if !strings.Contains(output, want) { + t.Errorf("output is missing %q:\n%s", want, output) + } + } +} + +func TestRenderSnapshotHighlightsHeaderWhenColorEnabled(t *testing.T) { + t.Parallel() + + start := time.Date(2026, 8, 7, 1, 0, 0, 0, time.UTC) + snapshot := monitorSnapshot{ + RunID: "run-1", + JobName: "job-1", + PhaseStart: start, + Now: start.Add(time.Minute), + RefreshInterval: time.Second, + NodeCount: 1000, + Bins: []minuteBin{{Minute: 0, PeerOutcomes: map[string]float64{}}}, + PeerTotals: map[string]float64{}, + PodStates: podStateCounts{Completed: 1, Running: 2, Creating: 997}, + Color: true, + } + + output := renderSnapshot(snapshot) + for _, want := range []string{ + "\033[1;36mGantry benchmark live monitor\033[0m", + "\033[2mtime:", + "\033[1mpods: 1/1000 completed | 2 running | 997 creating", + } { + if !strings.Contains(output, want) { + t.Errorf("colored output is missing %q:\n%q", want, output) + } + } +} + +func TestPrometheusExpressionScopesCurrentRevision(t *testing.T) { + expression := prometheusExpression(monitorSession{revision: "gantry-abc123"}, "gantry-system") + for _, want := range []string{ + `p2p_peer_fetch_total`, + `gantry_mirror_bytes_served_total`, + `namespace="gantry-system"`, + `controller_revision_hash="gantry-abc123"`, + `kind="layer"`, + ` or `, + } { + if !strings.Contains(expression, want) { + t.Errorf("expression %q is missing %q", expression, want) + } + } +} + +func TestProgressExpressionScopesCurrentImage(t *testing.T) { + expression := progressExpression(monitorSession{ + revision: "gantry-abc123", + image: "registry.example/pull@sha256:image", + }, monitorConfig{ + gantryNamespace: "gantry-system", + benchmarkNamespace: "gantry-benchmark", + }) + + for _, want := range []string{ + `gantry_layer_download_completed_timestamp_seconds`, + `controller_revision_hash="gantry-abc123"`, + `image_digest="sha256:image"`, + `gantry_benchmark_(image_unpack_started|image_unpacked|layer_unpacked)_timestamp_seconds`, + `image="registry.example/pull@sha256:image"`, + `node_cpu_seconds_total`, + `node_memory_MemAvailable_bytes`, + } { + if !strings.Contains(expression, want) { + t.Errorf("expression %q is missing %q", expression, want) + } + } +} + +func TestCommaInteger(t *testing.T) { + for value, want := range map[float64]string{ + 0: "0", + 999: "999", + 1000: "1,000", + 1162435: "1,162,435", + -1234567: "-1,234,567", + } { + if got := commaInteger(value); got != want { + t.Errorf("commaInteger(%.0f) = %q, want %q", value, got, want) + } + } +} + +func TestParseRangeResponseRejectsFailure(t *testing.T) { + if _, err := parseRangeResponse([]byte(`{"status":"error"}`)); err == nil { + t.Fatal("parseRangeResponse succeeded for an error response") + } +} diff --git a/hack/cmd/gantry-benchmark-monitor/pods.go b/hack/cmd/gantry-benchmark-monitor/pods.go new file mode 100644 index 000000000..e47df9307 --- /dev/null +++ b/hack/cmd/gantry-benchmark-monitor/pods.go @@ -0,0 +1,300 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package main + +import ( + "context" + "fmt" + "sort" + "sync" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" +) + +type podState string + +const ( + podStateCreating podState = "creating" + podStateImagePull podState = "image-pull" + podStateRunning podState = "running" + podStateCompleted podState = "completed" + podStateFailed podState = "failed" + podStateUnscheduled podState = "unscheduled" + podStateOther podState = "other" +) + +type podStateCounts struct { + Creating int + ImagePull int + Running int + Completed int + Failed int + Unscheduled int + Other int +} + +type podStateTracker struct { + mu sync.RWMutex + states map[string]podState + nodes map[string]string + err error +} + +func loadKubeConfig(path string) (clientcmd.ClientConfig, error) { + rules := clientcmd.NewDefaultClientConfigLoadingRules() + if path != "" { + rules.ExplicitPath = path + } + + return clientcmd.NewNonInteractiveDeferredLoadingClientConfig(rules, &clientcmd.ConfigOverrides{}), nil +} + +func newPodStateTracker(ctx context.Context, kubeconfig, namespace, jobName string) (*podStateTracker, error) { + clientConfig, err := loadKubeConfig(kubeconfig) + if err != nil { + return nil, err + } + + restConfig, err := clientConfig.ClientConfig() + if err != nil { + return nil, fmt.Errorf("load Kubernetes client config: %w", err) + } + + client, err := kubernetes.NewForConfig(restConfig) + if err != nil { + return nil, fmt.Errorf("create Kubernetes client: %w", err) + } + + tracker := &podStateTracker{states: map[string]podState{}, nodes: map[string]string{}} + if err := tracker.replaceFromList(ctx, client, namespace, jobName); err != nil { + return nil, err + } + + go tracker.run(ctx, client, namespace, jobName) + + return tracker, nil +} + +func (t *podStateTracker) replaceFromList(ctx context.Context, client kubernetes.Interface, namespace, jobName string) error { + list, err := client.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{LabelSelector: "job-name=" + jobName}) + if err != nil { + return fmt.Errorf("list pull Job pods: %w", err) + } + + states := make(map[string]podState, len(list.Items)) + nodes := make(map[string]string, len(list.Items)) + + for index := range list.Items { + pod := &list.Items[index] + key := string(pod.UID) + states[key] = classifyPod(pod) + nodes[key] = pod.Spec.NodeName + } + + t.mu.Lock() + t.states = states + t.nodes = nodes + t.err = nil + t.mu.Unlock() + + return nil +} + +func (t *podStateTracker) run(ctx context.Context, client kubernetes.Interface, namespace, jobName string) { + for ctx.Err() == nil { + list, err := client.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{LabelSelector: "job-name=" + jobName}) + if err != nil { + t.setError(err) + + if !waitForRetry(ctx) { + return + } + + continue + } + + states := make(map[string]podState, len(list.Items)) + nodes := make(map[string]string, len(list.Items)) + + for index := range list.Items { + pod := &list.Items[index] + key := string(pod.UID) + states[key] = classifyPod(pod) + nodes[key] = pod.Spec.NodeName + } + + t.replace(states, nodes) + + watcher, err := client.CoreV1().Pods(namespace).Watch(ctx, metav1.ListOptions{ + LabelSelector: "job-name=" + jobName, + ResourceVersion: list.ResourceVersion, + AllowWatchBookmarks: true, + }) + if err != nil { + t.setError(err) + + if !waitForRetry(ctx) { + return + } + + continue + } + + closed := t.consumeWatch(ctx, watcher) + watcher.Stop() + + if !closed { + return + } + } +} + +func (t *podStateTracker) consumeWatch(ctx context.Context, watcher watch.Interface) bool { + for { + select { + case <-ctx.Done(): + return false + case event, ok := <-watcher.ResultChan(): + if !ok { + return true + } + + pod, ok := event.Object.(*corev1.Pod) + if !ok { + if event.Type == watch.Error { + t.setError(fmt.Errorf("pod watch returned an error event")) + return true + } + + continue + } + + key := string(pod.UID) + + t.mu.Lock() + if event.Type == watch.Deleted { + delete(t.states, key) + delete(t.nodes, key) + } else { + t.states[key] = classifyPod(pod) + t.nodes[key] = pod.Spec.NodeName + } + + t.err = nil + t.mu.Unlock() + } + } +} + +func waitForRetry(ctx context.Context) bool { + timer := time.NewTimer(time.Second) + defer timer.Stop() + + select { + case <-ctx.Done(): + return false + case <-timer.C: + return true + } +} + +func (t *podStateTracker) replace(states map[string]podState, nodes map[string]string) { + t.mu.Lock() + t.states = states + t.nodes = nodes + t.err = nil + t.mu.Unlock() +} + +func (t *podStateTracker) setError(err error) { + t.mu.Lock() + t.err = err + t.mu.Unlock() +} + +func (t *podStateTracker) snapshot() (podStateCounts, error) { + t.mu.RLock() + defer t.mu.RUnlock() + + counts := podStateCounts{} + + for _, state := range t.states { + switch state { + case podStateCreating: + counts.Creating++ + case podStateImagePull: + counts.ImagePull++ + case podStateRunning: + counts.Running++ + case podStateCompleted: + counts.Completed++ + case podStateFailed: + counts.Failed++ + case podStateUnscheduled: + counts.Unscheduled++ + case podStateOther: + counts.Other++ + } + } + + return counts, t.err +} + +func (t *podStateTracker) snapshotNodes() []string { + t.mu.RLock() + defer t.mu.RUnlock() + + seen := make(map[string]struct{}, len(t.nodes)) + for _, node := range t.nodes { + if node != "" { + seen[node] = struct{}{} + } + } + + nodes := make([]string, 0, len(seen)) + for node := range seen { + nodes = append(nodes, node) + } + + sort.Strings(nodes) + + return nodes +} + +func classifyPod(pod *corev1.Pod) podState { + switch pod.Status.Phase { + case corev1.PodSucceeded: + return podStateCompleted + case corev1.PodFailed: + return podStateFailed + case corev1.PodRunning: + return podStateRunning + } + + for _, status := range pod.Status.ContainerStatuses { + if status.Name != "pull" || status.State.Waiting == nil { + continue + } + + switch status.State.Waiting.Reason { + case "ImagePullBackOff", "ErrImagePull", "RegistryUnavailable": + return podStateImagePull + case "ContainerCreating", "PodInitializing": + return podStateCreating + default: + return podStateOther + } + } + + if pod.Spec.NodeName == "" { + return podStateUnscheduled + } + + return podStateCreating +} diff --git a/hack/cmd/gantry-benchmark-monitor/pods_test.go b/hack/cmd/gantry-benchmark-monitor/pods_test.go new file mode 100644 index 000000000..679f2e356 --- /dev/null +++ b/hack/cmd/gantry-benchmark-monitor/pods_test.go @@ -0,0 +1,103 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package main + +import ( + "errors" + "testing" + + corev1 "k8s.io/api/core/v1" +) + +func waitingPod(reason, node string) *corev1.Pod { + return &corev1.Pod{ + Spec: corev1.PodSpec{NodeName: node}, + Status: corev1.PodStatus{ + Phase: corev1.PodPending, + ContainerStatuses: []corev1.ContainerStatus{{ + Name: "pull", + State: corev1.ContainerState{ + Waiting: &corev1.ContainerStateWaiting{Reason: reason}, + }, + }}, + }, + } +} + +func TestClassifyPod(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + pod *corev1.Pod + want podState + }{ + {name: "completed", pod: &corev1.Pod{Status: corev1.PodStatus{Phase: corev1.PodSucceeded}}, want: podStateCompleted}, + {name: "failed", pod: &corev1.Pod{Status: corev1.PodStatus{Phase: corev1.PodFailed}}, want: podStateFailed}, + {name: "running", pod: &corev1.Pod{Status: corev1.PodStatus{Phase: corev1.PodRunning}}, want: podStateRunning}, + {name: "creating", pod: waitingPod("ContainerCreating", "node-a"), want: podStateCreating}, + {name: "initializing", pod: waitingPod("PodInitializing", "node-a"), want: podStateCreating}, + {name: "pull backoff", pod: waitingPod("ImagePullBackOff", "node-a"), want: podStateImagePull}, + {name: "pull error", pod: waitingPod("ErrImagePull", "node-a"), want: podStateImagePull}, + {name: "registry unavailable", pod: waitingPod("RegistryUnavailable", "node-a"), want: podStateImagePull}, + {name: "other waiting", pod: waitingPod("CreateContainerConfigError", "node-a"), want: podStateOther}, + {name: "unscheduled", pod: &corev1.Pod{Status: corev1.PodStatus{Phase: corev1.PodPending}}, want: podStateUnscheduled}, + {name: "scheduled pulling", pod: &corev1.Pod{Spec: corev1.PodSpec{NodeName: "node-a"}, Status: corev1.PodStatus{Phase: corev1.PodPending}}, want: podStateCreating}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + if got := classifyPod(test.pod); got != test.want { + t.Fatalf("classifyPod() = %q, want %q", got, test.want) + } + }) + } +} + +func TestPodStateTrackerSnapshot(t *testing.T) { + t.Parallel() + + tracker := &podStateTracker{ + states: map[string]podState{ + "a": podStateCompleted, + "b": podStateRunning, + "c": podStateCreating, + "d": podStateImagePull, + "e": podStateFailed, + "f": podStateUnscheduled, + "g": podStateOther, + }, + err: errors.New("watch reconnecting"), + } + + counts, err := tracker.snapshot() + if err == nil || err.Error() != "watch reconnecting" { + t.Fatalf("snapshot error = %v", err) + } + + want := podStateCounts{Completed: 1, Running: 1, Creating: 1, ImagePull: 1, Failed: 1, Unscheduled: 1, Other: 1} + if counts != want { + t.Fatalf("counts = %#v, want %#v", counts, want) + } +} + +func TestPodStateTrackerSnapshotNodes(t *testing.T) { + t.Parallel() + + tracker := &podStateTracker{nodes: map[string]string{ + "a": "node-c", + "b": "node-a", + "c": "node-c", + "d": "", + }} + + got := tracker.snapshotNodes() + want := []string{"node-a", "node-c"} + + if len(got) != len(want) || got[0] != want[0] || got[1] != want[1] { + t.Fatalf("snapshotNodes() = %v, want %v", got, want) + } +} diff --git a/hack/cmd/gantry-benchmark-monitor/resources.go b/hack/cmd/gantry-benchmark-monitor/resources.go new file mode 100644 index 000000000..7fba73bad --- /dev/null +++ b/hack/cmd/gantry-benchmark-monitor/resources.go @@ -0,0 +1,183 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package main + +import ( + "fmt" + "math" + "sort" + "strconv" + "strings" +) + +type nodeResourceGrid struct { + Nodes []string + CPUPercent map[string]float64 + MemoryPercent map[string]float64 +} + +type resourceSummary struct { + Samples int + P50 float64 + P95 float64 + Max float64 + MaxNode string +} + +func nodeResourceExpression(config monitorConfig) string { + labels := fmt.Sprintf(`namespace=%s,gantry_benchmark="true"`, strconv.Quote(config.benchmarkNamespace)) + cpu := fmt.Sprintf( + `label_replace(((1-avg by(instance)(rate(node_cpu_seconds_total{%s,mode="idle"}[1m])))*100) * on(instance) group_left(nodename) node_uname_info{%s},"resource","cpu","nodename",".*")`, + labels, + labels, + ) + memory := fmt.Sprintf( + `label_replace(((1-node_memory_MemAvailable_bytes{%s}/node_memory_MemTotal_bytes{%s})*100) * on(instance) group_left(nodename) node_uname_info{%s},"resource","memory","nodename",".*")`, + labels, + labels, + labels, + ) + + return cpu + " or " + memory +} + +func aggregateNodeResources(response instantResponse, nodes []string) nodeResourceGrid { + resources := nodeResourceGrid{ + Nodes: append([]string(nil), nodes...), + CPUPercent: map[string]float64{}, + MemoryPercent: map[string]float64{}, + } + sort.Strings(resources.Nodes) + + for _, series := range response.Series { + node := strings.ToLower(series.Metric["nodename"]) + if node == "" { + node = strings.ToLower(series.Metric["node"]) + } + + if node == "" { + continue + } + + value := min(100, max(0, series.Value)) + + switch series.Metric["resource"] { + case "cpu": + resources.CPUPercent[node] = value + case "memory": + resources.MemoryPercent[node] = value + } + } + + return resources +} + +func renderNodeResources(builder *strings.Builder, snapshot monitorSnapshot) { + if snapshot.GridError != "" { + return + } + + resources := snapshot.Resources + if len(resources.Nodes) == 0 || len(resources.CPUPercent)+len(resources.MemoryPercent) == 0 { + fmt.Fprintln(builder, "\nnode resources: waiting for node-exporter samples") + + return + } + + nodes, page, pages := pageNodes(resources.Nodes, snapshot.NodePage, snapshot.NodesPerPage) + + fmt.Fprintln(builder, "\n=== Node CPU and memory x nodes ===") + fmt.Fprintf(builder, "page %d/%d; nodes %d; showing %s .. %s\n", page, pages, len(resources.Nodes), nodes[0], nodes[len(nodes)-1]) + renderNodeHeader(builder, nodes) + renderResourceRow(builder, "CPU 1m", nodes, resources.CPUPercent) + renderResourceRow(builder, "MEM used", nodes, resources.MemoryPercent) + fmt.Fprintln(builder, "legend: 0=0-9%, 1=10-19%, ..., 9=90-100%, .=sample unavailable") + renderResourceSummary(builder, "fleet CPU 1m", summarizeResource(resources.CPUPercent, resources.Nodes), len(resources.Nodes)) + renderResourceSummary(builder, "fleet memory used", summarizeResource(resources.MemoryPercent, resources.Nodes), len(resources.Nodes)) +} + +func renderResourceRow(builder *strings.Builder, label string, nodes []string, values map[string]float64) { + fmt.Fprintf(builder, "%-10s", label) + + for _, node := range nodes { + builder.WriteByte(resourceCell(values, node)) + } + + builder.WriteByte('\n') +} + +func resourceCell(values map[string]float64, node string) byte { + value, ok := values[node] + if !ok || math.IsNaN(value) || math.IsInf(value, 0) { + return '.' + } + + decile := int(min(99.999, max(0, value)) / 10) + + return byte('0' + decile) +} + +func summarizeResource(values map[string]float64, nodes []string) resourceSummary { + summary := resourceSummary{} + samples := make([]float64, 0, len(nodes)) + + for _, node := range nodes { + value, ok := values[node] + if !ok || math.IsNaN(value) || math.IsInf(value, 0) { + continue + } + + samples = append(samples, value) + if summary.MaxNode == "" || value > summary.Max { + summary.Max = value + summary.MaxNode = node + } + } + + sort.Float64s(samples) + + summary.Samples = len(samples) + if len(samples) > 0 { + summary.P50 = resourceQuantile(samples, 0.50) + summary.P95 = resourceQuantile(samples, 0.95) + } + + return summary +} + +func resourceQuantile(sortedValues []float64, quantile float64) float64 { + if len(sortedValues) == 0 { + return 0 + } + + position := quantile * float64(len(sortedValues)-1) + lower := int(math.Floor(position)) + + upper := int(math.Ceil(position)) + if lower == upper { + return sortedValues[lower] + } + + weight := position - float64(lower) + + return sortedValues[lower]*(1-weight) + sortedValues[upper]*weight +} + +func renderResourceSummary(builder *strings.Builder, label string, summary resourceSummary, total int) { + if summary.Samples == 0 { + fmt.Fprintf(builder, "%s: 0/%d sampled\n", label, total) + + return + } + + fmt.Fprintf(builder, "%s: %d/%d sampled, p50 %.1f%%, p95 %.1f%%, max %.1f%% (%s)\n", + label, + summary.Samples, + total, + summary.P50, + summary.P95, + summary.Max, + summary.MaxNode, + ) +} diff --git a/hack/cmd/gantry-benchmark-monitor/resources_test.go b/hack/cmd/gantry-benchmark-monitor/resources_test.go new file mode 100644 index 000000000..5476db442 --- /dev/null +++ b/hack/cmd/gantry-benchmark-monitor/resources_test.go @@ -0,0 +1,78 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package main + +import ( + "strings" + "testing" +) + +func TestAggregateAndRenderNodeResources(t *testing.T) { + response := instantResponse{Series: []instantSeries{ + {Metric: map[string]string{"resource": "cpu", "nodename": "NODE-A"}, Value: 12}, + {Metric: map[string]string{"resource": "cpu", "nodename": "node-b"}, Value: 89}, + {Metric: map[string]string{"resource": "memory", "nodename": "NODE-A"}, Value: 55}, + {Metric: map[string]string{"resource": "memory", "nodename": "node-b"}, Value: 99}, + }} + resources := aggregateNodeResources(response, []string{"node-c", "node-b", "node-a"}) + + if resources.CPUPercent["node-a"] != 12 || resources.MemoryPercent["node-b"] != 99 { + t.Fatalf("resources = %#v", resources) + } + + var builder strings.Builder + renderNodeResources(&builder, monitorSnapshot{ + Resources: resources, + NodePage: 1, + NodesPerPage: 2, + }) + + output := builder.String() + for _, want := range []string{ + "=== Node CPU and memory x nodes ===", + "page 1/2; nodes 3; showing node-a .. node-b", + "CPU 1m 18", + "MEM used 59", + "fleet CPU 1m: 2/3 sampled", + "max 89.0% (node-b)", + "fleet memory used: 2/3 sampled", + } { + if !strings.Contains(output, want) { + t.Fatalf("resource grid missing %q:\n%s", want, output) + } + } +} + +func TestNodeResourceExpression(t *testing.T) { + expression := nodeResourceExpression(monitorConfig{benchmarkNamespace: "gantry-benchmark"}) + + for _, want := range []string{ + `node_cpu_seconds_total`, + `mode="idle"`, + `node_memory_MemAvailable_bytes`, + `node_memory_MemTotal_bytes`, + `node_uname_info`, + `namespace="gantry-benchmark"`, + `"resource","cpu"`, + `"resource","memory"`, + } { + if !strings.Contains(expression, want) { + t.Errorf("expression %q is missing %q", expression, want) + } + } +} + +func TestResourceCell(t *testing.T) { + values := map[string]float64{"low": 0, "middle": 55, "high": 100} + for node, want := range map[string]byte{ + "low": '0', + "middle": '5', + "high": '9', + "missing": '.', + } { + if got := resourceCell(values, node); got != want { + t.Errorf("resourceCell(%q) = %q, want %q", node, got, want) + } + } +} diff --git a/hack/cmd/gantry-benchmark/azure_preflight_test.go b/hack/cmd/gantry-benchmark/azure_preflight_test.go index b904c4f1d..f2e1420dc 100644 --- a/hack/cmd/gantry-benchmark/azure_preflight_test.go +++ b/hack/cmd/gantry-benchmark/azure_preflight_test.go @@ -193,3 +193,28 @@ func TestCheckAzureTelemetryRejectsPublicGantryACR(t *testing.T) { t.Fatalf("error = %v, want public Gantry ACR rejection", err) } } + +func TestDecodeAzureDiagnosticSettingsSupportsCLIAndARMShapes(t *testing.T) { + const setting = `{ + "logAnalyticsDestinationType":"Dedicated", + "workspaceId":"/subscriptions/s/workspaces/law", + "logs":[{"category":"kube-audit-admin","enabled":true}] + }` + + for name, raw := range map[string]string{ + "cli": `[ ` + setting + ` ]`, + "arm": `{"value":[{"properties":` + setting + `}]}`, + } { + t.Run(name, func(t *testing.T) { + settings, err := decodeAzureDiagnosticSettings([]byte(raw)) + if err != nil { + t.Fatalf("decodeAzureDiagnosticSettings: %v", err) + } + + if len(settings) != 1 || settings[0].WorkspaceID != "/subscriptions/s/workspaces/law" || + len(settings[0].Logs) != 1 || !settings[0].Logs[0].Enabled { + t.Fatalf("settings = %+v, want one dedicated audit setting", settings) + } + }) + } +} diff --git a/hack/cmd/gantry-benchmark/command.go b/hack/cmd/gantry-benchmark/command.go index 10c48046e..c2cd75add 100644 --- a/hack/cmd/gantry-benchmark/command.go +++ b/hack/cmd/gantry-benchmark/command.go @@ -10,12 +10,21 @@ import ( "io" "os/exec" "strings" + "sync" + "time" ) type commandRunner interface { Run(ctx context.Context, stdin []byte, name string, args ...string) ([]byte, error) } +// streamingCommandRunner is an optional capability. Long image builds and +// pushes emit progress for many minutes, so callers stream it live instead of +// buffering until the process exits. +type streamingCommandRunner interface { + RunStreaming(ctx context.Context, stdin []byte, progress io.Writer, name string, args ...string) ([]byte, error) +} + type execCommandRunner struct { directory string } @@ -36,6 +45,152 @@ func (r execCommandRunner) Run(ctx context.Context, stdin []byte, name string, a return output, nil } +func (r execCommandRunner) RunStreaming( + ctx context.Context, + stdin []byte, + progress io.Writer, + name string, + args ...string, +) ([]byte, error) { + command := exec.CommandContext(ctx, name, args...) + + command.Dir = r.directory + if stdin != nil { + command.Stdin = bytes.NewReader(stdin) + } + + var output bytes.Buffer + + // os/exec serializes writes when Stdout and Stderr are the same value. + sink := io.Writer(io.MultiWriter(&output, progress)) + command.Stdout = sink + command.Stderr = sink + + if err := command.Run(); err != nil { + return output.Bytes(), fmt.Errorf("%s %s: %w\n%s", name, strings.Join(args, " "), err, strings.TrimSpace(output.String())) + } + + return output.Bytes(), nil +} + +// prefixWriter reproduces child-process output one line at a time with a +// prefix. Carriage returns terminate a line so redrawn progress bars stream +// rather than accumulating into a single unbounded line. +type prefixWriter struct { + target io.Writer + prefix string + pending []byte +} + +func (w *prefixWriter) Write(p []byte) (int, error) { + w.pending = append(w.pending, p...) + + for { + index := bytes.IndexAny(w.pending, "\n\r") + if index < 0 { + break + } + + line := w.pending[:index] + w.pending = w.pending[index+1:] + + w.emit(line) + } + + return len(p), nil +} + +func (w *prefixWriter) Flush() { + if len(w.pending) == 0 { + return + } + + line := w.pending + w.pending = nil + + w.emit(line) +} + +func (w *prefixWriter) emit(line []byte) { + trimmed := strings.TrimRight(string(line), " \t") + if strings.TrimSpace(trimmed) == "" { + return + } + + writeAll(w.target, w.prefix+trimmed+"\n") +} + +// timestampWriter prefixes each complete line with a UTC timestamp. It is +// applied once at the process boundary so every progress line is stamped, and +// its lock also serializes the concurrent pull-progress reporter. +type timestampWriter struct { + target io.Writer + now func() time.Time + mu sync.Mutex + pending []byte +} + +func (w *timestampWriter) Write(p []byte) (int, error) { + w.mu.Lock() + defer w.mu.Unlock() + + w.pending = append(w.pending, p...) + + for { + index := bytes.IndexByte(w.pending, '\n') + if index < 0 { + break + } + + line := w.pending[:index] + w.pending = w.pending[index+1:] + + if err := w.emit(line); err != nil { + return len(p), err + } + } + + return len(p), nil +} + +// Flush writes any line not terminated by a newline, so a final partial line +// is not lost when the process exits. +func (w *timestampWriter) Flush() { + w.mu.Lock() + defer w.mu.Unlock() + + if len(w.pending) == 0 { + return + } + + line := w.pending + w.pending = nil + + _ = w.emit(line) //nolint:errcheck // Progress output is best effort. +} + +func (w *timestampWriter) emit(line []byte) error { + // Blank separator lines stay blank rather than becoming a bare timestamp. + if len(bytes.TrimSpace(line)) == 0 { + _, err := io.WriteString(w.target, "\n") + + return err + } + + clock := w.now + if clock == nil { + clock = time.Now + } + + _, err := io.WriteString(w.target, clock().UTC().Format("2006-01-02T15:04:05Z")+" "+string(line)+"\n") + + return err +} + func writeAll(writer io.Writer, value string) { + if writer == nil { + return + } + _, _ = io.WriteString(writer, value) //nolint:errcheck // CLI progress output is best effort. } diff --git a/hack/cmd/gantry-benchmark/command_streaming_test.go b/hack/cmd/gantry-benchmark/command_streaming_test.go new file mode 100644 index 000000000..e521eb9ec --- /dev/null +++ b/hack/cmd/gantry-benchmark/command_streaming_test.go @@ -0,0 +1,251 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package main + +import ( + "bytes" + "context" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" +) + +// signalOnFirstWrite creates syncPath as soon as any output arrives, which lets +// a child process block until it observes that the parent already received it. +type signalOnFirstWrite struct { + syncPath string + mu sync.Mutex + buffer bytes.Buffer + signaled bool +} + +func (w *signalOnFirstWrite) Write(p []byte) (int, error) { + w.mu.Lock() + defer w.mu.Unlock() + + w.buffer.Write(p) + + if !w.signaled { + w.signaled = true + if err := os.WriteFile(w.syncPath, []byte("go\n"), 0o600); err != nil { + return 0, err + } + } + + return len(p), nil +} + +func (w *signalOnFirstWrite) String() string { + w.mu.Lock() + defer w.mu.Unlock() + + return w.buffer.String() +} + +// The child only emits its second line after the parent has already consumed +// the first, so this deadlocks and times out if output is buffered until exit. +func TestRunStreamingDeliversOutputBeforeProcessExits(t *testing.T) { + directory := t.TempDir() + syncPath := filepath.Join(directory, "received.flag") + progress := &signalOnFirstWrite{syncPath: syncPath} + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + script := fmt.Sprintf( + "echo first; while [ ! -f %q ]; do sleep 0.01; done; echo second", + syncPath, + ) + + output, err := execCommandRunner{directory: directory}.RunStreaming(ctx, nil, progress, "sh", "-c", script) + if err != nil { + t.Fatalf("RunStreaming: %v", err) + } + + for _, want := range []string{"first", "second"} { + if !strings.Contains(string(output), want) { + t.Fatalf("returned output %q missing %q", string(output), want) + } + + if !strings.Contains(progress.String(), want) { + t.Fatalf("streamed output %q missing %q", progress.String(), want) + } + } +} + +func TestRunStreamingCapturesStderrAndReportsFailure(t *testing.T) { + var progress bytes.Buffer + + output, err := execCommandRunner{directory: t.TempDir()}.RunStreaming( + context.Background(), nil, &progress, "sh", "-c", "echo to-stderr >&2; exit 3", + ) + if err == nil { + t.Fatal("RunStreaming succeeded, want failure") + } + + if !strings.Contains(string(output), "to-stderr") { + t.Fatalf("returned output = %q, want stderr content", string(output)) + } + + if !strings.Contains(progress.String(), "to-stderr") { + t.Fatalf("streamed output = %q, want stderr content", progress.String()) + } +} + +func TestPrefixWriterSplitsProgressRedrawsAndFlushesRemainder(t *testing.T) { + var target bytes.Buffer + + writer := &prefixWriter{target: &target, prefix: " [push] "} + + // Carriage returns are how push progress bars redraw in place. + if _, err := writer.Write([]byte("Copying blob 10%\rCopying blob 60%\rCopying blob 100%\n")); err != nil { + t.Fatalf("Write: %v", err) + } + + if _, err := writer.Write([]byte("trailing without newline")); err != nil { + t.Fatalf("Write: %v", err) + } + + if strings.Contains(target.String(), "trailing") { + t.Fatalf("partial line emitted before Flush: %q", target.String()) + } + + writer.Flush() + + want := []string{ + " [push] Copying blob 10%", + " [push] Copying blob 60%", + " [push] Copying blob 100%", + " [push] trailing without newline", + } + if got := strings.Split(strings.TrimRight(target.String(), "\n"), "\n"); !slicesEqual(got, want) { + t.Fatalf("lines = %q, want %q", got, want) + } +} + +func TestPrefixWriterSkipsBlankLines(t *testing.T) { + var target bytes.Buffer + + writer := &prefixWriter{target: &target, prefix: "> "} + + if _, err := writer.Write([]byte("\n\r\n \nreal\n")); err != nil { + t.Fatalf("Write: %v", err) + } + + if target.String() != "> real\n" { + t.Fatalf("output = %q, want %q", target.String(), "> real\n") + } +} + +func slicesEqual(a, b []string) bool { + if len(a) != len(b) { + return false + } + + for i := range a { + if a[i] != b[i] { + return false + } + } + + return true +} + +func TestTimestampWriterStampsEveryLine(t *testing.T) { + var target bytes.Buffer + + fixed := time.Date(2026, 8, 6, 19, 30, 5, 0, time.UTC) + writer := ×tampWriter{target: &target, now: func() time.Time { return fixed }} + + if _, err := writer.Write([]byte("first\nsecond\n")); err != nil { + t.Fatalf("Write: %v", err) + } + + want := "2026-08-06T19:30:05Z first\n2026-08-06T19:30:05Z second\n" + if target.String() != want { + t.Fatalf("output = %q, want %q", target.String(), want) + } +} + +// Progress arriving in fragments must still produce exactly one stamp per line. +func TestTimestampWriterStampsOncePerLineAcrossPartialWrites(t *testing.T) { + var target bytes.Buffer + + fixed := time.Date(2026, 8, 6, 19, 30, 5, 0, time.UTC) + writer := ×tampWriter{target: &target, now: func() time.Time { return fixed }} + + for _, fragment := range []string{"pay", "load ", "layer 1/40"} { + if _, err := writer.Write([]byte(fragment)); err != nil { + t.Fatalf("Write: %v", err) + } + } + + if target.String() != "" { + t.Fatalf("partial line emitted before newline: %q", target.String()) + } + + if _, err := writer.Write([]byte("\n")); err != nil { + t.Fatalf("Write: %v", err) + } + + want := "2026-08-06T19:30:05Z payload layer 1/40\n" + if target.String() != want { + t.Fatalf("output = %q, want %q", target.String(), want) + } +} + +func TestTimestampWriterFlushEmitsTrailingLine(t *testing.T) { + var target bytes.Buffer + + fixed := time.Date(2026, 8, 6, 19, 30, 5, 0, time.UTC) + writer := ×tampWriter{target: &target, now: func() time.Time { return fixed }} + + if _, err := writer.Write([]byte("no trailing newline")); err != nil { + t.Fatalf("Write: %v", err) + } + + writer.Flush() + + want := "2026-08-06T19:30:05Z no trailing newline\n" + if target.String() != want { + t.Fatalf("output = %q, want %q", target.String(), want) + } +} + +func TestTimestampWriterIsConcurrencySafe(t *testing.T) { + var target bytes.Buffer + + writer := ×tampWriter{target: &target} + + var group sync.WaitGroup + + for i := range 8 { + group.Add(1) + + go func() { + defer group.Done() + + for j := range 25 { + writeAll(writer, fmt.Sprintf("worker %d line %d\n", i, j)) + } + }() + } + + group.Wait() + + lines := strings.Split(strings.TrimRight(target.String(), "\n"), "\n") + if len(lines) != 200 { + t.Fatalf("lines = %d, want 200", len(lines)) + } + + for _, line := range lines { + if !strings.Contains(line, "Z worker ") { + t.Fatalf("line is missing its timestamp: %q", line) + } + } +} diff --git a/hack/cmd/gantry-benchmark/config.go b/hack/cmd/gantry-benchmark/config.go index 05420db7e..2b34210cf 100644 --- a/hack/cmd/gantry-benchmark/config.go +++ b/hack/cmd/gantry-benchmark/config.go @@ -70,7 +70,10 @@ type benchmarkConfig struct { ACRPrivateEndpointResourceID string TelemetryTimeout time.Duration TelemetryPollInterval time.Duration + JobProgressInterval time.Duration StateRoot string + ImagePoolRoot string + ImagePoolBuildRoot string } type phaseRegistry struct { @@ -130,6 +133,11 @@ func loadBenchmarkConfig(getenv func(string) string) (benchmarkConfig, error) { return benchmarkConfig{}, err } + jobProgressInterval, err := envDuration(getenv, "BENCHMARK_JOB_PROGRESS_INTERVAL", 15*time.Second) + if err != nil { + return benchmarkConfig{}, err + } + mode := benchmarkMode(envDefault(getenv, "BENCHMARK_MODE", string(benchmarkModeProxy))) if mode != benchmarkModeProxy && mode != benchmarkModeDirect { return benchmarkConfig{}, fmt.Errorf( @@ -140,6 +148,8 @@ func loadBenchmarkConfig(getenv func(string) string) (benchmarkConfig, error) { ) } + stateRoot := filepath.Join(repoRoot, "tmp", "gantry-benchmark") + config := benchmarkConfig{ RepoRoot: repoRoot, Mode: mode, @@ -182,7 +192,10 @@ func loadBenchmarkConfig(getenv func(string) string) (benchmarkConfig, error) { ACRPrivateEndpointResourceID: getenv("AZURE_ACR_PRIVATE_ENDPOINT_RESOURCE_ID"), TelemetryTimeout: telemetryTimeout, TelemetryPollInterval: telemetryPollInterval, - StateRoot: filepath.Join(repoRoot, "tmp", "gantry-benchmark"), + JobProgressInterval: jobProgressInterval, + StateRoot: stateRoot, + ImagePoolRoot: envDefault(getenv, "BENCHMARK_IMAGE_POOL_ROOT", filepath.Join(stateRoot, "image-pool")), + ImagePoolBuildRoot: envDefault(getenv, "BENCHMARK_IMAGE_POOL_BUILD_ROOT", filepath.Join(stateRoot, "image-pool-build")), } if config.NodeCount <= 0 { diff --git a/hack/cmd/gantry-benchmark/direct_mode_test.go b/hack/cmd/gantry-benchmark/direct_mode_test.go index 2c51bcf19..ce568db50 100644 --- a/hack/cmd/gantry-benchmark/direct_mode_test.go +++ b/hack/cmd/gantry-benchmark/direct_mode_test.go @@ -457,6 +457,20 @@ func TestLoadBenchmarkConfigJobTimeoutIsFourHours(t *testing.T) { } } +func TestLoadBenchmarkConfigImagePoolRoots(t *testing.T) { + config, err := loadBenchmarkConfig(envFromMap(map[string]string{ + "BENCHMARK_IMAGE_POOL_ROOT": "/durable/pool", + "BENCHMARK_IMAGE_POOL_BUILD_ROOT": "/build/pool", + })) + if err != nil { + t.Fatalf("loadBenchmarkConfig: %v", err) + } + + if config.ImagePoolRoot != "/durable/pool" || config.ImagePoolBuildRoot != "/build/pool" { + t.Fatalf("image pool roots = %q and %q", config.ImagePoolRoot, config.ImagePoolBuildRoot) + } +} + func TestLoadBenchmarkConfigRejectsUnknownMode(t *testing.T) { _, err := loadBenchmarkConfig(envFromMap(map[string]string{"BENCHMARK_MODE": "proxyless"})) if err == nil { diff --git a/hack/cmd/gantry-benchmark/enable.go b/hack/cmd/gantry-benchmark/enable.go index 88aa6a3bf..57792e627 100644 --- a/hack/cmd/gantry-benchmark/enable.go +++ b/hack/cmd/gantry-benchmark/enable.go @@ -146,6 +146,8 @@ func (b *benchmark) enable(ctx context.Context) (returnErr error) { Namespace: b.config.Namespace, GantryNamespace: b.config.GantryNamespace, MonitoringLabel: b.config.KPSRelease, + NodeOS: strings.SplitN(b.config.ImagePlatform, "/", 2)[0], + NodeArch: strings.SplitN(b.config.ImagePlatform, "/", 2)[1], ProxyImage: b.config.ProxyImage, ACRLoginServer: b.config.ACRLoginServer, RunID: runID, @@ -162,6 +164,16 @@ func (b *benchmark) enable(ctx context.Context) (returnErr error) { return err } + if _, err := b.commands.Run( + ctx, + nil, + "kubectl", "-n", b.config.Namespace, + "rollout", "status", "daemonset/gantry-benchmark-node-observer", + "--timeout", b.config.RolloutTimeout.String(), + ); err != nil { + return fmt.Errorf("wait for benchmark node observer: %w", err) + } + if b.config.usesProxy() { secret := map[string]any{ "apiVersion": "v1", @@ -323,6 +335,8 @@ type proxyManifestData struct { Namespace string GantryNamespace string MonitoringLabel string + NodeOS string + NodeArch string ProxyImage string ACRLoginServer string RunID string diff --git a/hack/cmd/gantry-benchmark/enable_test.go b/hack/cmd/gantry-benchmark/enable_test.go index 1813ec14a..b30453d51 100644 --- a/hack/cmd/gantry-benchmark/enable_test.go +++ b/hack/cmd/gantry-benchmark/enable_test.go @@ -6,9 +6,14 @@ package main import ( "bytes" "io" + "os" + "os/exec" + "path/filepath" + "slices" "strings" "testing" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" utilyaml "k8s.io/apimachinery/pkg/util/yaml" ) @@ -68,6 +73,8 @@ func TestRenderMonitoringManifest(t *testing.T) { Namespace: "gantry-benchmark", GantryNamespace: "gantry-system", MonitoringLabel: "kps", + NodeOS: "linux", + NodeArch: "amd64", RunID: "run-1", }) if err != nil { @@ -83,19 +90,230 @@ func TestRenderMonitoringManifest(t *testing.T) { t.Fatalf("rendered manifest is missing benchmark scrape or Gantry revision labels") } - if !strings.Contains(string(rendered), `action: keep`) || - !strings.Contains(string(rendered), `gantry_storage_mode_info|p2p_dht_health_score|gantry_peer_serve_bytes_total`) { + if strings.Count(string(rendered), `gantry_benchmark: "true"`) != 2 { + t.Fatalf("rendered manifest does not label both benchmark PodMonitors for discovery") + } + + if !strings.Contains(string(rendered), `action: keep`) { t.Fatalf("rendered manifest does not limit Gantry metric cardinality") } + if !strings.Contains(string(rendered), `systemctl show --property MainPID --value containerd`) { + t.Fatalf("rendered manifest does not validate the running containerd debug configuration") + } + + if !strings.Contains(string(rendered), `- port: ctr-metrics`) || strings.Contains(string(rendered), `- port: containerd-metrics`) { + t.Fatalf("rendered manifest does not use the Kubernetes-valid containerd metrics port name") + } + + if !strings.Contains(string(rendered), `--web.listen-address=:29100`) { + t.Fatalf("rendered manifest does not use the benchmark node-exporter port") + } + + for _, metric := range []string{ + "p2p_peer_fetch_duration_seconds_(bucket|sum|count)", + "p2p_dht_lookup_duration_seconds_(bucket|sum|count)", + "gantry_peer_fetch_last_timestamp_seconds", + "gantry_mirror_response_completed_timestamp_seconds", + "gantry_layer_download_completed_timestamp_seconds", + "gantry_benchmark_(image_unpack_started|image_unpacked|layer_unpacked)_timestamp_seconds", + "gantry_containerd_commit_observation_duration_seconds_(bucket|sum|count)", + "gantry_containerd_commit_latest_observation_duration_seconds", + "node_uname_info", + "node_disk_(read|written)_bytes_total", + "node_network_speed_bytes", + "node_network_(receive|transmit)_(bytes|drop|errs)_total", + "containerd_.*|grpc_server_.*", + "process_(cpu_seconds_total|resident_memory_bytes|virtual_memory_bytes)", + } { + if !strings.Contains(string(rendered), metric) { + t.Fatalf("rendered manifest does not retain required metric %q", metric) + } + } + if strings.Contains(string(rendered), "acr-origin-proxy") { t.Fatalf("monitoring manifest must not reference the proxy") } + journalScript := renderedContainerScript(t, rendered, "gantry-benchmark-node-observer", "containerd-journal") + command := exec.Command("sh", "-n") + + command.Stdin = strings.NewReader(journalScript) + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("containerd-journal script syntax: %v: %s", err, output) + } + kinds := decodeManifestKinds(t, rendered) - if len(kinds) != 1 || kinds[0] != "PodMonitor" { - t.Fatalf("rendered kinds = %v, want [PodMonitor]", kinds) + wantKinds := []string{"PodMonitor", "DaemonSet", "PodMonitor"} + if !slices.Equal(kinds, wantKinds) { + t.Fatalf("rendered kinds = %v, want %v", kinds, wantKinds) + } +} + +func TestContainerdJournalProgressScript(t *testing.T) { + repoRoot, err := findRepoRoot() + if err != nil { + t.Fatalf("findRepoRoot: %v", err) + } + + benchmark := &benchmark{config: benchmarkConfig{RepoRoot: repoRoot}} + + rendered, err := benchmark.renderManifest(monitoringManifestPath, proxyManifestData{ + Namespace: "gantry-benchmark", + GantryNamespace: "gantry-system", + MonitoringLabel: "kps", + NodeOS: "linux", + NodeArch: "amd64", + RunID: "run-1", + }) + if err != nil { + t.Fatalf("renderManifest: %v", err) + } + + script := renderedContainerScript(t, rendered, "gantry-benchmark-node-observer", "containerd-journal") + tempDir := t.TempDir() + progressDir := filepath.Join(tempDir, "textfile") + + if err := os.Mkdir(progressDir, 0o750); err != nil { + t.Fatalf("mkdir textfile: %v", err) + } + + fakeChroot := `#!/bin/sh +shift +case "$1" in + systemctl) echo 123 ;; + readlink) echo /usr/bin/containerd ;; + /usr/bin/containerd) echo 'level = "debug"' ;; + journalctl) + cat <<'EOF' +level=info msg="PullImage request" image="registry.example/gantry-benchmark-pull@sha256:image" +level=debug msg="layer unpacked" duration=1s layer=sha256:aaaaaaaa +level=debug msg="image unpacked" duration=2s +EOF + ;; + *) echo "unexpected chroot command: $*" >&2; exit 1 ;; +esac +` + + fakePath := filepath.Join(tempDir, "chroot") + if err := os.WriteFile(fakePath, []byte(fakeChroot), 0o750); err != nil { + t.Fatalf("write fake chroot: %v", err) + } + + command := exec.Command("sh", "-c", script) + + command.Env = append(os.Environ(), + "PATH="+tempDir+":"+os.Getenv("PATH"), + "NODE_NAME=node-a", + "TEXTFILE_DIR="+progressDir, + ) + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("run containerd-journal script: %v: %s", err, output) + } + + wantFiles := map[string][]string{ + "image-start.prom": { + `gantry_benchmark_image_unpack_started_timestamp_seconds`, + `node="node-a"`, + `image="registry.example/gantry-benchmark-pull@sha256:image"`, + }, + "layer-aaaaaaaa.prom": { + `gantry_benchmark_layer_unpacked_timestamp_seconds`, + `layer_digest="sha256:aaaaaaaa"`, + }, + "image-complete.prom": { + `gantry_benchmark_image_unpacked_timestamp_seconds`, + }, + } + for name, fragments := range wantFiles { + content, err := os.ReadFile(filepath.Join(progressDir, name)) + if err != nil { + t.Fatalf("read %s: %v", name, err) + } + + for _, fragment := range fragments { + if !bytes.Contains(content, []byte(fragment)) { + t.Errorf("%s missing %q: %s", name, fragment, content) + } + } + } +} + +func renderedContainerScript(t *testing.T, rendered []byte, objectName, containerName string) string { + t.Helper() + + decoder := utilyaml.NewYAMLOrJSONDecoder(bytes.NewReader(rendered), 4096) + + for { + object := &unstructured.Unstructured{} + if err := decoder.Decode(object); err != nil { + if err == io.EOF { + break + } + + t.Fatalf("decode rendered manifest: %v", err) + } + + if object.GetName() != objectName { + continue + } + + containers, found, err := unstructured.NestedSlice(object.Object, "spec", "template", "spec", "containers") + if err != nil || !found { + t.Fatalf("containers for %s: found=%t err=%v", objectName, found, err) + } + + for _, raw := range containers { + container, ok := raw.(map[string]any) + if !ok || container["name"] != containerName { + continue + } + + command, ok := container["command"].([]any) + if !ok || len(command) < 3 { + t.Fatalf("container %s command = %#v", containerName, container["command"]) + } + + script, ok := command[len(command)-1].(string) + if !ok { + t.Fatalf("container %s script = %#v", containerName, command[len(command)-1]) + } + + return script + } + } + + t.Fatalf("container %s in %s not found", containerName, objectName) + + return "" +} + +func TestContainerdBenchmarkManifest(t *testing.T) { + repoRoot, err := findRepoRoot() + if err != nil { + t.Fatalf("findRepoRoot: %v", err) + } + + manifest, err := os.ReadFile(filepath.Join(repoRoot, "hack/gantry-benchmark/manifests/containerd.yaml")) + if err != nil { + t.Fatalf("read containerd manifest: %v", err) + } + + wantKinds := []string{"ConfigMap", "DaemonSet"} + if kinds := decodeManifestKinds(t, manifest); !slices.Equal(kinds, wantKinds) { + t.Fatalf("manifest kinds = %v, want %v", kinds, wantKinds) + } + + for _, setting := range []string{ + `level = "debug"`, + `image_pull_progress_timeout = "15m"`, + `max_concurrent_downloads = 6`, + `systemd-run`, + } { + if !bytes.Contains(manifest, []byte(setting)) { + t.Fatalf("containerd manifest is missing %q", setting) + } } } diff --git a/hack/cmd/gantry-benchmark/gantry_only.go b/hack/cmd/gantry-benchmark/gantry_only.go index 223c5d7bc..9ebe6798d 100644 --- a/hack/cmd/gantry-benchmark/gantry_only.go +++ b/hack/cmd/gantry-benchmark/gantry_only.go @@ -642,6 +642,11 @@ func (b *benchmark) runGantryOnly(ctx context.Context) (returnErr error) { return err } + diagnosticsBefore, err := b.fetchGantryDiagnosticSnapshot(ctx, revision) + if err != nil { + return err + } + writeAll(b.stdout, fmt.Sprintf("running Gantry-only cold pull on %d nodes\n", b.config.NodeCount)) job, err := b.runPullJob(ctx, state, proxyPhaseGantryCold, gantryImage) @@ -673,15 +678,47 @@ func (b *benchmark) runGantryOnly(ctx context.Context) (returnErr error) { return err } + diagnosticsAfter, err := b.fetchGantryDiagnosticSnapshot(ctx, revision) + if err != nil { + return err + } + + diagnosticTimestamps, err := b.fetchGantryDiagnosticTimestamps(ctx, revision, telemetryWindow{ + StartedAt: job.PhaseStartedAt, + FinishedAt: job.PhaseFinishedAt, + }) + if err != nil { + return err + } + + if err := requireFinalLayerResponseTimestamps(diagnosticTimestamps, diagnosticsAfter.PodNodes); err != nil { + return err + } + + diagnostics, err := subtractGantryDiagnosticSnapshots(diagnosticsBefore, diagnosticsAfter, diagnosticTimestamps) + if err != nil { + return err + } + bytes, bytesSource := deriveOriginBytes(b.config, proxyPhaseGantryCold, proxyPhaseTotals{}, metrics, job) + performance, err := b.capturePhasePerformanceTelemetry(ctx, proxyPhaseGantryCold, job) + if err != nil { + return err + } + + if err := b.writePerformanceTelemetryArtifact(state.RunID, proxyPhaseGantryCold, performance); err != nil { + return err + } + gantryResult := phaseResult{ RunID: state.RunID, Phase: proxyPhaseGantryCold, Image: gantryImage, ImageSizeMiB: b.config.ImageSizeMiB, ImageLayers: b.config.ImageLayers, PayloadSHA: state.WorkloadPayloadSHA256, WorkloadComparisonMode: state.WorkloadComparisonMode, - Gantry: metrics, GantryPeer: peer, + Gantry: metrics, GantryPeer: peer, GantryDiagnostics: diagnostics, Azure: azurePhaseMeasurement{Window: telemetryWindow{StartedAt: windowStart, FinishedAt: windowFinish}}, - Job: job, OriginBytes: bytes, OriginBytesSource: bytesSource, RecordedAt: time.Now().UTC(), + Job: job, OriginBytes: bytes, OriginBytesSource: bytesSource, + PerformanceTelemetryArtifact: string(proxyPhaseGantryCold) + "-performance.json", RecordedAt: time.Now().UTC(), } if b.config.AzureTelemetry { if err := b.collectAndPersistAzurePhase(ctx, &gantryResult, "gantry-cold.json"); err != nil { diff --git a/hack/cmd/gantry-benchmark/image.go b/hack/cmd/gantry-benchmark/image.go index 0bfc40a51..22ba3a562 100644 --- a/hack/cmd/gantry-benchmark/image.go +++ b/hack/cmd/gantry-benchmark/image.go @@ -14,10 +14,37 @@ import ( "os" "path/filepath" "strings" + "time" "github.com/opencontainers/go-digest" ) +// runStreaming reports child-process output live. Image builds and pushes run +// for many minutes, so buffering until exit hides all progress. +func (b *benchmark) runStreaming(ctx context.Context, prefix, name string, args ...string) ([]byte, error) { + streamer, ok := b.commands.(streamingCommandRunner) + if !ok { + return b.commands.Run(ctx, nil, name, args...) + } + + writer := &prefixWriter{target: b.stdout, prefix: prefix} + defer writer.Flush() + + return streamer.RunStreaming(ctx, nil, writer, name, args...) +} + +func formatMiB(sizeMiB int) string { + if sizeMiB >= 1024 { + return fmt.Sprintf("%.2f GiB", float64(sizeMiB)/1024) + } + + return fmt.Sprintf("%d MiB", sizeMiB) +} + +func formatElapsed(since time.Time) string { + return time.Since(since).Round(time.Second).String() +} + func (b *benchmark) loginRegistry(ctx context.Context, loginServer, username, password string) error { _, err := b.commands.Run( ctx, @@ -130,6 +157,62 @@ func (b *benchmark) prepareImages(ctx context.Context) error { return nil } +func (b *benchmark) prepareAdoptedImages(ctx context.Context, baselineImage, gantryImage, payloadSHA string) error { + state, err := b.loadState(ctx) + if err != nil { + return err + } + + if state.Status != "enabled" { + return fmt.Errorf("benchmark state is %q, run enable before prepare-adopt", state.Status) + } + + if state.usesProxy() { + return fmt.Errorf("prepare-adopt requires direct dual-ACR mode") + } + + if err := b.requireLock(ctx, state.RunID); err != nil { + return err + } + + if err := b.validateContext(ctx); err != nil { + return err + } + + state, err = adoptPreparedImages(state, baselineImage, gantryImage, payloadSHA) + if err != nil { + return err + } + + if err := b.saveState(ctx, state); err != nil { + return err + } + + writeAll(b.stdout, fmt.Sprintf("adopted digest-pinned images for %s using shared payload %s\n", state.RunID, payloadSHA)) + + return nil +} + +func adoptPreparedImages(state benchmarkState, baselineImage, gantryImage, payloadSHA string) (benchmarkState, error) { + payloadDigest, err := digest.Parse(payloadSHA) + if err != nil || payloadDigest.Algorithm() != digest.SHA256 { + return benchmarkState{}, fmt.Errorf("adopted payload fingerprint %q must be a valid sha256 digest", payloadSHA) + } + + state.BaselineImage = baselineImage + state.GantryColdImage = gantryImage + state.WorkloadPayloadSHA256 = payloadSHA + + state.WorkloadComparisonMode = workloadComparisonIdenticalPayload + if _, _, err := state.preparedImages(); err != nil { + return benchmarkState{}, fmt.Errorf("validate adopted images: %w", err) + } + + state.Status = "images-prepared" + + return state, nil +} + func (b *benchmark) buildDualACRImages(ctx context.Context, state benchmarkState) (string, string, string, error) { buildDirectory := filepath.Join(b.config.StateRoot, state.RunID, "build", "shared-payload") if err := os.RemoveAll(buildDirectory); err != nil { @@ -147,11 +230,17 @@ func (b *benchmark) buildDualACRImages(ctx context.Context, state benchmarkState defer removePayloads(payloadPaths) + hashStarted := time.Now() + + writeAll(b.stdout, fmt.Sprintf("hashing %s shared payload\n", formatMiB(b.config.ImageSizeMiB))) + payloadSHA, err := payloadSHA256(payloadPaths) if err != nil { return "", "", "", err } + writeAll(b.stdout, fmt.Sprintf("shared payload fingerprint %s in %s\n", payloadSHA, formatElapsed(hashStarted))) + for _, phase := range []proxyPhase{proxyPhaseBaseline, proxyPhaseGantryCold} { dockerfile := dualACRDockerfile(phase, payloadPaths, payloadSHA) if err := os.WriteFile(filepath.Join(buildDirectory, "Dockerfile."+string(phase)), []byte(dockerfile), 0o640); err != nil { @@ -163,11 +252,15 @@ func (b *benchmark) buildDualACRImages(ctx context.Context, state benchmarkState baselineTagged := fmt.Sprintf("%s/%s:%s", state.BaselineACRLoginServer, b.config.WorkloadRepository, tag) gantryTagged := fmt.Sprintf("%s/%s:%s", state.GantryACRLoginServer, b.config.WorkloadRepository, tag) + writeAll(b.stdout, fmt.Sprintf("image 1 of 2: baseline -> %s\n", baselineTagged)) + baselineDigest, err := b.buildAndPushPreparedImage(ctx, buildDirectory, proxyPhaseBaseline, baselineTagged) if err != nil { return "", "", "", fmt.Errorf("build and push baseline ACR image: %w", err) } + writeAll(b.stdout, fmt.Sprintf("image 2 of 2: Gantry -> %s\n", gantryTagged)) + gantryDigest, err := b.buildAndPushPreparedImage(ctx, buildDirectory, proxyPhaseGantryCold, gantryTagged) if err != nil { return "", "", "", fmt.Errorf("build and push Gantry ACR image: %w", err) @@ -189,6 +282,14 @@ func (b *benchmark) writeImagePayloads(buildDirectory string) ([]string, error) remainderMiB := b.config.ImageSizeMiB % layers payloadPaths := make([]string, 0, layers) + started := time.Now() + writtenMiB := 0 + + writeAll(b.stdout, fmt.Sprintf( + "generating %s of random payload across %d layers in %s\n", + formatMiB(b.config.ImageSizeMiB), layers, buildDirectory, + )) + for index := range layers { sizeMiB := perLayerMiB if index == layers-1 { @@ -196,6 +297,9 @@ func (b *benchmark) writeImagePayloads(buildDirectory string) ([]string, error) } path := filepath.Join(buildDirectory, fmt.Sprintf("payload%d.bin", index)) + + layerStarted := time.Now() + if err := writeRandomPayload(path, int64(sizeMiB)*mibibyte); err != nil { removePayloads(payloadPaths) @@ -203,8 +307,21 @@ func (b *benchmark) writeImagePayloads(buildDirectory string) ([]string, error) } payloadPaths = append(payloadPaths, path) + writtenMiB += sizeMiB + + writeAll(b.stdout, fmt.Sprintf( + " payload layer %d/%d: %s in %s (%s of %s, %.0f%%)\n", + index+1, layers, formatMiB(sizeMiB), formatElapsed(layerStarted), + formatMiB(writtenMiB), formatMiB(b.config.ImageSizeMiB), + float64(writtenMiB)/float64(b.config.ImageSizeMiB)*100, + )) } + writeAll(b.stdout, fmt.Sprintf( + "payload generation complete: %s in %s\n", + formatMiB(b.config.ImageSizeMiB), formatElapsed(started), + )) + return payloadPaths, nil } @@ -259,11 +376,16 @@ func (b *benchmark) buildAndPushPreparedImage(ctx context.Context, buildDirector switch b.config.ContainerEngine { case "docker": - if _, err := b.commands.Run( + buildStarted := time.Now() + + writeAll(b.stdout, fmt.Sprintf(" [%s] docker buildx build and push starting\n", phase)) + + if _, err := b.runStreaming( ctx, - nil, + " ["+string(phase)+"] ", "docker", "buildx", "build", "--platform", b.config.ImagePlatform, + "--progress", "plain", "--file", dockerfilePath, "--tag", taggedImage, "--output", "type=image,push=true,oci-mediatypes=true", @@ -275,6 +397,8 @@ func (b *benchmark) buildAndPushPreparedImage(ctx context.Context, buildDirector return "", err } + writeAll(b.stdout, fmt.Sprintf(" [%s] build and push complete in %s\n", phase, formatElapsed(buildStarted))) + metadata, err := os.ReadFile(metadataPath) if err != nil { return "", fmt.Errorf("read Buildx metadata: %w", err) @@ -289,9 +413,13 @@ func (b *benchmark) buildAndPushPreparedImage(ctx context.Context, buildDirector imageDigest = parsed.Digest case "podman": - if _, err := b.commands.Run( + buildStarted := time.Now() + + writeAll(b.stdout, fmt.Sprintf(" [%s] podman build starting\n", phase)) + + if _, err := b.runStreaming( ctx, - nil, + " ["+string(phase)+" build] ", "podman", "build", "--isolation", "chroot", "--platform", b.config.ImagePlatform, @@ -303,10 +431,22 @@ func (b *benchmark) buildAndPushPreparedImage(ctx context.Context, buildDirector return "", err } - if _, err := b.commands.Run(ctx, nil, "podman", "push", "--digestfile", digestPath, taggedImage); err != nil { + writeAll(b.stdout, fmt.Sprintf(" [%s] build complete in %s\n", phase, formatElapsed(buildStarted))) + + pushStarted := time.Now() + + writeAll(b.stdout, fmt.Sprintf(" [%s] pushing %s to %s\n", phase, formatMiB(b.config.ImageSizeMiB), taggedImage)) + + if _, err := b.runStreaming( + ctx, + " ["+string(phase)+" push] ", + "podman", "push", "--digestfile", digestPath, taggedImage, + ); err != nil { return "", err } + writeAll(b.stdout, fmt.Sprintf(" [%s] push complete in %s\n", phase, formatElapsed(pushStarted))) + pushedDigest, err := os.ReadFile(digestPath) if err != nil { return "", fmt.Errorf("read Podman push digest: %w", err) diff --git a/hack/cmd/gantry-benchmark/image_pool.go b/hack/cmd/gantry-benchmark/image_pool.go new file mode 100644 index 000000000..e83f4c8d7 --- /dev/null +++ b/hack/cmd/gantry-benchmark/image_pool.go @@ -0,0 +1,502 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "sort" + "strconv" + "time" +) + +const ( + gantryImagePoolSchemaVersion = 1 + maxGantryImagePrebuildCount = 100 +) + +type gantryImagePoolEntry struct { + SchemaVersion int `json:"schema_version"` + ID string `json:"id"` + CreatedAt time.Time `json:"created_at"` + Image string `json:"image"` + PayloadSHA256 string `json:"payload_sha256"` + ImageSizeMiB int `json:"image_size_mib"` + ImageLayers int `json:"image_layers"` + ImagePlatform string `json:"image_platform"` + WorkloadRepository string `json:"workload_repository"` + GantryACRLoginServer string `json:"gantry_acr_login_server"` + ClaimedByRunID string `json:"claimed_by_run_id,omitempty"` + ClaimedAt *time.Time `json:"claimed_at,omitempty"` +} + +func (b *benchmark) prebuildGantryImages(ctx context.Context, count int) error { + if count < 1 || count > maxGantryImagePrebuildCount { + return fmt.Errorf("prebuild count must be between 1 and %d, got %d", maxGantryImagePrebuildCount, count) + } + + if b.config.usesProxy() { + return errors.New("prebuild-gantry requires direct dual-ACR mode") + } + + if b.config.GantryACRLoginServer == "" || b.config.GantryACRUsername == "" || b.config.GantryACRPassword == "" { + return errors.New("prebuild-gantry requires GANTRY_ACR_LOGIN_SERVER, GANTRY_ACR_USERNAME, and GANTRY_ACR_PASSWORD") + } + + if err := b.ensureImagePoolDirectories(); err != nil { + return err + } + + if err := b.loginRegistry(ctx, b.config.GantryACRLoginServer, b.config.GantryACRUsername, b.config.GantryACRPassword); err != nil { + return fmt.Errorf("log in to Gantry ACR: %w", err) + } + + writeAll(b.stdout, fmt.Sprintf( + "prebuilding %d Gantry images (%s, %d layers) into %s\n", + count, + formatMiB(b.config.ImageSizeMiB), + b.config.ImageLayers, + b.config.ImagePoolRoot, + )) + + for index := range count { + entryID, err := newImagePoolEntryID() + if err != nil { + return err + } + + writeAll(b.stdout, fmt.Sprintf("pool image %d/%d: %s\n", index+1, count, entryID)) + + entry, taggedImage, err := b.buildGantryPoolImage(ctx, entryID) + if taggedImage != "" { + b.removeLocalImage(taggedImage) + } + + if err != nil { + return fmt.Errorf("prebuild pool image %s: %w", entryID, err) + } + + if err := writeJSONAtomic(b.imagePoolReadyPath(entry.ID), entry); err != nil { + return fmt.Errorf("record pool image %s: %w", entry.ID, err) + } + + writeAll(b.stdout, fmt.Sprintf( + "pool image ready: id=%s image=%s payload=%s\n", + entry.ID, + entry.Image, + entry.PayloadSHA256, + )) + } + + return b.printImagePoolStatus() +} + +func (b *benchmark) buildGantryPoolImage(ctx context.Context, entryID string) (gantryImagePoolEntry, string, error) { + buildDirectory := filepath.Join(b.imagePoolBuildDirectory(), entryID) + if err := os.RemoveAll(buildDirectory); err != nil { + return gantryImagePoolEntry{}, "", fmt.Errorf("clear image pool build directory: %w", err) + } + + defer func() { _ = os.RemoveAll(buildDirectory) }() //nolint:errcheck // Pool metadata and the pushed image are authoritative. + + if err := os.MkdirAll(buildDirectory, 0o750); err != nil { + return gantryImagePoolEntry{}, "", fmt.Errorf("create image pool build directory: %w", err) + } + + payloadPaths, err := b.writeImagePayloads(buildDirectory) + if err != nil { + return gantryImagePoolEntry{}, "", err + } + defer removePayloads(payloadPaths) + + payloadSHA, err := payloadSHA256(payloadPaths) + if err != nil { + return gantryImagePoolEntry{}, "", err + } + + dockerfile := dualACRDockerfile(proxyPhase("gantry-pool-"+entryID), payloadPaths, payloadSHA) + if err := os.WriteFile(filepath.Join(buildDirectory, "Dockerfile."+string(proxyPhaseGantryCold)), []byte(dockerfile), 0o640); err != nil { + return gantryImagePoolEntry{}, "", fmt.Errorf("write image pool Dockerfile: %w", err) + } + + taggedImage := fmt.Sprintf("%s/%s:%s", b.config.GantryACRLoginServer, b.config.WorkloadRepository, entryID) + + imageDigest, err := b.buildAndPushPreparedImage(ctx, buildDirectory, proxyPhaseGantryCold, taggedImage) + if err != nil { + return gantryImagePoolEntry{}, taggedImage, fmt.Errorf("build and push image pool entry: %w", err) + } + + return gantryImagePoolEntry{ + SchemaVersion: gantryImagePoolSchemaVersion, + ID: entryID, + CreatedAt: time.Now().UTC(), + Image: fmt.Sprintf("%s/%s@%s", b.config.GantryACRLoginServer, b.config.WorkloadRepository, imageDigest), + PayloadSHA256: payloadSHA, + ImageSizeMiB: b.config.ImageSizeMiB, + ImageLayers: b.config.ImageLayers, + ImagePlatform: b.config.ImagePlatform, + WorkloadRepository: b.config.WorkloadRepository, + GantryACRLoginServer: b.config.GantryACRLoginServer, + }, taggedImage, nil +} + +func (b *benchmark) removeLocalImage(taggedImage string) { + var args []string + + switch b.config.ContainerEngine { + case "podman", "docker": + args = []string{"image", "rm", "-f", taggedImage} + default: + return + } + + cleanupContext, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + if _, err := b.commands.Run(cleanupContext, nil, b.config.ContainerEngine, args...); err != nil { + writeAll(b.stderr, fmt.Sprintf("warning: remove local pool image %s: %v\n", taggedImage, err)) + } +} + +func (b *benchmark) prepareGantryOnlyFromPool(ctx context.Context, baselineRunID string) error { + state, err := b.loadState(ctx) + if err != nil { + return err + } + + if state.Status != "enabled" { + return fmt.Errorf("benchmark state is %q, run enable before prepare-gantry-pool", state.Status) + } + + if state.usesProxy() { + return errors.New("prepare-gantry-pool requires direct dual-ACR mode") + } + + if filepath.Base(baselineRunID) != baselineRunID || baselineRunID == "." || baselineRunID == "" { + return fmt.Errorf("invalid baseline run ID %q", baselineRunID) + } + + if err := b.requireLock(ctx, state.RunID); err != nil { + return err + } + + if err := b.validateContext(ctx); err != nil { + return err + } + + baselineState, err := b.readLocalState(baselineRunID) + if err != nil { + return fmt.Errorf("read baseline run state: %w", err) + } + + baselineResult, err := b.readPhaseResult(baselineRunID, "baseline.json") + if err != nil { + return fmt.Errorf("read retained baseline result: %w", err) + } + + if err := validateGantryOnlySource(state, baselineState, baselineResult); err != nil { + return err + } + + entry, readyPath, claimedPath, err := b.claimImagePoolEntry(state, baselineState) + if err != nil { + return err + } + + adopted := false + + defer func() { + if !adopted { + entry.ClaimedByRunID = "" + entry.ClaimedAt = nil + + if err := writeJSONAtomic(claimedPath, entry); err == nil { + _ = os.Rename(claimedPath, readyPath) //nolint:errcheck // Preserve the primary adoption error. + } + } + }() + + baselineResult.RunID = state.RunID + baselineResult.ImageLayers = state.ImageLayers + baselineResult.WorkloadComparisonMode = workloadComparisonRandomShape + state.BaselineImage = baselineState.BaselineImage + state.GantryColdImage = entry.Image + state.WorkloadPayloadSHA256 = entry.PayloadSHA256 + state.WorkloadComparisonMode = workloadComparisonRandomShape + state.Status = "images-prepared" + + if err := b.writeJSONArtifact(state.RunID, "baseline.json", baselineResult); err != nil { + return err + } + + if err := b.saveState(ctx, state); err != nil { + return err + } + + adopted = true + + writeAll(b.stdout, fmt.Sprintf( + "claimed prebuilt Gantry image %s for %s from pool entry %s\n", + entry.Image, + state.RunID, + entry.ID, + )) + + return nil +} + +func (b *benchmark) claimImagePoolEntry(current, baseline benchmarkState) (gantryImagePoolEntry, string, string, error) { + if err := b.ensureImagePoolDirectories(); err != nil { + return gantryImagePoolEntry{}, "", "", err + } + + entries, err := os.ReadDir(b.imagePoolReadyDirectory()) + if err != nil { + return gantryImagePoolEntry{}, "", "", fmt.Errorf("list ready image pool entries: %w", err) + } + + sort.Slice(entries, func(i, j int) bool { return entries[i].Name() < entries[j].Name() }) + + compatible := 0 + + for _, file := range entries { + if file.IsDir() || filepath.Ext(file.Name()) != ".json" { + continue + } + + readyPath := filepath.Join(b.imagePoolReadyDirectory(), file.Name()) + + entry, err := readImagePoolEntry(readyPath) + if err != nil || file.Name() != entry.ID+".json" || validateImagePoolEntry(entry, b.config, current, baseline) != nil { + continue + } + + compatible++ + + claimedName := current.RunID + "--" + file.Name() + + claimedPath := filepath.Join(b.imagePoolClaimedDirectory(), claimedName) + if err := os.Rename(readyPath, claimedPath); err != nil { + if errors.Is(err, fs.ErrNotExist) { + continue + } + + return gantryImagePoolEntry{}, "", "", fmt.Errorf("claim image pool entry %s: %w", entry.ID, err) + } + + entry.ClaimedByRunID = current.RunID + claimedAt := time.Now().UTC() + + entry.ClaimedAt = &claimedAt + if err := writeJSONAtomic(claimedPath, entry); err != nil { + _ = os.Rename(claimedPath, readyPath) //nolint:errcheck // Preserve metadata write error. + + return gantryImagePoolEntry{}, "", "", fmt.Errorf("record image pool claim %s: %w", entry.ID, err) + } + + return entry, readyPath, claimedPath, nil + } + + return gantryImagePoolEntry{}, "", "", fmt.Errorf( + "no compatible ready Gantry image in %s (examined %d entries, %d compatible before claim races)", + b.imagePoolReadyDirectory(), + len(entries), + compatible, + ) +} + +func validateImagePoolEntry(entry gantryImagePoolEntry, config benchmarkConfig, current, baseline benchmarkState) error { + if entry.SchemaVersion != gantryImagePoolSchemaVersion { + return fmt.Errorf("pool entry schema %d, want %d", entry.SchemaVersion, gantryImagePoolSchemaVersion) + } + + if entry.ID == "" || entry.ID == "." || entry.ID == ".." || filepath.Base(entry.ID) != entry.ID { + return fmt.Errorf("invalid pool entry ID %q", entry.ID) + } + + if entry.ImageSizeMiB != current.ImageSizeMiB || entry.ImageLayers != current.ImageLayers || + entry.ImagePlatform != current.ImagePlatform || entry.WorkloadRepository != current.WorkloadRepository || + entry.GantryACRLoginServer != current.GantryACRLoginServer { + return errors.New("pool entry shape does not match current benchmark") + } + + if entry.ImageSizeMiB != config.ImageSizeMiB || entry.ImageLayers != config.ImageLayers || + entry.ImagePlatform != config.ImagePlatform || entry.WorkloadRepository != config.WorkloadRepository || + entry.GantryACRLoginServer != config.GantryACRLoginServer { + return errors.New("pool entry shape does not match configured builder") + } + + return validateAdoptedFreshGantryImage(current, baseline, entry.Image, entry.PayloadSHA256) +} + +func (b *benchmark) printImagePoolStatus() error { + if err := b.ensureImagePoolDirectories(); err != nil { + return err + } + + ready, err := readImagePoolDirectory(b.imagePoolReadyDirectory()) + if err != nil { + return err + } + + claimed, err := readImagePoolDirectory(b.imagePoolClaimedDirectory()) + if err != nil { + return err + } + + writeAll(b.stdout, fmt.Sprintf("Gantry image pool: root=%s ready=%d claimed=%d\n", b.config.ImagePoolRoot, len(ready), len(claimed))) + + for _, entry := range ready { + writeAll(b.stdout, fmt.Sprintf( + "READY %s %s %s %s/%d\n", + entry.ID, + entry.CreatedAt.Format(time.RFC3339), + entry.Image, + formatMiB(entry.ImageSizeMiB), + entry.ImageLayers, + )) + } + + for _, entry := range claimed { + claimedAt := "unknown" + if entry.ClaimedAt != nil { + claimedAt = entry.ClaimedAt.Format(time.RFC3339) + } + + writeAll(b.stdout, fmt.Sprintf( + "CLAIMED %s run=%s at=%s image=%s\n", + entry.ID, + entry.ClaimedByRunID, + claimedAt, + entry.Image, + )) + } + + return nil +} + +func readImagePoolDirectory(directory string) ([]gantryImagePoolEntry, error) { + files, err := os.ReadDir(directory) + if err != nil { + return nil, fmt.Errorf("read image pool directory %s: %w", directory, err) + } + + entries := make([]gantryImagePoolEntry, 0, len(files)) + for _, file := range files { + if file.IsDir() || filepath.Ext(file.Name()) != ".json" { + continue + } + + entry, err := readImagePoolEntry(filepath.Join(directory, file.Name())) + if err != nil { + return nil, err + } + + entries = append(entries, entry) + } + + sort.Slice(entries, func(i, j int) bool { return entries[i].ID < entries[j].ID }) + + return entries, nil +} + +func readImagePoolEntry(path string) (gantryImagePoolEntry, error) { + var entry gantryImagePoolEntry + if err := readJSONFile(path, &entry); err != nil { + return gantryImagePoolEntry{}, fmt.Errorf("read image pool entry %s: %w", path, err) + } + + return entry, nil +} + +func writeJSONAtomic(path string, value any) error { + encoded, err := json.MarshalIndent(value, "", " ") + if err != nil { + return err + } + + if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil { + return err + } + + temporary, err := os.CreateTemp(filepath.Dir(path), ".pool-entry-*.tmp") + if err != nil { + return err + } + + temporaryPath := temporary.Name() + + defer func() { _ = os.Remove(temporaryPath) }() //nolint:errcheck // Rename removes successful temporary files. + + if err := temporary.Chmod(0o640); err != nil { + return errors.Join(err, temporary.Close()) + } + + if _, err := temporary.Write(append(encoded, '\n')); err != nil { + return errors.Join(err, temporary.Close()) + } + + if err := temporary.Sync(); err != nil { + return errors.Join(err, temporary.Close()) + } + + if err := temporary.Close(); err != nil { + return err + } + + return os.Rename(temporaryPath, path) +} + +func newImagePoolEntryID() (string, error) { + suffix, err := randomHex(4) + if err != nil { + return "", err + } + + return "pool-" + time.Now().UTC().Format("20060102-150405.000000000") + "-" + suffix, nil +} + +func (b *benchmark) ensureImagePoolDirectories() error { + for _, directory := range []string{b.imagePoolReadyDirectory(), b.imagePoolClaimedDirectory(), b.imagePoolBuildDirectory()} { + if err := os.MkdirAll(directory, 0o750); err != nil { + return fmt.Errorf("create image pool directory %s: %w", directory, err) + } + } + + return nil +} + +func (b *benchmark) imagePoolReadyDirectory() string { + return filepath.Join(b.config.ImagePoolRoot, "ready") +} + +func (b *benchmark) imagePoolClaimedDirectory() string { + return filepath.Join(b.config.ImagePoolRoot, "claimed") +} + +func (b *benchmark) imagePoolBuildDirectory() string { + if b.config.ImagePoolBuildRoot != "" { + return b.config.ImagePoolBuildRoot + } + + return filepath.Join(b.config.ImagePoolRoot, "build") +} + +func (b *benchmark) imagePoolReadyPath(entryID string) string { + return filepath.Join(b.imagePoolReadyDirectory(), entryID+".json") +} + +func parsePrebuildCount(value string) (int, error) { + count, err := strconv.Atoi(value) + if err != nil { + return 0, fmt.Errorf("parse prebuild count %q: %w", value, err) + } + + return count, nil +} diff --git a/hack/cmd/gantry-benchmark/image_pool_test.go b/hack/cmd/gantry-benchmark/image_pool_test.go new file mode 100644 index 000000000..670e7492c --- /dev/null +++ b/hack/cmd/gantry-benchmark/image_pool_test.go @@ -0,0 +1,229 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package main + +import ( + "bytes" + "context" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func poolTestStates() (benchmarkState, benchmarkState) { + current := benchmarkState{ + RunID: "run-new", Mode: benchmarkModeDirect, NodeCount: 1000, + ImagePlatform: "linux/amd64", ImageSizeMiB: 40960, ImageLayers: 40, + WorkloadRepository: "gantry-benchmark-pull", + BaselineACRLoginServer: "baseline.example", + GantryACRLoginServer: "gantry.example", + } + baseline := current + baseline.RunID = "run-baseline" + baseline.BaselineImage = "baseline.example/gantry-benchmark-pull@sha256:" + repeatHex("a") + baseline.WorkloadPayloadSHA256 = "sha256:" + repeatHex("b") + + return current, baseline +} + +func poolTestEntry(id, imageHex, payloadHex string) gantryImagePoolEntry { + return gantryImagePoolEntry{ + SchemaVersion: gantryImagePoolSchemaVersion, + ID: id, + CreatedAt: time.Date(2026, 8, 7, 1, 2, 3, 0, time.UTC), + Image: "gantry.example/gantry-benchmark-pull@sha256:" + repeatHex(imageHex), + PayloadSHA256: "sha256:" + repeatHex(payloadHex), + ImageSizeMiB: 40960, + ImageLayers: 40, + ImagePlatform: "linux/amd64", + WorkloadRepository: "gantry-benchmark-pull", + GantryACRLoginServer: "gantry.example", + } +} + +func TestValidateImagePoolEntry(t *testing.T) { + current, baseline := poolTestStates() + config := benchmarkConfig{ + Mode: benchmarkModeDirect, ImageSizeMiB: 40960, ImageLayers: 40, + ImagePlatform: "linux/amd64", WorkloadRepository: "gantry-benchmark-pull", + GantryACRLoginServer: "gantry.example", + } + entry := poolTestEntry("pool-a", "c", "d") + + if err := validateImagePoolEntry(entry, config, current, baseline); err != nil { + t.Fatalf("validate matching entry: %v", err) + } + + entry.ImageLayers++ + if err := validateImagePoolEntry(entry, config, current, baseline); err == nil { + t.Fatal("expected mismatched layer count to fail") + } + + entry = poolTestEntry("..", "c", "d") + if err := validateImagePoolEntry(entry, config, current, baseline); err == nil { + t.Fatal("expected unsafe entry ID to fail") + } +} + +func TestClaimImagePoolEntryMovesReadyEntryOnce(t *testing.T) { + root := t.TempDir() + current, baseline := poolTestStates() + + benchmark := &benchmark{config: benchmarkConfig{ + Mode: benchmarkModeDirect, ImagePoolRoot: root, + ImageSizeMiB: 40960, ImageLayers: 40, ImagePlatform: "linux/amd64", + WorkloadRepository: "gantry-benchmark-pull", GantryACRLoginServer: "gantry.example", + }} + if err := benchmark.ensureImagePoolDirectories(); err != nil { + t.Fatal(err) + } + + entry := poolTestEntry("pool-a", "c", "d") + + readyPath := benchmark.imagePoolReadyPath(entry.ID) + if err := writeJSONAtomic(readyPath, entry); err != nil { + t.Fatal(err) + } + + claimed, gotReadyPath, claimedPath, err := benchmark.claimImagePoolEntry(current, baseline) + if err != nil { + t.Fatalf("claimImagePoolEntry: %v", err) + } + + if claimed.ID != entry.ID || gotReadyPath != readyPath || claimed.ClaimedByRunID != current.RunID || claimed.ClaimedAt == nil { + t.Fatalf("claimed entry = %#v, readyPath=%q", claimed, gotReadyPath) + } + + if _, err := os.Stat(readyPath); !os.IsNotExist(err) { + t.Fatalf("ready entry still exists: %v", err) + } + + if _, err := os.Stat(claimedPath); err != nil { + t.Fatalf("claimed entry missing: %v", err) + } + + if _, _, _, err := benchmark.claimImagePoolEntry(current, baseline); err == nil { + t.Fatal("second claim unexpectedly succeeded") + } +} + +func TestReadImagePoolDirectorySortsEntries(t *testing.T) { + directory := t.TempDir() + for _, id := range []string{"pool-z", "pool-a"} { + if err := writeJSONAtomic(filepath.Join(directory, id+".json"), poolTestEntry(id, "c", "d")); err != nil { + t.Fatal(err) + } + } + + entries, err := readImagePoolDirectory(directory) + if err != nil { + t.Fatal(err) + } + + if len(entries) != 2 || entries[0].ID != "pool-a" || entries[1].ID != "pool-z" { + t.Fatalf("entries = %#v", entries) + } +} + +func TestPoolEntryClaimFieldsOmitWhenCleared(t *testing.T) { + path := filepath.Join(t.TempDir(), "pool-a.json") + entry := poolTestEntry("pool-a", "c", "d") + entry.ClaimedByRunID = "" + + entry.ClaimedAt = nil + if err := writeJSONAtomic(path, entry); err != nil { + t.Fatal(err) + } + + content, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + + if strings.Contains(string(content), "claimed_by_run_id") || strings.Contains(string(content), "claimed_at") { + t.Fatalf("cleared claim fields persisted: %s", content) + } +} + +func TestParsePrebuildCount(t *testing.T) { + if count, err := parsePrebuildCount("10"); err != nil || count != 10 { + t.Fatalf("parsePrebuildCount = %d, %v", count, err) + } + + if _, err := parsePrebuildCount("ten"); err == nil { + t.Fatal("expected invalid count to fail") + } +} + +func TestPrebuildGantryImagesCreatesReadyEntriesAndCleansBuilds(t *testing.T) { + poolRoot := filepath.Join(t.TempDir(), "pool") + buildRoot := filepath.Join(t.TempDir(), "build") + runner := &dualACRImageRunner{} + + var progress bytes.Buffer + + benchmark := &benchmark{ + config: benchmarkConfig{ + Mode: benchmarkModeDirect, + ImagePoolRoot: poolRoot, + ImagePoolBuildRoot: buildRoot, + ContainerEngine: "podman", + ImagePlatform: "linux/amd64", + ImageSizeMiB: 2, + ImageLayers: 2, + WorkloadRepository: "gantry-benchmark-pull", + GantryACRLoginServer: "gantry.azurecr.io", + GantryACRUsername: "user", + GantryACRPassword: "password", + }, + commands: runner, + stdout: &progress, + stderr: &progress, + } + + if err := benchmark.prebuildGantryImages(context.Background(), 2); err != nil { + t.Fatalf("prebuildGantryImages: %v", err) + } + + entries, err := readImagePoolDirectory(benchmark.imagePoolReadyDirectory()) + if err != nil { + t.Fatal(err) + } + + if len(entries) != 2 { + t.Fatalf("ready entries = %d, want 2", len(entries)) + } + + if entries[0].PayloadSHA256 == entries[1].PayloadSHA256 { + t.Fatal("pool entries unexpectedly reused random payload bytes") + } + + buildFiles, err := os.ReadDir(buildRoot) + if err != nil { + t.Fatal(err) + } + + if len(buildFiles) != 0 { + t.Fatalf("pool build root contains %d entries after cleanup", len(buildFiles)) + } + + commands := strings.Join(runner.commands, "\n") + if got := strings.Count(commands, "podman login "); got != 1 { + t.Fatalf("registry login count = %d, want 1:\n%s", got, commands) + } + + if got := strings.Count(commands, "podman push "); got != 2 { + t.Fatalf("push count = %d, want 2:\n%s", got, commands) + } + + if got := strings.Count(commands, "podman image rm -f "); got != 2 { + t.Fatalf("local image removal count = %d, want 2:\n%s", got, commands) + } + + if strings.Contains(commands, "kubectl") { + t.Fatalf("standalone prebuild unexpectedly invoked Kubernetes:\n%s", commands) + } +} diff --git a/hack/cmd/gantry-benchmark/image_test.go b/hack/cmd/gantry-benchmark/image_test.go index dd69553d4..77f802361 100644 --- a/hack/cmd/gantry-benchmark/image_test.go +++ b/hack/cmd/gantry-benchmark/image_test.go @@ -4,6 +4,7 @@ package main import ( + "bytes" "context" "os" "path/filepath" @@ -58,6 +59,9 @@ func (r *dualACRImageRunner) Run(_ context.Context, _ []byte, name string, args func TestBuildDualACRImagesUsesSharedPayloadAndSameImageName(t *testing.T) { runner := &dualACRImageRunner{} + + var progress bytes.Buffer + benchmark := &benchmark{ config: benchmarkConfig{ StateRoot: t.TempDir(), @@ -68,6 +72,7 @@ func TestBuildDualACRImagesUsesSharedPayloadAndSameImageName(t *testing.T) { WorkloadRepository: "benchmark-pull", }, commands: runner, + stdout: &progress, } state := benchmarkState{ RunID: "run-1", @@ -122,4 +127,66 @@ func TestBuildDualACRImagesUsesSharedPayloadAndSameImageName(t *testing.T) { string(baselineDockerfile) == string(gantryDockerfile) { t.Fatalf("phase Dockerfiles do not isolate content cache:\nbaseline:\n%s\nGantry:\n%s", baselineDockerfile, gantryDockerfile) } + + for _, want := range []string{ + "payload layer 1/2", + "payload layer 2/2", + "payload generation complete", + "shared payload fingerprint sha256:", + "image 1 of 2: baseline -> baseline.azurecr.io/benchmark-pull:run-1", + "image 2 of 2: Gantry -> gantry.azurecr.io/benchmark-pull:run-1", + "[baseline] build complete", + "[baseline] push complete", + "[gantry_cold] build complete", + "[gantry_cold] push complete", + } { + if !strings.Contains(progress.String(), want) { + t.Fatalf("progress output is missing %q:\n%s", want, progress.String()) + } + } +} + +func TestAdoptPreparedImages(t *testing.T) { + state := benchmarkState{ + Mode: benchmarkModeDirect, + Status: "enabled", + WorkloadRepository: "gantry-benchmark-pull", + BaselineACRLoginServer: "baseline.azurecr.io", + GantryACRLoginServer: "gantry.azurecr.io", + } + baseline := "baseline.azurecr.io/gantry-benchmark-pull@sha256:" + strings.Repeat("a", 64) + gantry := "gantry.azurecr.io/gantry-benchmark-pull@sha256:" + strings.Repeat("b", 64) + payload := "sha256:" + strings.Repeat("c", 64) + + adopted, err := adoptPreparedImages(state, baseline, gantry, payload) + if err != nil { + t.Fatalf("adoptPreparedImages: %v", err) + } + + if adopted.Status != "images-prepared" || adopted.BaselineImage != baseline || + adopted.GantryColdImage != gantry || adopted.WorkloadPayloadSHA256 != payload || + adopted.WorkloadComparisonMode != workloadComparisonIdenticalPayload { + t.Fatalf("adopted state = %+v", adopted) + } +} + +func TestAdoptPreparedImagesRejectsInvalidInputs(t *testing.T) { + state := benchmarkState{ + Mode: benchmarkModeDirect, + WorkloadRepository: "gantry-benchmark-pull", + BaselineACRLoginServer: "baseline.azurecr.io", + GantryACRLoginServer: "gantry.azurecr.io", + } + digestValue := "sha256:" + strings.Repeat("a", 64) + baseline := "baseline.azurecr.io/gantry-benchmark-pull@" + digestValue + gantry := "gantry.azurecr.io/gantry-benchmark-pull@" + digestValue + + if _, err := adoptPreparedImages(state, baseline, gantry, "not-a-digest"); err == nil { + t.Fatal("expected invalid payload digest rejection") + } + + if _, err := adoptPreparedImages(state, baseline, gantry, "sha256:"+strings.Repeat("c", 64)); err == nil || + !strings.Contains(err.Error(), "would reuse") { + t.Fatalf("error = %v, want identical image digest rejection", err) + } } diff --git a/hack/cmd/gantry-benchmark/job.go b/hack/cmd/gantry-benchmark/job.go index 1160c9446..ed091d3ac 100644 --- a/hack/cmd/gantry-benchmark/job.go +++ b/hack/cmd/gantry-benchmark/job.go @@ -19,14 +19,23 @@ type latencySummary struct { } type jobObservation struct { - JobName string `json:"job_name"` - PhaseStartedAt time.Time `json:"phase_started_at"` - PhaseFinishedAt time.Time `json:"phase_finished_at"` - Nodes []string `json:"nodes"` - Pods []string `json:"pods"` - PodNodes map[string]string `json:"pod_nodes"` - PodStartLatency latencySummary `json:"pod_start_latency"` - PodFinishLatency latencySummary `json:"pod_finish_latency"` + JobName string `json:"job_name"` + PhaseStartedAt time.Time `json:"phase_started_at"` + PhaseFinishedAt time.Time `json:"phase_finished_at"` + Nodes []string `json:"nodes"` + Pods []string `json:"pods"` + PodNodes map[string]string `json:"pod_nodes"` + PodTimings map[string]podTimingObservation `json:"pod_timings"` + PodStartLatency latencySummary `json:"pod_start_latency"` + PodFinishLatency latencySummary `json:"pod_finish_latency"` +} + +type podTimingObservation struct { + NodeName string `json:"node_name"` + ContainerStartedAt time.Time `json:"container_started_at"` + ContainerFinishedAt time.Time `json:"container_finished_at"` + StartLatencySeconds float64 `json:"start_latency_seconds"` + FinishLatencySeconds float64 `json:"finish_latency_seconds"` } type podList struct { @@ -127,14 +136,21 @@ func (b *benchmark) runPullJob(ctx context.Context, state benchmarkState, phase waitContext, cancel := context.WithTimeout(ctx, b.config.JobTimeout) defer cancel() - if _, err := b.commands.Run( + progressStopped := b.startPullJobProgress(waitContext, jobName, phaseStartedAt) + + _, waitErr := b.commands.Run( waitContext, nil, "kubectl", "-n", b.config.Namespace, "wait", "--for=condition=complete", "job/"+jobName, "--timeout", b.config.JobTimeout.String(), - ); err != nil { - return jobObservation{}, err + ) + + cancel() + <-progressStopped + + if waitErr != nil { + return jobObservation{}, waitErr } phaseFinishedAt := time.Now().UTC() @@ -173,8 +189,188 @@ func pullContainer(image string) map[string]any { } } +// pullPodList is the reduced projection used for live progress. It reads the +// waiting reason that podList intentionally omits. +type pullPodList struct { + Items []struct { + Spec struct { + NodeName string `json:"nodeName"` + } `json:"spec"` + Status struct { + Phase string `json:"phase"` + ContainerStatuses []struct { + Name string `json:"name"` + State struct { + Waiting *struct { + Reason string `json:"reason"` + } `json:"waiting"` + } `json:"state"` + } `json:"containerStatuses"` + } `json:"status"` + } `json:"items"` +} + +// pullProgress is a point-in-time breakdown of the pull Job's pods, reported +// while the phase runs so a stalled pull is visible before the Job times out. +type pullProgress struct { + Total int + Succeeded int + Running int + ContainerCreating int + PullingImage int + ImagePullBackOff int + Unscheduled int + Failed int + Other int + BackOffReasons map[string]int +} + +func (p pullProgress) String() string { + parts := []string{fmt.Sprintf("%d/%d succeeded", p.Succeeded, p.Total)} + + for _, entry := range []struct { + label string + count int + }{ + {"running", p.Running}, + {"pulling", p.PullingImage}, + {"creating", p.ContainerCreating}, + {"unscheduled", p.Unscheduled}, + {"failed", p.Failed}, + {"other", p.Other}, + } { + if entry.count > 0 { + parts = append(parts, fmt.Sprintf("%d %s", entry.count, entry.label)) + } + } + + if p.ImagePullBackOff > 0 { + reasons := make([]string, 0, len(p.BackOffReasons)) + for reason, count := range p.BackOffReasons { + reasons = append(reasons, fmt.Sprintf("%s=%d", reason, count)) + } + + sort.Strings(reasons) + parts = append(parts, fmt.Sprintf("%d image-pull-backoff (%s)", p.ImagePullBackOff, strings.Join(reasons, ","))) + } + + return strings.Join(parts, ", ") +} + +func summarizePullProgress(raw []byte, expectedPods int) (pullProgress, error) { + var pods pullPodList + if err := json.Unmarshal(raw, &pods); err != nil { + return pullProgress{}, fmt.Errorf("decode pull Job pod progress: %w", err) + } + + progress := pullProgress{Total: expectedPods, BackOffReasons: map[string]int{}} + + for _, pod := range pods.Items { + switch pod.Status.Phase { + case "Succeeded": + progress.Succeeded++ + + continue + case "Failed": + progress.Failed++ + + continue + case "Running": + progress.Running++ + + continue + } + + waitingReason := "" + + for _, status := range pod.Status.ContainerStatuses { + if status.Name == "pull" && status.State.Waiting != nil { + waitingReason = status.State.Waiting.Reason + } + } + + switch waitingReason { + case "ImagePullBackOff", "ErrImagePull", "RegistryUnavailable": + progress.ImagePullBackOff++ + progress.BackOffReasons[waitingReason]++ + case "ContainerCreating", "PodInitializing": + progress.ContainerCreating++ + case "": + if pod.Spec.NodeName == "" { + progress.Unscheduled++ + } else { + progress.PullingImage++ + } + default: + progress.Other++ + } + } + + return progress, nil +} + +// startPullJobProgress streams pod-state counts until ctx is cancelled. The +// returned channel closes once the reporter has stopped, so the caller can +// avoid interleaving output with whatever it prints next. +func (b *benchmark) startPullJobProgress(ctx context.Context, jobName string, phaseStartedAt time.Time) <-chan struct{} { + stopped := make(chan struct{}) + + if b.config.JobProgressInterval <= 0 { + close(stopped) + + return stopped + } + + go func() { + defer close(stopped) + + ticker := time.NewTicker(b.config.JobProgressInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + + output, err := b.commands.Run( + ctx, + nil, + "kubectl", "-n", b.config.Namespace, + "get", "pods", "-l", "job-name="+jobName, + "-o", "json", + ) + if err != nil { + if ctx.Err() != nil { + return + } + + writeAll(b.stdout, fmt.Sprintf(" [%s] pod progress unavailable: %v\n", jobName, err)) + + continue + } + + progress, err := summarizePullProgress(output, b.config.NodeCount) + if err != nil { + writeAll(b.stdout, fmt.Sprintf(" [%s] pod progress unreadable: %v\n", jobName, err)) + + continue + } + + writeAll(b.stdout, fmt.Sprintf( + " [%s] %s (elapsed %s)\n", + jobName, progress, time.Since(phaseStartedAt).Round(time.Second), + )) + } + }() + + return stopped +} + func parseJobObservation(raw []byte, expectedPods int, phaseStartedAt time.Time) (jobObservation, error) { var pods podList + if err := json.Unmarshal(raw, &pods); err != nil { return jobObservation{}, fmt.Errorf("decode pull Job pods: %w", err) } @@ -186,6 +382,7 @@ func parseJobObservation(raw []byte, expectedPods int, phaseStartedAt time.Time) nodeSet := make(map[string]struct{}, expectedPods) podNames := make([]string, 0, expectedPods) podNodes := make(map[string]string, expectedPods) + podTimings := make(map[string]podTimingObservation, expectedPods) startLatencies := make([]time.Duration, 0, expectedPods) finishLatencies := make([]time.Duration, 0, expectedPods) @@ -224,6 +421,13 @@ func parseJobObservation(raw []byte, expectedPods int, phaseStartedAt time.Time) startLatencies = append(startLatencies, terminated.StartedAt.Sub(phaseStartedAt)) finishLatencies = append(finishLatencies, terminated.FinishedAt.Sub(phaseStartedAt)) + podTimings[pod.Metadata.Name] = podTimingObservation{ + NodeName: pod.Spec.NodeName, + ContainerStartedAt: terminated.StartedAt, + ContainerFinishedAt: terminated.FinishedAt, + StartLatencySeconds: terminated.StartedAt.Sub(phaseStartedAt).Seconds(), + FinishLatencySeconds: terminated.FinishedAt.Sub(phaseStartedAt).Seconds(), + } terminatedFound = true break @@ -246,6 +450,7 @@ func parseJobObservation(raw []byte, expectedPods int, phaseStartedAt time.Time) Nodes: nodes, Pods: podNames, PodNodes: podNodes, + PodTimings: podTimings, PodStartLatency: summarizeLatencies(startLatencies), PodFinishLatency: summarizeLatencies(finishLatencies), }, nil diff --git a/hack/cmd/gantry-benchmark/job_progress_test.go b/hack/cmd/gantry-benchmark/job_progress_test.go new file mode 100644 index 000000000..f8a38e857 --- /dev/null +++ b/hack/cmd/gantry-benchmark/job_progress_test.go @@ -0,0 +1,224 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package main + +import ( + "bytes" + "context" + "encoding/json" + "strings" + "sync" + "testing" + "time" +) + +func pullProgressFixture(t *testing.T, pods []map[string]any) []byte { + t.Helper() + + raw, err := json.Marshal(map[string]any{"items": pods}) + if err != nil { + t.Fatalf("marshal fixture: %v", err) + } + + return raw +} + +func waitingPod(node, reason string) map[string]any { + return map[string]any{ + "spec": map[string]any{"nodeName": node}, + "status": map[string]any{ + "phase": "Pending", + "containerStatuses": []any{map[string]any{ + "name": "pull", + "state": map[string]any{"waiting": map[string]any{"reason": reason}}, + }}, + }, + } +} + +func phasePod(node, phase string) map[string]any { + return map[string]any{ + "spec": map[string]any{"nodeName": node}, + "status": map[string]any{"phase": phase}, + } +} + +func TestSummarizePullProgressClassifiesEveryPodState(t *testing.T) { + raw := pullProgressFixture(t, []map[string]any{ + phasePod("node-a", "Succeeded"), + phasePod("node-b", "Succeeded"), + phasePod("node-c", "Running"), + phasePod("node-d", "Failed"), + waitingPod("node-e", "ContainerCreating"), + waitingPod("node-f", "PodInitializing"), + waitingPod("node-g", "ImagePullBackOff"), + waitingPod("node-h", "ErrImagePull"), + waitingPod("node-i", "CreateContainerConfigError"), + // Scheduled with no waiting state: kubelet is pulling the image. + phasePod("node-j", "Pending"), + // No node assigned yet. + phasePod("", "Pending"), + }) + + progress, err := summarizePullProgress(raw, 11) + if err != nil { + t.Fatalf("summarizePullProgress: %v", err) + } + + for _, check := range []struct { + name string + got int + want int + }{ + {"total", progress.Total, 11}, + {"succeeded", progress.Succeeded, 2}, + {"running", progress.Running, 1}, + {"failed", progress.Failed, 1}, + {"containerCreating", progress.ContainerCreating, 2}, + {"imagePullBackOff", progress.ImagePullBackOff, 2}, + {"other", progress.Other, 1}, + {"pullingImage", progress.PullingImage, 1}, + {"unscheduled", progress.Unscheduled, 1}, + } { + if check.got != check.want { + t.Errorf("%s = %d, want %d", check.name, check.got, check.want) + } + } + + rendered := progress.String() + for _, want := range []string{ + "2/11 succeeded", + "1 running", + "2 creating", + "1 unscheduled", + "1 failed", + "2 image-pull-backoff (ErrImagePull=1,ImagePullBackOff=1)", + } { + if !strings.Contains(rendered, want) { + t.Errorf("rendered progress %q is missing %q", rendered, want) + } + } +} + +func TestSummarizePullProgressOmitsEmptyCategories(t *testing.T) { + raw := pullProgressFixture(t, []map[string]any{ + phasePod("node-a", "Succeeded"), + phasePod("node-b", "Succeeded"), + }) + + progress, err := summarizePullProgress(raw, 2) + if err != nil { + t.Fatalf("summarizePullProgress: %v", err) + } + + if got := progress.String(); got != "2/2 succeeded" { + t.Fatalf("rendered progress = %q, want %q", got, "2/2 succeeded") + } +} + +type progressPollRunner struct { + mu sync.Mutex + calls int + output []byte +} + +func (r *progressPollRunner) Run(_ context.Context, _ []byte, _ string, _ ...string) ([]byte, error) { + r.mu.Lock() + defer r.mu.Unlock() + + r.calls++ + + return r.output, nil +} + +func (r *progressPollRunner) callCount() int { + r.mu.Lock() + defer r.mu.Unlock() + + return r.calls +} + +func TestStartPullJobProgressReportsUntilCancelled(t *testing.T) { + runner := &progressPollRunner{ + output: pullProgressFixture(t, []map[string]any{ + phasePod("node-a", "Succeeded"), + waitingPod("node-b", "ImagePullBackOff"), + }), + } + + var progress lockedBuffer + + bench := &benchmark{ + config: benchmarkConfig{ + Namespace: "gantry-benchmark", + NodeCount: 2, + JobProgressInterval: time.Millisecond, + }, + commands: runner, + stdout: &progress, + } + + ctx, cancel := context.WithCancel(context.Background()) + stopped := bench.startPullJobProgress(ctx, "pull-job", time.Now()) + + deadline := time.After(10 * time.Second) + + for runner.callCount() == 0 { + select { + case <-deadline: + cancel() + <-stopped + t.Fatal("progress reporter never polled") + default: + } + } + + cancel() + <-stopped + + if !strings.Contains(progress.String(), "1/2 succeeded") || + !strings.Contains(progress.String(), "image-pull-backoff") { + t.Fatalf("progress output = %q", progress.String()) + } +} + +func TestStartPullJobProgressDisabledWhenIntervalUnset(t *testing.T) { + runner := &progressPollRunner{} + bench := &benchmark{ + config: benchmarkConfig{Namespace: "gantry-benchmark", NodeCount: 2}, + commands: runner, + stdout: &bytes.Buffer{}, + } + + stopped := bench.startPullJobProgress(context.Background(), "pull-job", time.Now()) + + select { + case <-stopped: + case <-time.After(5 * time.Second): + t.Fatal("disabled reporter did not close its channel") + } + + if got := runner.callCount(); got != 0 { + t.Fatalf("poll calls = %d, want 0", got) + } +} + +type lockedBuffer struct { + mu sync.Mutex + buffer bytes.Buffer +} + +func (b *lockedBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + + return b.buffer.Write(p) +} + +func (b *lockedBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + + return b.buffer.String() +} diff --git a/hack/cmd/gantry-benchmark/job_test.go b/hack/cmd/gantry-benchmark/job_test.go index 99542decd..de4e90246 100644 --- a/hack/cmd/gantry-benchmark/job_test.go +++ b/hack/cmd/gantry-benchmark/job_test.go @@ -37,8 +37,14 @@ func TestParseJobObservation(t *testing.T) { t.Fatalf("start latency = %+v", observation.PodStartLatency) } - if len(observation.Pods) != 4 || len(observation.PodNodes) != 4 { - t.Fatalf("pod identities = %v nodes=%v, want four", observation.Pods, observation.PodNodes) + if len(observation.Pods) != 4 || len(observation.PodNodes) != 4 || len(observation.PodTimings) != 4 { + t.Fatalf("pod identities = %v nodes=%v timings=%v, want four", observation.Pods, observation.PodNodes, observation.PodTimings) + } + + podB := observation.PodTimings["pod-b"] + if podB.NodeName != "node-b" || podB.StartLatencySeconds != 20 || podB.FinishLatencySeconds != 21 || + !podB.ContainerStartedAt.Equal(phaseStartedAt.Add(20*time.Second)) { + t.Fatalf("pod-b timing = %+v, want exact node and timestamps", podB) } } diff --git a/hack/cmd/gantry-benchmark/main.go b/hack/cmd/gantry-benchmark/main.go index 0a8262614..7e73faaff 100644 --- a/hack/cmd/gantry-benchmark/main.go +++ b/hack/cmd/gantry-benchmark/main.go @@ -23,8 +23,18 @@ func main() { ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() - if err := runCLI(ctx, os.Args[1:], os.Stdout, os.Stderr); err != nil { - fmt.Fprintln(os.Stderr, err) + stdout := ×tampWriter{target: os.Stdout} + stderr := ×tampWriter{target: os.Stderr} + + defer func() { + stdout.Flush() + stderr.Flush() + }() + + if err := runCLI(ctx, os.Args[1:], stdout, stderr); err != nil { + writeAll(stderr, err.Error()+"\n") + stdout.Flush() + stderr.Flush() os.Exit(1) } } @@ -49,12 +59,31 @@ func runCLI(ctx context.Context, args []string, stdout, stderr io.Writer) error } switch args[0] { + case "image-pool-status": + return benchmark.printImagePoolStatus() + case "prebuild-gantry": + if len(args) != 2 { + return fmt.Errorf("usage: gantry-benchmark prebuild-gantry ") + } + + count, err := parsePrebuildCount(args[1]) + if err != nil { + return err + } + + return benchmark.prebuildGantryImages(ctx, count) case "disable": return benchmark.disable(ctx) case "enable": return benchmark.enable(ctx) case "prepare": return benchmark.prepareImages(ctx) + case "prepare-adopt": + if len(args) != 4 { + return fmt.Errorf("usage: gantry-benchmark prepare-adopt ") + } + + return benchmark.prepareAdoptedImages(ctx, args[1], args[2], args[3]) case "prepare-gantry": if len(args) < 2 || len(args) > 3 { return fmt.Errorf("usage: gantry-benchmark prepare-gantry [prepared-run-id]") @@ -78,6 +107,12 @@ func runCLI(ctx context.Context, args []string, stdout, stderr io.Writer) error } return benchmark.prepareAdoptedFreshGantryOnly(ctx, args[1], args[2], args[3]) + case "prepare-gantry-pool": + if len(args) != 2 { + return fmt.Errorf("usage: gantry-benchmark prepare-gantry-pool ") + } + + return benchmark.prepareGantryOnlyFromPool(ctx, args[1]) case "preflight": return benchmark.preflight(ctx) case "run": @@ -97,15 +132,23 @@ func printUsage(writer io.Writer) { writeAll(writer, `Usage: gantry-benchmark Subcommands: + image-pool-status + list ready and claimed prebuilt Gantry images + prebuild-gantry + build and push reusable Gantry images without enabling a benchmark disable restore the cluster and remove benchmark instrumentation enable install benchmark instrumentation after safety checks prepare build and push both digest-pinned images before ACR goes private + prepare-adopt + adopt already-pushed direct-mode images with one shared payload prepare-gantry [prepared-run-id] rebuild only the Gantry image, or reuse an already-prepared image prepare-gantry-fresh generate new random bytes and build only a fresh Gantry image prepare-gantry-adopt adopt an already-pushed fresh Gantry image by immutable digest + prepare-gantry-pool + atomically claim and adopt one compatible prebuilt Gantry image preflight validate Azure sources, monitoring, Gantry, and all target nodes run execute baseline and Gantry cold phases, then restore routing run-gantry execute only Gantry cold against the retained baseline diff --git a/hack/cmd/gantry-benchmark/peer_telemetry.go b/hack/cmd/gantry-benchmark/peer_telemetry.go index 53ac7c51a..d4b4b945a 100644 --- a/hack/cmd/gantry-benchmark/peer_telemetry.go +++ b/hack/cmd/gantry-benchmark/peer_telemetry.go @@ -11,6 +11,8 @@ import ( "net/url" "sort" "strconv" + "strings" + "time" ) type gantryPeerPodMeasurement struct { @@ -37,6 +39,25 @@ type peerByteSnapshot struct { Counters map[string]map[string]uint64 } +type gantryPodDiagnosticMeasurement struct { + PodName string `json:"pod_name"` + NodeName string `json:"node_name"` + CounterDeltas map[string]float64 `json:"counter_deltas"` + TimestampSeconds map[string]float64 `json:"timestamp_seconds"` + FinalLayerResponseCompletedTimestampSeconds float64 `json:"final_layer_response_completed_timestamp_seconds,omitempty"` +} + +type gantryDiagnosticPhaseMeasurement struct { + Pods []gantryPodDiagnosticMeasurement `json:"pods"` + Source string `json:"source"` + Complete bool `json:"complete"` +} + +type gantryDiagnosticSnapshot struct { + PodNodes map[string]string + Counters map[string]map[string]float64 +} + func (b *benchmark) queryPrometheusSamples(ctx context.Context, query string) ([]prometheusSample, error) { rawPath := fmt.Sprintf( "/api/v1/namespaces/%s/services/http:%s:9090/proxy/api/v1/query?query=%s", @@ -133,6 +154,235 @@ func (b *benchmark) gantryPodNodes(ctx context.Context, revision string) (map[st return result, nil } +func (b *benchmark) fetchGantryDiagnosticSnapshot(ctx context.Context, revision string) (gantryDiagnosticSnapshot, error) { + podNodes, err := b.gantryPodNodes(ctx, revision) + if err != nil { + return gantryDiagnosticSnapshot{}, err + } + + query := fmt.Sprintf( + `{__name__=~"p2p_peer_fetch_total|p2p_peer_fetch_duration_seconds_(sum|count)|p2p_dht_lookup_total|p2p_dht_lookup_duration_seconds_(sum|count)|gantry_peer_fetch_bytes_total|gantry_containerd_commit_observed_total|gantry_containerd_commit_observation_duration_seconds_(sum|count)|gantry_containerd_commit_missing_after_stream_total",namespace=%q,gantry_benchmark="true",controller_revision_hash=%q}`, + b.config.GantryNamespace, + revision, + ) + + samples, err := b.queryPrometheusSamples(ctx, query) + if err != nil { + return gantryDiagnosticSnapshot{}, err + } + + counters := make(map[string]map[string]float64, len(podNodes)) + for pod := range podNodes { + counters[pod] = map[string]float64{} + } + + for _, sample := range samples { + pod := sample.Metric["pod"] + if _, ok := podNodes[pod]; !ok { + return gantryDiagnosticSnapshot{}, fmt.Errorf("diagnostic sample belongs to unexpected pod %q", pod) + } + + if sample.Value < 0 { + return gantryDiagnosticSnapshot{}, fmt.Errorf("diagnostic sample for pod %s is negative: %v", pod, sample.Value) + } + + key, err := diagnosticMetricKey(sample.Metric) + if err != nil { + return gantryDiagnosticSnapshot{}, err + } + + counters[pod][key] = sample.Value + } + + return gantryDiagnosticSnapshot{PodNodes: podNodes, Counters: counters}, nil +} + +func diagnosticMetricKey(labels map[string]string) (string, error) { + name := labels["__name__"] + if name == "" { + return "", fmt.Errorf("diagnostic sample has no __name__ label") + } + + parts := make([]string, 0, 3) + + for _, label := range []string{"kind", "outcome", "source"} { + if value := labels[label]; value != "" { + parts = append(parts, label+"="+value) + } + } + + if len(parts) == 0 { + return name, nil + } + + return name + "{" + strings.Join(parts, ",") + "}", nil +} + +func (b *benchmark) fetchGantryDiagnosticTimestamps( + ctx context.Context, + revision string, + window telemetryWindow, +) (map[string]map[string]float64, error) { + query := fmt.Sprintf( + `gantry_mirror_response_completed_timestamp_seconds{namespace=%q,kind="layer",gantry_benchmark="true",controller_revision_hash=%q} or gantry_containerd_commit_observed_timestamp_seconds{namespace=%q,gantry_benchmark="true",controller_revision_hash=%q} or gantry_peer_fetch_last_timestamp_seconds{namespace=%q,outcome=~"busy|stall",gantry_benchmark="true",controller_revision_hash=%q}`, + b.config.GantryNamespace, + revision, + b.config.GantryNamespace, + revision, + b.config.GantryNamespace, + revision, + ) + + raw, err := b.queryPrometheusRange(ctx, query, window, performanceTelemetryStep) + if err != nil { + return nil, err + } + + if err := validatePrometheusRangePodCoverage("gantry diagnostic timestamps", raw, b.config.NodeCount); err != nil { + return nil, err + } + + var response struct { + Data struct { + Result []struct { + Metric map[string]string `json:"metric"` + Values [][2]any `json:"values"` + } `json:"result"` + } `json:"data"` + } + if err := json.Unmarshal(raw, &response); err != nil { + return nil, fmt.Errorf("decode diagnostic timestamp range response: %w", err) + } + + result := map[string]map[string]float64{} + + for _, series := range response.Data.Result { + key, err := diagnosticMetricKey(series.Metric) + if err != nil { + return nil, err + } + + pod := series.Metric["pod"] + for _, pair := range series.Values { + rawValue, ok := pair[1].(string) + if !ok { + return nil, fmt.Errorf("diagnostic timestamp sample has non-string value") + } + + value, err := strconv.ParseFloat(rawValue, 64) + if err != nil { + return nil, fmt.Errorf("parse diagnostic timestamp sample %q: %w", rawValue, err) + } + + observedAt := time.Unix(0, int64(value*float64(time.Second))) + if observedAt.Before(window.StartedAt) || observedAt.After(window.FinishedAt) { + continue + } + + if result[pod] == nil { + result[pod] = map[string]float64{} + } + + if value > result[pod][key] { + result[pod][key] = value + } + } + } + + return result, nil +} + +func subtractGantryDiagnosticSnapshots( + before, after gantryDiagnosticSnapshot, + timestamps map[string]map[string]float64, +) (gantryDiagnosticPhaseMeasurement, error) { + if len(before.PodNodes) != len(after.PodNodes) { + return gantryDiagnosticPhaseMeasurement{}, fmt.Errorf( + "gantry diagnostic pod set changed during phase: before=%d after=%d", + len(before.PodNodes), + len(after.PodNodes), + ) + } + + pods := make([]gantryPodDiagnosticMeasurement, 0, len(before.PodNodes)) + for _, pod := range sortedMapKeys(before.PodNodes) { + node := before.PodNodes[pod] + if after.PodNodes[pod] != node { + return gantryDiagnosticPhaseMeasurement{}, fmt.Errorf("gantry pod %s disappeared or moved from node %s", pod, node) + } + + deltas := map[string]float64{} + + for key, afterValue := range after.Counters[pod] { + beforeValue := before.Counters[pod][key] + if afterValue < beforeValue { + return gantryDiagnosticPhaseMeasurement{}, fmt.Errorf( + "gantry diagnostic counter decreased for pod %s metric %s: before=%v after=%v", + pod, + key, + beforeValue, + afterValue, + ) + } + + if delta := afterValue - beforeValue; delta != 0 { + deltas[key] = delta + } + } + + pods = append(pods, gantryPodDiagnosticMeasurement{ + PodName: pod, + NodeName: node, + CounterDeltas: deltas, + TimestampSeconds: timestamps[pod], + FinalLayerResponseCompletedTimestampSeconds: finalLayerResponseCompletedTimestamp(timestamps[pod]), + }) + } + + return gantryDiagnosticPhaseMeasurement{ + Pods: pods, + Source: "per-pod Prometheus counter deltas and timestamp gauges", + Complete: len(pods) > 0, + }, nil +} + +func finalLayerResponseCompletedTimestamp(timestamps map[string]float64) float64 { + var latest float64 + for key, value := range timestamps { + if strings.HasPrefix(key, "gantry_mirror_response_completed_timestamp_seconds{kind=layer,") && value > latest { + latest = value + } + } + + return latest +} + +func requireFinalLayerResponseTimestamps( + timestamps map[string]map[string]float64, + podNodes map[string]string, +) error { + missing := make([]string, 0) + + for pod := range podNodes { + if finalLayerResponseCompletedTimestamp(timestamps[pod]) == 0 { + missing = append(missing, pod) + } + } + + if len(missing) == 0 { + return nil + } + + sort.Strings(missing) + + return fmt.Errorf( + "final layer response completion timestamp missing for %d/%d Gantry pods: %s", + len(missing), + len(podNodes), + strings.Join(missing, ","), + ) +} + func (b *benchmark) fetchGantryPeerByteSnapshot(ctx context.Context, revision string) (peerByteSnapshot, error) { podNodes, err := b.gantryPodNodes(ctx, revision) if err != nil { diff --git a/hack/cmd/gantry-benchmark/peer_telemetry_test.go b/hack/cmd/gantry-benchmark/peer_telemetry_test.go index e00f342a6..167af404a 100644 --- a/hack/cmd/gantry-benchmark/peer_telemetry_test.go +++ b/hack/cmd/gantry-benchmark/peer_telemetry_test.go @@ -3,7 +3,88 @@ package main -import "testing" +import ( + "context" + "fmt" + "strings" + "testing" + "time" +) + +type diagnosticTimestampRunner struct { + queryPath string +} + +func (r *diagnosticTimestampRunner) Run(_ context.Context, _ []byte, _ string, args ...string) ([]byte, error) { + r.queryPath = args[2] + + return []byte(fmt.Sprintf(`{"status":"success","data":{"resultType":"matrix","result":[ + {"metric":{"__name__":"gantry_mirror_response_completed_timestamp_seconds","pod":"gantry-a","kind":"layer","source":"peer"},"values":[[0,"%d"],[0,"%d"]]}, + {"metric":{"__name__":"gantry_containerd_commit_observed_timestamp_seconds","pod":"gantry-a"},"values":[[0,"%d"]]}, + {"metric":{"__name__":"gantry_containerd_commit_observed_timestamp_seconds","pod":"gantry-b"},"values":[[0,"%d"]]} + ]}}`, + time.Date(2026, time.August, 4, 1, 2, 30, 0, time.UTC).Unix(), + time.Date(2026, time.August, 4, 1, 3, 0, 0, time.UTC).Unix(), + time.Date(2026, time.August, 4, 1, 4, 0, 0, time.UTC).Unix(), + time.Date(2026, time.August, 4, 1, 5, 1, 0, time.UTC).Unix(), + )), nil +} + +func TestFetchGantryDiagnosticTimestampsUsesExactJobWindow(t *testing.T) { + runner := &diagnosticTimestampRunner{} + benchmark := &benchmark{ + config: benchmarkConfig{ + GantryNamespace: "gantry-system", + MonitoringNamespace: "monitoring", + PrometheusService: "prometheus", + NodeCount: 2, + }, + commands: runner, + } + window := telemetryWindow{ + StartedAt: time.Date(2026, time.August, 4, 1, 2, 0, 0, time.UTC), + FinishedAt: time.Date(2026, time.August, 4, 1, 5, 0, 0, time.UTC), + } + + timestamps, err := benchmark.fetchGantryDiagnosticTimestamps(context.Background(), "revision-a", window) + if err != nil { + t.Fatalf("fetchGantryDiagnosticTimestamps: %v", err) + } + + if len(timestamps["gantry-a"]) != 2 || timestamps["gantry-b"] != nil { + t.Fatalf("timestamps = %v, want only two in-window gantry-a values", timestamps) + } + + if timestamps["gantry-a"]["gantry_mirror_response_completed_timestamp_seconds{kind=layer,source=peer}"] != + float64(time.Date(2026, time.August, 4, 1, 3, 0, 0, time.UTC).Unix()) { + t.Fatalf("timestamps = %v, want latest in-window layer completion", timestamps) + } + + if !strings.Contains(runner.queryPath, `kind%3D%22layer%22`) { + t.Fatalf("query path %q does not restrict completion timestamps to layers", runner.queryPath) + } +} + +func TestRequireFinalLayerResponseTimestamps(t *testing.T) { + timestamps := map[string]map[string]float64{ + "gantry-a": { + "gantry_mirror_response_completed_timestamp_seconds{kind=layer,source=peer}": 1234, + }, + } + podNodes := map[string]string{"gantry-a": "node-a", "gantry-b": "node-b"} + + err := requireFinalLayerResponseTimestamps(timestamps, podNodes) + if err == nil || !strings.Contains(err.Error(), "gantry-b") { + t.Fatalf("error = %v, want missing gantry-b completion", err) + } + + timestamps["gantry-b"] = map[string]float64{ + "gantry_mirror_response_completed_timestamp_seconds{kind=layer,source=origin}": 1235, + } + if err := requireFinalLayerResponseTimestamps(timestamps, podNodes); err != nil { + t.Fatalf("requireFinalLayerResponseTimestamps: %v", err) + } +} func TestSubtractPeerByteSnapshots(t *testing.T) { before := peerByteSnapshot{ @@ -84,3 +165,69 @@ func TestSubtractPeerByteSnapshotsDefaultsMissingKindsToZero(t *testing.T) { t.Fatalf("measurement = %+v, want missing kinds treated as zero", measurement) } } + +func TestDiagnosticMetricKey(t *testing.T) { + key, err := diagnosticMetricKey(map[string]string{ + "__name__": "p2p_peer_fetch_total", + "outcome": "busy", + }) + if err != nil { + t.Fatalf("diagnosticMetricKey: %v", err) + } + + if key != "p2p_peer_fetch_total{outcome=busy}" { + t.Fatalf("key = %q, want p2p_peer_fetch_total{outcome=busy}", key) + } +} + +func TestSubtractGantryDiagnosticSnapshots(t *testing.T) { + before := gantryDiagnosticSnapshot{ + PodNodes: map[string]string{"gantry-a": "node-a"}, + Counters: map[string]map[string]float64{ + "gantry-a": {"p2p_peer_fetch_total{outcome=busy}": 10}, + }, + } + after := gantryDiagnosticSnapshot{ + PodNodes: map[string]string{"gantry-a": "node-a"}, + Counters: map[string]map[string]float64{ + "gantry-a": { + "p2p_peer_fetch_total{outcome=busy}": 13, + "p2p_dht_lookup_duration_seconds_sum{outcome=hit}": 1.25, + }, + }, + } + timestamps := map[string]map[string]float64{ + "gantry-a": {"gantry_mirror_response_completed_timestamp_seconds{kind=layer,source=peer}": 1234}, + } + + measurement, err := subtractGantryDiagnosticSnapshots(before, after, timestamps) + if err != nil { + t.Fatalf("subtractGantryDiagnosticSnapshots: %v", err) + } + + if !measurement.Complete || len(measurement.Pods) != 1 { + t.Fatalf("measurement = %+v, want one complete pod", measurement) + } + + pod := measurement.Pods[0] + if pod.NodeName != "node-a" || pod.CounterDeltas["p2p_peer_fetch_total{outcome=busy}"] != 3 || + pod.TimestampSeconds["gantry_mirror_response_completed_timestamp_seconds{kind=layer,source=peer}"] != 1234 || + pod.FinalLayerResponseCompletedTimestampSeconds != 1234 { + t.Fatalf("pod = %+v, want correlated deltas and timestamp", pod) + } +} + +func TestSubtractGantryDiagnosticSnapshotsRejectsReset(t *testing.T) { + before := gantryDiagnosticSnapshot{ + PodNodes: map[string]string{"gantry-a": "node-a"}, + Counters: map[string]map[string]float64{"gantry-a": {"counter": 2}}, + } + after := gantryDiagnosticSnapshot{ + PodNodes: map[string]string{"gantry-a": "node-a"}, + Counters: map[string]map[string]float64{"gantry-a": {"counter": 1}}, + } + + if _, err := subtractGantryDiagnosticSnapshots(before, after, nil); err == nil { + t.Fatal("expected a counter reset to invalidate diagnostic deltas") + } +} diff --git a/hack/cmd/gantry-benchmark/performance_telemetry.go b/hack/cmd/gantry-benchmark/performance_telemetry.go new file mode 100644 index 000000000..e3b6e65d4 --- /dev/null +++ b/hack/cmd/gantry-benchmark/performance_telemetry.go @@ -0,0 +1,410 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package main + +import ( + "context" + "encoding/json" + "fmt" + "net/url" + "regexp" + "strings" + "time" +) + +const ( + performanceTelemetryStep = 10 * time.Second + grpcHandledTelemetryStep = 5 * time.Minute + maxPrometheusRangeResponseBytes = 256 * 1024 * 1024 +) + +type prometheusRangeCapture struct { + Name string `json:"name"` + Query string `json:"query"` + StepSeconds int `json:"step_seconds"` + Response json.RawMessage `json:"response"` +} + +type performanceTelemetryQuery struct { + name string + query string + step time.Duration +} + +type containerdJournalEvent struct { + ObserverPod string `json:"observer_pod"` + NodeName string `json:"node_name"` + Timestamp time.Time `json:"timestamp"` + Type string `json:"type"` + DurationSeconds float64 `json:"duration_seconds,omitempty"` + LayerDigest string `json:"layer_digest,omitempty"` + Message string `json:"message"` +} + +type phasePerformanceTelemetry struct { + Window telemetryWindow `json:"window"` + ObserverPodNodes map[string]string `json:"observer_pod_nodes"` + Prometheus []prometheusRangeCapture `json:"prometheus"` + ContainerdJournal string `json:"containerd_journal"` + ContainerdJournalEvents []containerdJournalEvent `json:"containerd_journal_events"` + Complete bool `json:"complete"` +} + +var journalFieldPattern = regexp.MustCompile(`(?:^|[[:space:]])([a-zA-Z_]+)="?([^"[:space:]]+)"?`) + +func performanceTelemetryQueries() []performanceTelemetryQuery { + return []performanceTelemetryQuery{ + {name: "node_disk_read_bytes_per_second", query: `rate(node_disk_read_bytes_total{gantry_benchmark="true"}[30s])`}, + {name: "node_disk_written_bytes_per_second", query: `rate(node_disk_written_bytes_total{gantry_benchmark="true"}[30s])`}, + {name: "node_disk_busy_ratio", query: `rate(node_disk_io_time_seconds_total{gantry_benchmark="true"}[30s])`}, + {name: "node_disk_weighted_io_seconds_per_second", query: `rate(node_disk_io_time_weighted_seconds_total{gantry_benchmark="true"}[30s])`}, + {name: "node_network_receive_bytes_per_second", query: `rate(node_network_receive_bytes_total{gantry_benchmark="true",device!="lo"}[30s])`}, + {name: "node_network_transmit_bytes_per_second", query: `rate(node_network_transmit_bytes_total{gantry_benchmark="true",device!="lo"}[30s])`}, + {name: "node_network_receive_utilization_ratio", query: `rate(node_network_receive_bytes_total{gantry_benchmark="true",device!="lo"}[30s]) / node_network_speed_bytes{gantry_benchmark="true",device!="lo"}`}, + {name: "node_network_transmit_utilization_ratio", query: `rate(node_network_transmit_bytes_total{gantry_benchmark="true",device!="lo"}[30s]) / node_network_speed_bytes{gantry_benchmark="true",device!="lo"}`}, + {name: "node_network_receive_drops_per_second", query: `rate(node_network_receive_drop_total{gantry_benchmark="true",device!="lo"}[30s])`}, + {name: "node_network_transmit_drops_per_second", query: `rate(node_network_transmit_drop_total{gantry_benchmark="true",device!="lo"}[30s])`}, + {name: "node_network_receive_errors_per_second", query: `rate(node_network_receive_errs_total{gantry_benchmark="true",device!="lo"}[30s])`}, + {name: "node_network_transmit_errors_per_second", query: `rate(node_network_transmit_errs_total{gantry_benchmark="true",device!="lo"}[30s])`}, + {name: "node_cpu_busy_ratio", query: `1 - avg by(pod) (rate(node_cpu_seconds_total{gantry_benchmark="true",mode="idle"}[30s]))`}, + {name: "node_memory_available_bytes", query: `node_memory_MemAvailable_bytes{gantry_benchmark="true"}`}, + {name: "containerd_process", query: `{__name__=~"process_(cpu_seconds_total|resident_memory_bytes|virtual_memory_bytes)",gantry_benchmark="true",endpoint="ctr-metrics"}`}, + {name: "containerd_image_pulls", query: `{__name__=~"containerd_cri_sandboxed_(image_pulls_total|in_progress_image_pulls_total|image_pulling_throughput_(sum|count))",gantry_benchmark="true"}`}, + {name: "containerd_grpc_started", query: `sum by (pod, grpc_service, grpc_method) (rate(grpc_server_started_total{gantry_benchmark="true"}[30s])) > 0`}, + {name: "containerd_grpc_handled", query: `sum by (pod, grpc_code) (rate(grpc_server_handled_total{gantry_benchmark="true"}[5m])) > 0`, step: grpcHandledTelemetryStep}, + {name: "gantry_peer_outcomes", query: `p2p_peer_fetch_total{gantry_benchmark="true"}`}, + {name: "gantry_peer_busy_stall_timestamps", query: `gantry_peer_fetch_last_timestamp_seconds{outcome=~"busy|stall",gantry_benchmark="true"}`}, + {name: "gantry_peer_duration", query: `{__name__=~"p2p_peer_fetch_duration_seconds_(bucket|sum|count)",outcome=~"busy|stall",gantry_benchmark="true"}`}, + {name: "gantry_dht_outcomes", query: `p2p_dht_lookup_total{gantry_benchmark="true"}`}, + {name: "gantry_dht_duration", query: `{__name__=~"p2p_dht_lookup_duration_seconds_(bucket|sum|count)",gantry_benchmark="true"}`}, + {name: "gantry_mirror_bytes", query: `gantry_mirror_bytes_served_total{gantry_benchmark="true"}`}, + {name: "gantry_response_completed", query: `gantry_mirror_response_completed_timestamp_seconds{kind="layer",gantry_benchmark="true"}`}, + {name: "gantry_commit_observation", query: `{__name__=~"gantry_containerd_commit_(observed_total|observed_timestamp_seconds|observation_duration_seconds_(sum|count)|latest_observation_duration_seconds|missing_after_stream_total)",gantry_benchmark="true"}`}, + // Seeding width: how many HRW pullers each layer actually activates, and + // how many please_pull requests convert into origin pulls versus dedup. + {name: "gantry_coord_seeding", query: `{__name__=~"p2p_coord_please_pull_(served|started|declined)_total|p2p_coord_pull_intent_served_total|p2p_prefetch_(batches|digests|groups)_total",gantry_benchmark="true"}`}, + {name: "gantry_prefetch_pullers", query: `{__name__=~"p2p_prefetch_pullers_per_manifest_(bucket|sum|count)",gantry_benchmark="true"}`}, + } +} + +func (b *benchmark) capturePhasePerformanceTelemetry( + ctx context.Context, + phase proxyPhase, + job jobObservation, +) (phasePerformanceTelemetry, error) { + window := telemetryWindow{ + StartedAt: job.PhaseStartedAt, + FinishedAt: job.PhaseFinishedAt, + } + + observerPods, err := b.observerPodNodes(ctx) + if err != nil { + return phasePerformanceTelemetry{}, err + } + + queries := performanceTelemetryQueries() + + captures := make([]prometheusRangeCapture, 0, len(queries)) + for _, item := range queries { + step := item.step + if step == 0 { + step = performanceTelemetryStep + } + + response, err := b.queryPrometheusRange(ctx, item.query, window, step) + if err != nil { + return phasePerformanceTelemetry{}, fmt.Errorf("capture %s: %w", item.name, err) + } + + if err := validatePrometheusRangePodCoverage(item.name, response, b.config.NodeCount); err != nil { + return phasePerformanceTelemetry{}, err + } + + captures = append(captures, prometheusRangeCapture{ + Name: item.name, + Query: item.query, + StepSeconds: int(step.Seconds()), + Response: response, + }) + } + + journal, err := b.collectContainerdJournal(ctx, window) + if err != nil { + return phasePerformanceTelemetry{}, err + } + + journalEvents, err := parseContainerdJournal(journal, observerPods, window) + if err != nil { + return phasePerformanceTelemetry{}, err + } + + return phasePerformanceTelemetry{ + Window: window, + ObserverPodNodes: observerPods, + Prometheus: captures, + ContainerdJournal: journal, + ContainerdJournalEvents: journalEvents, + Complete: true, + }, nil +} + +func validatePrometheusRangePodCoverage(name string, raw json.RawMessage, expectedPods int) error { + var response struct { + Data struct { + Result []struct { + Metric map[string]string `json:"metric"` + Values [][2]any `json:"values"` + } `json:"result"` + } `json:"data"` + } + if err := json.Unmarshal(raw, &response); err != nil { + return fmt.Errorf("decode %s range response for pod coverage: %w", name, err) + } + + pods := map[string]struct{}{} + + for _, series := range response.Data.Result { + pod := series.Metric["pod"] + if pod == "" { + return fmt.Errorf("%s range series has no pod label", name) + } + + if len(series.Values) > 0 { + pods[pod] = struct{}{} + } + } + + if len(pods) != expectedPods { + return fmt.Errorf("%s range capture has samples from %d/%d pods", name, len(pods), expectedPods) + } + + return nil +} + +func parseContainerdJournal( + raw string, + observerPodNodes map[string]string, + window telemetryWindow, +) ([]containerdJournalEvent, error) { + events := []containerdJournalEvent{} + observedPods := map[string]struct{}{} + + for _, line := range strings.Split(raw, "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + + prefixEnd := strings.Index(line, "] ") + if !strings.HasPrefix(line, "[pod/") || prefixEnd < 0 { + return nil, fmt.Errorf("parse containerd journal prefix: %q", line) + } + + prefixParts := strings.Split(strings.TrimSuffix(strings.TrimPrefix(line[:prefixEnd], "["), "]"), "/") + if len(prefixParts) != 3 || prefixParts[0] != "pod" || prefixParts[2] != "containerd-journal" { + return nil, fmt.Errorf("parse containerd journal source: %q", line[:prefixEnd+1]) + } + + observerPod := prefixParts[1] + + nodeName, ok := observerPodNodes[observerPod] + if !ok { + return nil, fmt.Errorf("containerd journal belongs to unexpected observer pod %q", observerPod) + } + + remainder := line[prefixEnd+2:] + + timestampEnd := strings.IndexByte(remainder, ' ') + if timestampEnd < 0 { + return nil, fmt.Errorf("parse containerd journal timestamp: %q", line) + } + + timestamp, err := time.Parse(time.RFC3339Nano, remainder[:timestampEnd]) + if err != nil { + return nil, fmt.Errorf("parse containerd journal timestamp %q: %w", remainder[:timestampEnd], err) + } + + if timestamp.Before(window.StartedAt) || timestamp.After(window.FinishedAt) { + continue + } + + message := remainder[timestampEnd+1:] + + eventType := classifyContainerdJournalEvent(message) + if eventType == "" { + return nil, fmt.Errorf("classify filtered containerd journal message: %q", message) + } + + event := containerdJournalEvent{ + ObserverPod: observerPod, + NodeName: nodeName, + Timestamp: timestamp, + Type: eventType, + Message: message, + } + for _, match := range journalFieldPattern.FindAllStringSubmatch(message, -1) { + switch match[1] { + case "duration": + duration, err := time.ParseDuration(match[2]) + if err != nil { + return nil, fmt.Errorf("parse containerd journal duration %q: %w", match[2], err) + } + + event.DurationSeconds = duration.Seconds() + case "layer": + event.LayerDigest = match[2] + } + } + + events = append(events, event) + observedPods[observerPod] = struct{}{} + } + + if len(observedPods) != len(observerPodNodes) { + return nil, fmt.Errorf( + "containerd journal has phase events from %d/%d observer pods", + len(observedPods), + len(observerPodNodes), + ) + } + + return events, nil +} + +func classifyContainerdJournalEvent(message string) string { + switch { + case strings.Contains(message, "layer unpacked"): + return "layer_unpacked" + case strings.Contains(message, "image unpacked"): + return "image_unpacked" + case strings.Contains(message, "cancel pulling image"): + return "pull_cancelled" + case strings.Contains(message, "Pulled image"): + return "pull_completed" + case strings.Contains(message, "PullImage "): + return "pull_started" + default: + return "" + } +} + +func (b *benchmark) queryPrometheusRange( + ctx context.Context, + query string, + window telemetryWindow, + step time.Duration, +) (json.RawMessage, error) { + rawPath := fmt.Sprintf( + "/api/v1/namespaces/%s/services/http:%s:9090/proxy/api/v1/query_range?query=%s&start=%s&end=%s&step=%d", + b.config.MonitoringNamespace, + b.config.PrometheusService, + url.QueryEscape(query), + url.QueryEscape(window.StartedAt.UTC().Format(time.RFC3339Nano)), + url.QueryEscape(window.FinishedAt.UTC().Format(time.RFC3339Nano)), + int(step.Seconds()), + ) + + output, err := b.commands.Run(ctx, nil, "kubectl", "get", "--raw", rawPath) + if err != nil { + return nil, err + } + + if err := validatePrometheusRangeResponseSize(output, maxPrometheusRangeResponseBytes); err != nil { + return nil, err + } + + var envelope struct { + Status string `json:"status"` + } + if err := json.Unmarshal(output, &envelope); err != nil { + return nil, fmt.Errorf("decode Prometheus range response: %w", err) + } + + if envelope.Status != "success" { + return nil, fmt.Errorf("prometheus range query status is %q", envelope.Status) + } + + return json.RawMessage(output), nil +} + +func validatePrometheusRangeResponseSize(output []byte, limit int) error { + if len(output) > limit { + return fmt.Errorf( + "prometheus range response is %d bytes, exceeds %d-byte capture limit", + len(output), + limit, + ) + } + + return nil +} + +func (b *benchmark) observerPodNodes(ctx context.Context) (map[string]string, error) { + output, err := b.commands.Run( + ctx, + nil, + "kubectl", "-n", b.config.Namespace, + "get", "pods", "-l", "app.kubernetes.io/name=gantry-benchmark-node-observer", + "-o", "json", + ) + if err != nil { + return nil, err + } + + var pods struct { + Items []struct { + Metadata struct { + Name string `json:"name"` + } `json:"metadata"` + Spec struct { + NodeName string `json:"nodeName"` + } `json:"spec"` + } `json:"items"` + } + if err := json.Unmarshal(output, &pods); err != nil { + return nil, fmt.Errorf("decode observer pods: %w", err) + } + + result := make(map[string]string, len(pods.Items)) + for _, pod := range pods.Items { + if pod.Metadata.Name == "" || pod.Spec.NodeName == "" { + return nil, fmt.Errorf("observer pod has empty name or nodeName") + } + + result[pod.Metadata.Name] = pod.Spec.NodeName + } + + if len(result) != b.config.NodeCount { + return nil, fmt.Errorf("observer pod/node map has %d pods, want %d", len(result), b.config.NodeCount) + } + + return result, nil +} + +func (b *benchmark) collectContainerdJournal(ctx context.Context, window telemetryWindow) (string, error) { + output, err := b.commands.Run( + ctx, + nil, + "kubectl", "-n", b.config.Namespace, + "logs", "-l", "app.kubernetes.io/name=gantry-benchmark-node-observer", + "-c", "containerd-journal", + "--prefix=true", + "--timestamps=true", + "--max-log-requests", fmt.Sprintf("%d", b.config.NodeCount), + "--since-time", window.StartedAt.UTC().Format(time.RFC3339Nano), + ) + if err != nil { + return "", fmt.Errorf("collect containerd journal: %w", err) + } + + return string(output), nil +} + +func (b *benchmark) writePerformanceTelemetryArtifact( + runID string, + phase proxyPhase, + measurement phasePerformanceTelemetry, +) error { + return b.writeJSONArtifact(runID, string(phase)+"-performance.json", measurement) +} diff --git a/hack/cmd/gantry-benchmark/performance_telemetry_test.go b/hack/cmd/gantry-benchmark/performance_telemetry_test.go new file mode 100644 index 000000000..28536c7d3 --- /dev/null +++ b/hack/cmd/gantry-benchmark/performance_telemetry_test.go @@ -0,0 +1,164 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package main + +import ( + "context" + "encoding/json" + "strings" + "testing" + "time" +) + +type performanceTelemetryRunner struct { + commands [][]string +} + +func (r *performanceTelemetryRunner) Run(_ context.Context, _ []byte, name string, args ...string) ([]byte, error) { + command := append([]string{name}, args...) + r.commands = append(r.commands, command) + + return []byte(`{"status":"success","data":{"resultType":"matrix","result":[]}}`), nil +} + +func TestQueryPrometheusRange(t *testing.T) { + runner := &performanceTelemetryRunner{} + benchmark := &benchmark{ + config: benchmarkConfig{ + MonitoringNamespace: "monitoring", + PrometheusService: "prometheus", + }, + commands: runner, + } + window := telemetryWindow{ + StartedAt: time.Date(2026, time.August, 4, 1, 2, 3, 0, time.UTC), + FinishedAt: time.Date(2026, time.August, 4, 1, 12, 3, 0, time.UTC), + } + + response, err := benchmark.queryPrometheusRange( + context.Background(), + `rate(node_disk_written_bytes_total[30s])`, + window, + 10*time.Second, + ) + if err != nil { + t.Fatalf("queryPrometheusRange: %v", err) + } + + if !strings.Contains(string(response), `"status":"success"`) { + t.Fatalf("response = %s, want successful raw envelope", response) + } + + if len(runner.commands) != 1 { + t.Fatalf("commands = %v, want one command", runner.commands) + } + + path := runner.commands[0][3] + for _, want := range []string{"/query_range?", "step=10", "node_disk_written_bytes_total", "2026-08-04T01%3A02%3A03Z"} { + if !strings.Contains(path, want) { + t.Fatalf("query path %q is missing %q", path, want) + } + } +} + +func TestPerformanceTelemetryQueriesBoundContainerdCardinality(t *testing.T) { + queries := performanceTelemetryQueries() + + byName := make(map[string]performanceTelemetryQuery, len(queries)) + for _, query := range queries { + byName[query.name] = query + if strings.Contains(query.query, `containerd_.*|grpc_server_.*`) { + t.Fatalf("query %q uses unbounded containerd and gRPC selector", query.name) + } + } + + for _, name := range []string{"containerd_image_pulls", "containerd_grpc_started", "containerd_grpc_handled"} { + if _, ok := byName[name]; !ok { + t.Fatalf("missing bounded telemetry query %q", name) + } + } + + if byName["containerd_image_pulls"].step != 0 || byName["containerd_grpc_started"].step != 0 { + t.Fatal("containerd pull and gRPC started queries must use the default 10-second step") + } + + if byName["containerd_grpc_handled"].step != 5*time.Minute { + t.Fatalf("gRPC handled step = %s, want 5m", byName["containerd_grpc_handled"].step) + } +} + +func TestValidatePrometheusRangeResponseSize(t *testing.T) { + if err := validatePrometheusRangeResponseSize([]byte("1234"), 4); err != nil { + t.Fatalf("response at limit: %v", err) + } + + if err := validatePrometheusRangeResponseSize([]byte("12345"), 4); err == nil || !strings.Contains(err.Error(), "5 bytes") { + t.Fatalf("oversized response error = %v", err) + } +} + +func TestParseContainerdJournal(t *testing.T) { + window := telemetryWindow{ + StartedAt: time.Date(2026, time.August, 4, 1, 2, 3, 0, time.UTC), + FinishedAt: time.Date(2026, time.August, 4, 1, 12, 3, 0, time.UTC), + } + raw := strings.Join([]string{ + `[pod/observer-a/containerd-journal] 2026-08-04T01:02:02Z level=debug msg="layer unpacked" duration=1s layer=sha256:before`, + `[pod/observer-a/containerd-journal] 2026-08-04T01:03:04.123456789Z level=debug msg="layer unpacked" duration=2.5s layer=sha256:abc`, + `[pod/observer-b/containerd-journal] 2026-08-04T01:04:05Z level=info msg="Pulled image" image="example/image@sha256:def"`, + `[pod/observer-a/containerd-journal] 2026-08-04T01:12:04Z level=debug msg="image unpacked" duration=3s`, + }, "\n") + + events, err := parseContainerdJournal(raw, map[string]string{ + "observer-a": "node-a", + "observer-b": "node-b", + }, window) + if err != nil { + t.Fatalf("parseContainerdJournal: %v", err) + } + + if len(events) != 2 { + t.Fatalf("events = %v, want two phase-bounded events", events) + } + + if events[0].NodeName != "node-a" || events[0].Type != "layer_unpacked" || + events[0].LayerDigest != "sha256:abc" || events[0].DurationSeconds != 2.5 { + t.Fatalf("first event = %+v, want parsed layer event", events[0]) + } + + if events[1].NodeName != "node-b" || events[1].Type != "pull_completed" { + t.Fatalf("second event = %+v, want correlated pull completion", events[1]) + } +} + +func TestParseContainerdJournalRequiresEveryObserver(t *testing.T) { + window := telemetryWindow{ + StartedAt: time.Date(2026, time.August, 4, 1, 2, 3, 0, time.UTC), + FinishedAt: time.Date(2026, time.August, 4, 1, 12, 3, 0, time.UTC), + } + raw := `[pod/observer-a/containerd-journal] 2026-08-04T01:03:04Z level=debug msg="image unpacked" duration=2s` + + _, err := parseContainerdJournal(raw, map[string]string{ + "observer-a": "node-a", + "observer-b": "node-b", + }, window) + if err == nil || !strings.Contains(err.Error(), "1/2 observer pods") { + t.Fatalf("error = %v, want incomplete observer coverage", err) + } +} + +func TestValidatePrometheusRangePodCoverage(t *testing.T) { + raw := json.RawMessage(`{"data":{"result":[ + {"metric":{"pod":"observer-a"},"values":[[1,"2"]]}, + {"metric":{"pod":"observer-b"},"values":[[1,"3"]]} + ]}}`) + + if err := validatePrometheusRangePodCoverage("disk", raw, 2); err != nil { + t.Fatalf("validatePrometheusRangePodCoverage: %v", err) + } + + if err := validatePrometheusRangePodCoverage("disk", raw, 3); err == nil || !strings.Contains(err.Error(), "2/3 pods") { + t.Fatalf("error = %v, want partial pod coverage", err) + } +} diff --git a/hack/cmd/gantry-benchmark/preflight.go b/hack/cmd/gantry-benchmark/preflight.go index e87f7ff16..a6ce5f1f9 100644 --- a/hack/cmd/gantry-benchmark/preflight.go +++ b/hack/cmd/gantry-benchmark/preflight.go @@ -368,6 +368,86 @@ func (b *benchmark) checkMonitoring(ctx context.Context, state benchmarkState) e } } + monitoringChecks := []struct { + description string + query string + }{ + { + description: "node-exporter identity", + query: fmt.Sprintf(`count(node_uname_info{namespace=%q,gantry_benchmark="true"})`, b.config.Namespace), + }, + { + description: "host disk throughput", + query: fmt.Sprintf(`count(count by(pod) (node_disk_written_bytes_total{namespace=%q,gantry_benchmark="true"}))`, b.config.Namespace), + }, + { + description: "host network throughput", + query: fmt.Sprintf(`count(count by(pod) (node_network_receive_bytes_total{namespace=%q,gantry_benchmark="true",device!="lo"}))`, b.config.Namespace), + }, + { + description: "host network link speed", + query: fmt.Sprintf(`count(count by(pod) (node_network_speed_bytes{namespace=%q,gantry_benchmark="true",device!="lo"} > 0))`, b.config.Namespace), + }, + { + description: "containerd build", + query: fmt.Sprintf(`count(containerd_build_info_total{namespace=%q,gantry_benchmark="true"})`, b.config.Namespace), + }, + { + description: "Gantry response completion", + query: fmt.Sprintf( + `count(count by(pod) (gantry_mirror_response_completed_timestamp_seconds{namespace=%q,kind="layer",gantry_benchmark="true",controller_revision_hash=%q}))`, + b.config.GantryNamespace, + revision, + ), + }, + { + description: "Gantry peer busy and stall outcomes", + query: fmt.Sprintf( + `count(count by(pod) (p2p_peer_fetch_total{namespace=%q,outcome=~"busy|stall",gantry_benchmark="true",controller_revision_hash=%q}))`, + b.config.GantryNamespace, + revision, + ), + }, + { + description: "Gantry peer busy and stall timestamps", + query: fmt.Sprintf( + `count(count by(pod) (gantry_peer_fetch_last_timestamp_seconds{namespace=%q,outcome=~"busy|stall",gantry_benchmark="true",controller_revision_hash=%q}))`, + b.config.GantryNamespace, + revision, + ), + }, + { + description: "Gantry DHT lookup durations", + query: fmt.Sprintf( + `count(count by(pod) (p2p_dht_lookup_duration_seconds_count{namespace=%q,gantry_benchmark="true",controller_revision_hash=%q}))`, + b.config.GantryNamespace, + revision, + ), + }, + { + description: "Gantry containerd commit observation", + query: fmt.Sprintf( + `count(gantry_containerd_commit_observation_duration_seconds_count{namespace=%q,gantry_benchmark="true",controller_revision_hash=%q})`, + b.config.GantryNamespace, + revision, + ), + }, + { + description: "Gantry latest containerd commit observation duration", + query: fmt.Sprintf( + `count(gantry_containerd_commit_latest_observation_duration_seconds{namespace=%q,gantry_benchmark="true",controller_revision_hash=%q})`, + b.config.GantryNamespace, + revision, + ), + }, + } + + for _, check := range monitoringChecks { + if err := b.waitForPrometheusMetricCoverage(ctx, check.description, check.query); err != nil { + return err + } + } + // Direct mode has no proxy, so there are no proxy samples to wait for. The // Gantry scrape checks above already prove the benchmark PodMonitor is // being honoured by Prometheus. @@ -400,6 +480,65 @@ func (b *benchmark) checkMonitoring(ctx context.Context, state benchmarkState) e } } +func (b *benchmark) waitForPrometheusMetricCoverage(ctx context.Context, description, query string) error { + pollContext, cancel := context.WithTimeout(ctx, b.config.TelemetryTimeout) + defer cancel() + + var ( + count float64 + queryErr error + ) + + for { + count, queryErr = b.queryPrometheus(pollContext, query) + if queryErr == nil && int(count) == b.config.NodeCount { + return nil + } + + if queryErr == nil { + writeAll(b.stdout, fmt.Sprintf( + "waiting for Prometheus %s coverage: %.0f/%d observer pods\n", + description, + count, + b.config.NodeCount, + )) + } + + timer := time.NewTimer(b.config.TelemetryPollInterval) + select { + case <-pollContext.Done(): + if !timer.Stop() { + select { + case <-timer.C: + default: + } + } + + if ctx.Err() != nil { + return ctx.Err() + } + + if queryErr != nil { + return fmt.Errorf( + "prometheus %s metrics were not queryable before %s: %w", + description, + b.config.TelemetryTimeout, + queryErr, + ) + } + + return fmt.Errorf( + "prometheus reports %s metrics for %.0f/%d observer pods after waiting %s", + description, + count, + b.config.NodeCount, + b.config.TelemetryTimeout, + ) + case <-timer.C: + } + } +} + func (b *benchmark) checkAzureTelemetry(ctx context.Context) error { if _, err := b.commands.Run(ctx, nil, "az", "account", "show", "--output", "none"); err != nil { return fmt.Errorf("validate Azure CLI authentication: %w", err) @@ -465,6 +604,33 @@ type azureDiagnosticSetting struct { } `json:"logs"` } +func decodeAzureDiagnosticSettings(raw []byte) ([]azureDiagnosticSetting, error) { + if strings.HasPrefix(strings.TrimSpace(string(raw)), "[") { + var settings []azureDiagnosticSetting + if err := json.Unmarshal(raw, &settings); err != nil { + return nil, err + } + + return settings, nil + } + + var envelope struct { + Value []struct { + Properties azureDiagnosticSetting `json:"properties"` + } `json:"value"` + } + if err := json.Unmarshal(raw, &envelope); err != nil { + return nil, err + } + + settings := make([]azureDiagnosticSetting, 0, len(envelope.Value)) + for _, entry := range envelope.Value { + settings = append(settings, entry.Properties) + } + + return settings, nil +} + func (b *benchmark) checkAKSAuditDiagnosticSetting(ctx context.Context) error { output, err := b.commands.Run( ctx, @@ -477,18 +643,12 @@ func (b *benchmark) checkAKSAuditDiagnosticSetting(ctx context.Context) error { return fmt.Errorf("read AKS diagnostic settings: %w", err) } - var settings struct { - Value []struct { - Properties azureDiagnosticSetting `json:"properties"` - } `json:"value"` - } - if err := json.Unmarshal(output, &settings); err != nil { + settings, err := decodeAzureDiagnosticSettings(output) + if err != nil { return fmt.Errorf("decode AKS diagnostic settings: %w", err) } - for _, entry := range settings.Value { - setting := entry.Properties - + for _, setting := range settings { if !strings.EqualFold(setting.LogAnalyticsDestinationType, "Dedicated") || setting.WorkspaceID == "" { continue } diff --git a/hack/cmd/gantry-benchmark/preflight_monitoring_test.go b/hack/cmd/gantry-benchmark/preflight_monitoring_test.go new file mode 100644 index 000000000..6ea226c26 --- /dev/null +++ b/hack/cmd/gantry-benchmark/preflight_monitoring_test.go @@ -0,0 +1,116 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package main + +import ( + "bytes" + "context" + "errors" + "fmt" + "strings" + "testing" + "time" +) + +type monitoringCoverageResult struct { + count float64 + err error +} + +type monitoringCoverageRunner struct { + results []monitoringCoverageResult + calls int +} + +func (r *monitoringCoverageRunner) Run(_ context.Context, _ []byte, _ string, _ ...string) ([]byte, error) { + index := r.calls + + r.calls++ + if index >= len(r.results) { + index = len(r.results) - 1 + } + + result := r.results[index] + if result.err != nil { + return nil, result.err + } + + return []byte(fmt.Sprintf( + `{"status":"success","data":{"result":[{"value":[0,%q]}]}}`, + fmt.Sprintf("%.0f", result.count), + )), nil +} + +func TestWaitForPrometheusMetricCoverageRetriesPartialScrape(t *testing.T) { + runner := &monitoringCoverageRunner{results: []monitoringCoverageResult{{count: 9}, {count: 1000}}} + + var stdout bytes.Buffer + + bench := benchmark{ + config: benchmarkConfig{ + MonitoringNamespace: "monitoring", + PrometheusService: "prometheus", + NodeCount: 1000, + TelemetryTimeout: time.Second, + TelemetryPollInterval: time.Millisecond, + }, + commands: runner, + stdout: &stdout, + } + + if err := bench.waitForPrometheusMetricCoverage(context.Background(), "containerd build", "count(metric)"); err != nil { + t.Fatal(err) + } + + if runner.calls != 2 { + t.Fatalf("query calls = %d, want 2", runner.calls) + } + + if !strings.Contains(stdout.String(), "9/1000 observer pods") { + t.Fatalf("progress output = %q", stdout.String()) + } +} + +func TestWaitForPrometheusMetricCoverageRetriesQueryError(t *testing.T) { + runner := &monitoringCoverageRunner{results: []monitoringCoverageResult{{err: errors.New("not ready")}, {count: 1000}}} + bench := benchmark{ + config: benchmarkConfig{ + MonitoringNamespace: "monitoring", + PrometheusService: "prometheus", + NodeCount: 1000, + TelemetryTimeout: time.Second, + TelemetryPollInterval: time.Millisecond, + }, + commands: runner, + stdout: &bytes.Buffer{}, + } + + if err := bench.waitForPrometheusMetricCoverage(context.Background(), "containerd build", "count(metric)"); err != nil { + t.Fatal(err) + } + + if runner.calls != 2 { + t.Fatalf("query calls = %d, want 2", runner.calls) + } +} + +func TestWaitForPrometheusMetricCoverageTimesOut(t *testing.T) { + runner := &monitoringCoverageRunner{results: []monitoringCoverageResult{{count: 9}}} + bench := benchmark{ + config: benchmarkConfig{ + MonitoringNamespace: "monitoring", + PrometheusService: "prometheus", + NodeCount: 1000, + TelemetryTimeout: time.Nanosecond, + TelemetryPollInterval: time.Hour, + }, + commands: runner, + stdout: &bytes.Buffer{}, + } + + err := bench.waitForPrometheusMetricCoverage(context.Background(), "containerd build", "count(metric)") + if err == nil || !strings.Contains(err.Error(), "9/1000 observer pods after waiting 1ns") { + t.Fatalf("timeout error = %v", err) + } +} diff --git a/hack/cmd/gantry-benchmark/results.go b/hack/cmd/gantry-benchmark/results.go index 466962be3..26771ff4f 100644 --- a/hack/cmd/gantry-benchmark/results.go +++ b/hack/cmd/gantry-benchmark/results.go @@ -72,24 +72,26 @@ const ( ) type phaseResult struct { - RunID string `json:"run_id"` - Phase proxyPhase `json:"phase"` - Image string `json:"image"` - ImageSizeMiB int `json:"image_size_mib"` - ImageLayers int `json:"image_layers,omitempty"` - PayloadSHA string `json:"workload_payload_sha256,omitempty"` - WorkloadComparisonMode workloadComparisonMode `json:"workload_comparison_mode,omitempty"` - Proxy proxyPhaseTotals `json:"proxy"` - Gantry gantryMetrics `json:"gantry"` - GantryPeer gantryPeerPhaseMeasurement `json:"gantry_peer"` - Azure azurePhaseMeasurement `json:"azure"` - Job jobObservation `json:"job"` + RunID string `json:"run_id"` + Phase proxyPhase `json:"phase"` + Image string `json:"image"` + ImageSizeMiB int `json:"image_size_mib"` + ImageLayers int `json:"image_layers,omitempty"` + PayloadSHA string `json:"workload_payload_sha256,omitempty"` + WorkloadComparisonMode workloadComparisonMode `json:"workload_comparison_mode,omitempty"` + Proxy proxyPhaseTotals `json:"proxy"` + Gantry gantryMetrics `json:"gantry"` + GantryPeer gantryPeerPhaseMeasurement `json:"gantry_peer"` + GantryDiagnostics gantryDiagnosticPhaseMeasurement `json:"gantry_diagnostics"` + Azure azurePhaseMeasurement `json:"azure"` + Job jobObservation `json:"job"` // OriginBytes is the phase's ACR traffic as attributed by OriginBytesSource. - OriginBytes uint64 `json:"origin_bytes"` - OriginBytesSource originByteSource `json:"origin_bytes_source"` - PodStartupLatency latencySummary `json:"pod_startup_latency"` - PodStartupLatencySource string `json:"pod_startup_latency_source"` - RecordedAt time.Time `json:"recorded_at"` + OriginBytes uint64 `json:"origin_bytes"` + OriginBytesSource originByteSource `json:"origin_bytes_source"` + PodStartupLatency latencySummary `json:"pod_startup_latency"` + PodStartupLatencySource string `json:"pod_startup_latency_source"` + PerformanceTelemetryArtifact string `json:"performance_telemetry_artifact"` + RecordedAt time.Time `json:"recorded_at"` } type benchmarkComparison struct { diff --git a/hack/cmd/gantry-benchmark/run.go b/hack/cmd/gantry-benchmark/run.go index a6a195fe2..24d4152a3 100644 --- a/hack/cmd/gantry-benchmark/run.go +++ b/hack/cmd/gantry-benchmark/run.go @@ -130,6 +130,11 @@ func (b *benchmark) runBenchmark(ctx context.Context) (returnErr error) { return err } + baselineDiagnosticsBefore, err := b.fetchGantryDiagnosticSnapshot(ctx, revision) + if err != nil { + return err + } + writeAll(b.stdout, fmt.Sprintf("running baseline pull on %d nodes\n", b.config.NodeCount)) baselineJob, err := b.runPullJob(ctx, state, proxyPhaseBaseline, baselineImage) @@ -170,8 +175,39 @@ func (b *benchmark) runBenchmark(ctx context.Context) (returnErr error) { return err } + baselineDiagnosticsAfter, err := b.fetchGantryDiagnosticSnapshot(ctx, revision) + if err != nil { + return err + } + + baselineDiagnosticTimestamps, err := b.fetchGantryDiagnosticTimestamps(ctx, revision, telemetryWindow{ + StartedAt: baselineJob.PhaseStartedAt, + FinishedAt: baselineJob.PhaseFinishedAt, + }) + if err != nil { + return err + } + + baselineDiagnostics, err := subtractGantryDiagnosticSnapshots( + baselineDiagnosticsBefore, + baselineDiagnosticsAfter, + baselineDiagnosticTimestamps, + ) + if err != nil { + return err + } + baselineBytes, baselineBytesSource := deriveOriginBytes(b.config, proxyPhaseBaseline, baselineProxy, baselineGantry, baselineJob) + baselinePerformance, err := b.capturePhasePerformanceTelemetry(ctx, proxyPhaseBaseline, baselineJob) + if err != nil { + return err + } + + if err := b.writePerformanceTelemetryArtifact(state.RunID, proxyPhaseBaseline, baselinePerformance); err != nil { + return err + } + baselineResult := phaseResult{ RunID: state.RunID, Phase: proxyPhaseBaseline, @@ -183,14 +219,16 @@ func (b *benchmark) runBenchmark(ctx context.Context) (returnErr error) { Proxy: baselineProxy, Gantry: baselineGantry, GantryPeer: baselinePeer, + GantryDiagnostics: baselineDiagnostics, Azure: azurePhaseMeasurement{Window: telemetryWindow{ StartedAt: baselineWindowStart, FinishedAt: baselineWindowFinish, }}, - Job: baselineJob, - OriginBytes: baselineBytes, - OriginBytesSource: baselineBytesSource, - RecordedAt: time.Now().UTC(), + Job: baselineJob, + OriginBytes: baselineBytes, + OriginBytesSource: baselineBytesSource, + PerformanceTelemetryArtifact: string(proxyPhaseBaseline) + "-performance.json", + RecordedAt: time.Now().UTC(), } if err := b.writeJSONArtifact(state.RunID, "baseline.json", baselineResult); err != nil { return err @@ -227,6 +265,11 @@ func (b *benchmark) runBenchmark(ctx context.Context) (returnErr error) { return err } + gantryDiagnosticsBefore, err := b.fetchGantryDiagnosticSnapshot(ctx, revision) + if err != nil { + return err + } + writeAll(b.stdout, fmt.Sprintf("running Gantry cold pull on %d nodes\n", b.config.NodeCount)) gantryJob, err := b.runPullJob(ctx, state, proxyPhaseGantryCold, gantryImage) @@ -258,6 +301,32 @@ func (b *benchmark) runBenchmark(ctx context.Context) (returnErr error) { return err } + gantryDiagnosticsAfter, err := b.fetchGantryDiagnosticSnapshot(ctx, revision) + if err != nil { + return err + } + + gantryDiagnosticTimestamps, err := b.fetchGantryDiagnosticTimestamps(ctx, revision, telemetryWindow{ + StartedAt: gantryJob.PhaseStartedAt, + FinishedAt: gantryJob.PhaseFinishedAt, + }) + if err != nil { + return err + } + + if err := requireFinalLayerResponseTimestamps(gantryDiagnosticTimestamps, gantryDiagnosticsAfter.PodNodes); err != nil { + return err + } + + gantryDiagnostics, err := subtractGantryDiagnosticSnapshots( + gantryDiagnosticsBefore, + gantryDiagnosticsAfter, + gantryDiagnosticTimestamps, + ) + if err != nil { + return err + } + var gantryProxy proxyPhaseTotals if state.usesProxy() { @@ -269,6 +338,15 @@ func (b *benchmark) runBenchmark(ctx context.Context) (returnErr error) { gantryBytes, gantryBytesSource := deriveOriginBytes(b.config, proxyPhaseGantryCold, gantryProxy, phaseMetrics, gantryJob) + gantryPerformance, err := b.capturePhasePerformanceTelemetry(ctx, proxyPhaseGantryCold, gantryJob) + if err != nil { + return err + } + + if err := b.writePerformanceTelemetryArtifact(state.RunID, proxyPhaseGantryCold, gantryPerformance); err != nil { + return err + } + gantryResult := phaseResult{ RunID: state.RunID, Phase: proxyPhaseGantryCold, @@ -280,14 +358,16 @@ func (b *benchmark) runBenchmark(ctx context.Context) (returnErr error) { Proxy: gantryProxy, Gantry: phaseMetrics, GantryPeer: gantryPeer, + GantryDiagnostics: gantryDiagnostics, Azure: azurePhaseMeasurement{Window: telemetryWindow{ StartedAt: gantryWindowStart, FinishedAt: gantryWindowFinish, }}, - Job: gantryJob, - OriginBytes: gantryBytes, - OriginBytesSource: gantryBytesSource, - RecordedAt: time.Now().UTC(), + Job: gantryJob, + OriginBytes: gantryBytes, + OriginBytesSource: gantryBytesSource, + PerformanceTelemetryArtifact: string(proxyPhaseGantryCold) + "-performance.json", + RecordedAt: time.Now().UTC(), } if err := b.writeJSONArtifact(state.RunID, "gantry-cold.json", gantryResult); err != nil { return err diff --git a/hack/gantry-benchmark/.gitignore b/hack/gantry-benchmark/.gitignore index 464fb68b3..d104893c3 100644 --- a/hack/gantry-benchmark/.gitignore +++ b/hack/gantry-benchmark/.gitignore @@ -1 +1,2 @@ -env.local \ No newline at end of file +env.local +deploy.env \ No newline at end of file diff --git a/hack/gantry-benchmark/Makefile b/hack/gantry-benchmark/Makefile index f043cf8b1..aa2c6ea2d 100644 --- a/hack/gantry-benchmark/Makefile +++ b/hack/gantry-benchmark/Makefile @@ -1,24 +1,38 @@ REPO_ROOT := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))/../..) ENV_FILE ?= $(CURDIR)/env.local +DEPLOY_CONFIG ?= $(CURDIR)/deploy.env -.PHONY: help test operator-vm-check operator-vm-provision operator-vm-status operator-vm-watch proxy-image proxy-push enable prepare prepare-gantry prepare-gantry-fresh prepare-gantry-adopt preflight run run-gantry status disable +.PHONY: help test monitor profile-gantry deploy deploy-plan deploy-status operator-vm-check operator-vm-provision operator-vm-status operator-vm-watch operator-vm-prebuild operator-vm-run-pool operator-vm-run-fresh operator-vm-image-pool-status proxy-image proxy-push image-pool-status prebuild-gantry enable prepare prepare-adopt prepare-gantry prepare-gantry-fresh prepare-gantry-adopt prepare-gantry-pool preflight run run-gantry status disable help: ## Show benchmark targets @echo "" @echo "Usage: make -C hack/gantry-benchmark [ENV_FILE=path]" @echo "" @echo " test Run focused proxy and benchmark tests" + @echo " monitor Live peer-outcome and layer-byte tables for the active Gantry phase" + @echo " profile-gantry CPU-profile the hottest Gantry pods in an active diagnostic run" + @echo " deploy Idempotently deploy the complete benchmark stack" + @echo " deploy-plan Print the resolved deployment contract without mutation" + @echo " deploy-status Report deployment readiness without mutation" @echo " operator-vm-check Validate operator VM scripts" @echo " operator-vm-provision Provision/bootstrap the private operator VM" @echo " operator-vm-status Print one live operator VM progress snapshot" @echo " operator-vm-watch Follow operator VM progress until completion" + @echo " operator-vm-prebuild Start an asynchronous operator-VM pool build" + @echo " operator-vm-run-pool Start Gantry-only using the next compatible pool image" + @echo " operator-vm-run-fresh Build a brand-new image, then run Gantry-only" + @echo " operator-vm-image-pool-status Show operator-VM pool builder and entries" @echo " proxy-image Build BENCHMARK_PROXY_IMAGE" @echo " proxy-push Push BENCHMARK_PROXY_IMAGE" + @echo " image-pool-status List ready and claimed prebuilt Gantry images" + @echo " prebuild-gantry Build GANTRY_IMAGE_POOL_COUNT reusable Gantry images" @echo " enable Install benchmark instrumentation" @echo " prepare Build and push both digest-pinned workload images" + @echo " prepare-adopt Adopt already-pushed direct-mode workload images" @echo " prepare-gantry Build only a cache-cold Gantry image from a retained baseline" @echo " prepare-gantry-fresh Build only Gantry with a brand-new random payload" @echo " prepare-gantry-adopt Adopt an already-pushed fresh Gantry image digest" + @echo " prepare-gantry-pool Claim one compatible prebuilt Gantry image" @echo " preflight Validate ACR, monitoring, Gantry, and all target nodes" @echo " run Execute baseline and Gantry cold phases" @echo " run-gantry Execute only Gantry cold against the retained baseline" @@ -27,10 +41,39 @@ help: ## Show benchmark targets test: $(MAKE) operator-vm-check - cd "$(REPO_ROOT)" && GOTOOLCHAIN=auto go test ./hack/cmd/acr-origin-proxy ./hack/cmd/gantry-benchmark + cd "$(REPO_ROOT)" && GOTOOLCHAIN=auto go test ./hack/cmd/acr-origin-proxy ./hack/cmd/gantry-benchmark ./hack/cmd/gantry-benchmark-monitor + +monitor: + set -a; \ + . "$(DEPLOY_CONFIG)"; \ + set +a; \ + export KUBECONFIG="$${DEPLOY_KUBECONFIG:-$(REPO_ROOT)/tmp/$${DEPLOYMENT_NAME}/kubeconfig}"; \ + cd "$(REPO_ROOT)" && GOTOOLCHAIN=auto go run ./hack/cmd/gantry-benchmark-monitor $(MONITOR_ARGS) + +profile-gantry: operator-vm-check + set -a; \ + . "$(DEPLOY_CONFIG)"; \ + set +a; \ + export KUBECONFIG="$${DEPLOY_KUBECONFIG:-$(REPO_ROOT)/tmp/$${DEPLOYMENT_NAME}/kubeconfig}"; \ + cd "$(REPO_ROOT)" && GANTRY_PPROF_SECONDS="$${GANTRY_PPROF_SECONDS:-30}" \ + GANTRY_PPROF_COUNT="$${GANTRY_PPROF_COUNT:-3}" \ + hack/gantry-benchmark/profile-gantry.sh operator-vm-check: - bash -n operator-vm-provision.sh operator-vm-bootstrap.sh operator-vm-run.sh operator-vm-status.sh operator-vm-watch.sh + bash -n deploy.sh profile-gantry.sh operator-vm-provision.sh operator-vm-bootstrap.sh operator-vm-build-images.sh operator-vm-run.sh operator-vm-prebuild-images.sh operator-vm-image-pool.sh operator-vm-status.sh operator-vm-watch.sh + ./deploy.sh plan deploy.env.example >/dev/null + ! grep -Eq 'az acr login|podman (build|push|login|pull|tag)' deploy.sh + ! grep -Eq '^[[:space:]]*az login([[:space:]]|$$)' deploy.sh + ! grep -Eq '^[[:space:]]*export START_BENCHMARK=false' deploy.sh + +deploy: operator-vm-check + cd "$(REPO_ROOT)" && hack/gantry-benchmark/deploy.sh deploy "$(DEPLOY_CONFIG)" + +deploy-plan: operator-vm-check + cd "$(REPO_ROOT)" && hack/gantry-benchmark/deploy.sh plan "$(DEPLOY_CONFIG)" + +deploy-status: operator-vm-check + cd "$(REPO_ROOT)" && hack/gantry-benchmark/deploy.sh status "$(DEPLOY_CONFIG)" operator-vm-provision: operator-vm-check cd "$(REPO_ROOT)" && hack/gantry-benchmark/operator-vm-provision.sh @@ -41,6 +84,21 @@ operator-vm-status: operator-vm-check operator-vm-watch: operator-vm-check cd "$(REPO_ROOT)" && hack/gantry-benchmark/operator-vm-watch.sh --follow +operator-vm-prebuild: operator-vm-check + @test -n "$(GANTRY_IMAGE_POOL_COUNT)" || { echo "GANTRY_IMAGE_POOL_COUNT is required" >&2; exit 2; } + cd "$(REPO_ROOT)" && hack/gantry-benchmark/operator-vm-image-pool.sh start "$(GANTRY_IMAGE_POOL_COUNT)" + +operator-vm-run-pool: operator-vm-check + @test -n "$(GANTRY_ONLY_BASELINE_RUN_ID)" || { echo "GANTRY_ONLY_BASELINE_RUN_ID is required" >&2; exit 2; } + cd "$(REPO_ROOT)" && hack/gantry-benchmark/operator-vm-image-pool.sh run "$(GANTRY_ONLY_BASELINE_RUN_ID)" + +operator-vm-run-fresh: operator-vm-check + @test -n "$(GANTRY_ONLY_BASELINE_RUN_ID)" || { echo "GANTRY_ONLY_BASELINE_RUN_ID is required" >&2; exit 2; } + cd "$(REPO_ROOT)" && hack/gantry-benchmark/operator-vm-image-pool.sh fresh "$(GANTRY_ONLY_BASELINE_RUN_ID)" + +operator-vm-image-pool-status: operator-vm-check + cd "$(REPO_ROOT)" && hack/gantry-benchmark/operator-vm-image-pool.sh status + proxy-image: set -a; \ [ ! -f "$(ENV_FILE)" ] || . "$(ENV_FILE)"; \ @@ -65,12 +123,35 @@ proxy-push: --username "$${ACR_USERNAME}" --password-stdin "$${ACR_LOGIN_SERVER}"; \ "$${CONTAINER_ENGINE:-podman}" push "$${BENCHMARK_PROXY_IMAGE}" +image-pool-status: + set -a; \ + [ ! -f "$(ENV_FILE)" ] || . "$(ENV_FILE)"; \ + set +a; \ + cd "$(REPO_ROOT)" && GOTOOLCHAIN=auto go run ./hack/cmd/gantry-benchmark image-pool-status + +prebuild-gantry: + @test -n "$(GANTRY_IMAGE_POOL_COUNT)" || { echo "GANTRY_IMAGE_POOL_COUNT is required" >&2; exit 2; } + set -a; \ + [ ! -f "$(ENV_FILE)" ] || . "$(ENV_FILE)"; \ + set +a; \ + cd "$(REPO_ROOT)" && GOTOOLCHAIN=auto go run ./hack/cmd/gantry-benchmark prebuild-gantry "$(GANTRY_IMAGE_POOL_COUNT)" + enable prepare preflight run run-gantry status disable: set -a; \ [ ! -f "$(ENV_FILE)" ] || . "$(ENV_FILE)"; \ set +a; \ cd "$(REPO_ROOT)" && GOTOOLCHAIN=auto go run ./hack/cmd/gantry-benchmark $@ +prepare-adopt: + @test -n "$(ADOPT_BASELINE_IMAGE)" || { echo "ADOPT_BASELINE_IMAGE is required" >&2; exit 2; } + @test -n "$(ADOPT_GANTRY_IMAGE)" || { echo "ADOPT_GANTRY_IMAGE is required" >&2; exit 2; } + @test -n "$(ADOPT_PAYLOAD_SHA256)" || { echo "ADOPT_PAYLOAD_SHA256 is required" >&2; exit 2; } + set -a; \ + [ ! -f "$(ENV_FILE)" ] || . "$(ENV_FILE)"; \ + set +a; \ + cd "$(REPO_ROOT)" && GOTOOLCHAIN=auto go run ./hack/cmd/gantry-benchmark prepare-adopt \ + "$(ADOPT_BASELINE_IMAGE)" "$(ADOPT_GANTRY_IMAGE)" "$(ADOPT_PAYLOAD_SHA256)" + prepare-gantry: @test -n "$(GANTRY_ONLY_BASELINE_RUN_ID)" || { echo "GANTRY_ONLY_BASELINE_RUN_ID is required" >&2; exit 2; } set -a; \ @@ -95,4 +176,12 @@ prepare-gantry-adopt: [ ! -f "$(ENV_FILE)" ] || . "$(ENV_FILE)"; \ set +a; \ cd "$(REPO_ROOT)" && GOTOOLCHAIN=auto go run ./hack/cmd/gantry-benchmark prepare-gantry-adopt \ - "$(GANTRY_ONLY_BASELINE_RUN_ID)" "$(GANTRY_ONLY_ADOPT_IMAGE)" "$(GANTRY_ONLY_ADOPT_PAYLOAD_SHA256)" \ No newline at end of file + "$(GANTRY_ONLY_BASELINE_RUN_ID)" "$(GANTRY_ONLY_ADOPT_IMAGE)" "$(GANTRY_ONLY_ADOPT_PAYLOAD_SHA256)" + +prepare-gantry-pool: + @test -n "$(GANTRY_ONLY_BASELINE_RUN_ID)" || { echo "GANTRY_ONLY_BASELINE_RUN_ID is required" >&2; exit 2; } + set -a; \ + [ ! -f "$(ENV_FILE)" ] || . "$(ENV_FILE)"; \ + set +a; \ + cd "$(REPO_ROOT)" && GOTOOLCHAIN=auto go run ./hack/cmd/gantry-benchmark prepare-gantry-pool \ + "$(GANTRY_ONLY_BASELINE_RUN_ID)" diff --git a/hack/gantry-benchmark/PULL-LATENCY-ANALYSIS.md b/hack/gantry-benchmark/PULL-LATENCY-ANALYSIS.md new file mode 100644 index 000000000..9d4c7e35c --- /dev/null +++ b/hack/gantry-benchmark/PULL-LATENCY-ANALYSIS.md @@ -0,0 +1,304 @@ +# Pod Startup Latency Analysis + +Where the ~750-1100 seconds of per-pod startup time actually goes on a +1000-node AKS cluster pulling a 40 GiB, 40-layer image, and which containerd +settings move it. + +Source runs, both Canada Central, `Standard_D8s_v3`, Kubernetes 1.35, Azure CNI +Overlay, containerd 2.x, images pulled over ACR Private Endpoints: + +| Run | containerd `max_concurrent_downloads` | +| --- | --- | +| `run-20260806-185139-9995e167` | 3 (containerd default) | +| `run-20260806-205719-51c38730` | 6 (benchmark drop-in) | + +`max_concurrent_unpacks` was 1 (the containerd default) in both. + +Measurements come from AKS audit timestamps, containerd debug journals captured +by the node-observer DaemonSet on all 1000 nodes, and Prometheus node metrics. +Percentiles are nearest-rank, matching the benchmark runner. + +## Scheduling is not the story + +| Component | Baseline | Gantry cold | +| --- | ---: | ---: | +| Scheduling, create to bind (P50) | 2.162s | 1.836s | +| Post-bind, bind to container started (P50) | 757.726s | 921.710s | +| Total pod startup (P50) | 759.746s | 923.473s | + +Scheduling is 0.3% of startup. Effectively all wall time is the image pull, so +the rest of this document decomposes post-bind. + +## Post-bind splits into byte-waiting and unpacking + +containerd logs one `layer unpacked` event per layer and one `image unpacked` +event per image, both with durations. The image event covers the whole pull, so +the difference between it and the sum of its layer events is time containerd +spent waiting for bytes rather than unpacking them. + +Per-layer unpack cost is stable across every phase and both runs: mean 8.331s +to 8.404s over 14,000 samples per phase. Forty layers therefore cost roughly +333s of unpack work per node, and that figure does not depend on where the bytes +came from. + +| Run / phase | Image unpack (P50) | Unpack work | Waiting for bytes | +| --- | ---: | ---: | ---: | +| downloads=3, baseline | 931.1s | 333.5s (35.8%) | 597.6s (64.2%) | +| downloads=3, Gantry cold | 1098.0s | 336.2s (30.6%) | 761.8s (69.4%) | +| downloads=6, baseline | 753.3s | 333.2s (44.2%) | 420.0s (55.8%) | +| downloads=6, Gantry cold | 917.3s | 334.0s (36.4%) | 583.3s (63.6%) | + +## Raising download concurrency to 6 is a 15-20% win + +| Metric | downloads=3 | downloads=6 | Change | +| --- | ---: | ---: | ---: | +| Baseline P50 | 939.368s | 759.746s | 19.1% faster | +| Baseline P95 | 1091.173s | 867.850s | 20.5% faster | +| Gantry P50 | 1105.766s | 923.473s | 16.5% faster | +| Gantry P95 | 1180.970s | 994.468s | 15.8% faster | + +The gain comes entirely out of byte-waiting. Baseline waiting fell from 597.6s +to 420.0s while unpack work stayed at 333s. + +## Unpacking is serialized, and that floor is now dominant + +containerd's transfer plugin defaults to `max_concurrent_unpacks = 1` +(`plugins/transfer/plugin.go`), and `core/transfer/local/transfer.go` only +builds the unpack semaphore when the value exceeds 1. Without that semaphore, +`Unpacker.supportParallel` returns false at its first branch and every layer is +unpacked sequentially. + +Both runs confirm this empirically: all 2000 `image unpacked` events in each run +carry `parallel=false`, and the journals contain no "snapshotter does not +support rebase capability" message. The rebase check was never reached, so the +only thing disabling parallel unpack was the concurrency setting. + +This matters because overlayfs does advertise the `rebase` capability required +for parallel unpack whenever containerd is not running inside a user namespace +(`plugins/snapshots/overlay/plugin/plugin.go`), which is the case on AKS. The +capability is available and unused. + +As delivery gets faster the serialized floor becomes proportionally larger. On +baseline it moved from 35.8% of the pull at 3 concurrent downloads to 44.2% at +6. Raising download concurrency further without also raising unpack concurrency +has diminishing returns. + +## Nothing is resource-saturated except Gantry's CPU + +| Resource | downloads=3 baseline | downloads=6 baseline | downloads=3 Gantry | downloads=6 Gantry | +| --- | ---: | ---: | ---: | ---: | +| NIC utilization (P95) | 1.26% | 1.71% | 8.89% | 11.67% | +| CPU busy (mean) | 23.5% | 28.1% | 50.2% | 53.7% | +| CPU busy (P95) | 49.4% | 53.4% | 85.2% | 90.2% | +| Disk busy (P95) | 69.3% | 73.7% | 64.9% | 64.7% | + +The NIC is roughly 98% idle in every configuration, so these pulls are +concurrency-limited rather than bandwidth-limited. Disk is the second-most +loaded resource and is the one to watch when raising unpack concurrency, +because unpacking is write-heavy. + +Gantry CPU is the exception. Gantry nodes pull layers, unpack them, and serve +roughly 42 TB to peers, which puts them at 90.2% P95 CPU on 8 vCPU at 6 +concurrent downloads. Baseline nodes, doing only the first two, sit at 53.4%. +That is worth watching as a headroom limit, but the delivery timeline below +shows it is not what makes the Gantry phase slower. + +## The Gantry phase is slower because of its cold start, not contention + +Peer fetch outcomes over the whole Gantry-cold phase, summed across 1000 nodes: + +| Outcome | Count | Mean duration | +| --- | ---: | ---: | +| busy (HTTP 429) | 1,162,435 | 0.0015s | +| hit | 41,789 | n/a | +| stall | 31,339 | 60.0007s | +| notfound | 10,164 | n/a | +| unavailable | 4,668 | n/a | +| digest_mismatch, auth, protocol, server, local error | 0 | n/a | + +Only 3.3% of attempts succeed, and there are 27.8 rejections per success. Those +headline numbers invite two wrong conclusions, so both are worth ruling out. + +First, the rejections are almost entirely a startup transient. 99.6% of them +occur in the first six minutes and 56% in the second minute alone, falling to +zero by minute eleven. At 1.5ms each they cost about 1.7 seconds per node in +total. + +Second, the stalls are not lost work. These historical runs used a 60s +`PeerFetchTimeout`, and the measured 60.0007s mean is that absolute deadline +firing on progressing transfers. `livePeerStream` streams through to the +containerd-facing response, records the verified byte offset, and resumes from +that offset after re-selection. The default is now 15m; the HTTP/2 transport's +10s idle-health probe remains the dead-connection detector. + +What does explain the gap is the rate at which layer bytes reach nodes: + +| Minute | Layer GB served, all nodes | MB/s per node | Cumulative | +| ---: | ---: | ---: | ---: | +| 0 | 0 | 0.0 | 0.0% | +| 1 | 936 | 15.6 | 2.2% | +| 2 | 2,220 | 37.0 | 7.3% | +| 3 | 4,306 | 71.8 | 17.4% | +| 4 | 4,953 | 82.6 | 28.9% | +| 5 to 9 | about 5,100 each | about 85 | 88.1% | +| 10 | 4,617 | 77.0 | 98.9% | +| 11 | 487 | 8.1 | 100.0% | + +Baseline reaches its full network rate inside the first minute because ACR is +already at capacity. Gantry needs four minutes, because at the start of a cold +run almost no node holds the image and there is nothing to serve. Supply has to +be built before it can be consumed, and the 429 storm is the visible signature +of that shortage rather than a cost in its own right. + +Once the swarm is warm, Gantry is the faster of the two. Node network receive +peaks near 350 MB/s during the Gantry phase against about 182 MB/s for baseline. + +At the observed steady rate of about 5,087 GB per minute, all 42.95 TB would +move in 8.4 minutes. It took roughly 11, so the cascade ramp costs about 2.6 +minutes. Byte delivery finishes at minute 11 while the phase runs to minute 18; +that closing stretch is the serialized unpack draining, which matches the 334s +of measured unpack work per node. + +| Component | Time | Attribution | +| --- | ---: | --- | +| Cascade cold-start ramp | about 2.6 min | Gantry only | +| Steady-state delivery | about 8.4 min | Gantry faster than baseline | +| Serialized unpack tail | about 5.6 min | both phases | + +Gantry's total penalty against baseline in this run was 1.7 minutes, less than +the ramp alone, because its steady-state throughput recovers part of the deficit. + +### The ramp exists because every node pulls layers in the same order + +containerd walks the manifest in order, so all 1000 nodes request layer +positions in the same sequence. The containerd journal records each +`layer unpacked` event with its digest, and the completions fall into strict +waves. Seconds are measured from the first event in the phase. + +| Gantry-cold layer | events | first | median | last | +| --- | ---: | ---: | ---: | ---: | +| `6a98e6e4b146` | 2000 | 0 | 114 | 232 | +| `b407264bc620` | 2000 | 8 | 121 | 243 | +| `06e50d442e52` | 2000 | 15 | 129 | 255 | +| `8d171b1dbb8e` | 2000 | 21 | 138 | 266 | +| `c3e575abbc22` | 2000 | 28 | 147 | 275 | +| `5843544c2584` | 2000 | 36 | 155 | 285 | +| `1af94d827f87` | 2000 | 43 | 163 | 297 | + +Each layer's first completion trails the previous one by roughly 7 seconds, and +the medians advance in the same order. Independent per-node layer selection +would instead produce overlapping distributions starting near zero. The 7 second +step also matches the serialized unpack cost, so the wavefront advances at about +the speed of one layer. + +Extrapolating that step across 40 layers puts the first seed for the final layer +near 280 seconds, which is the same scale as the observed four minute ramp. +Until some node has worked through the preceding layers, the later positions +have no seeder anywhere in the swarm, so demand for them cannot be served at any +price. + +Baseline is the control. It shows the same stagger, with first completions at 0, +6, 13, 22, 29, 36 and 44 seconds, because it uses the same manifest order. It +has no ramp at all, because ACR already holds every layer at t=0. Uniform +ordering is therefore harmless when supply exists and expensive only when supply +has to be built. + +This also re-explains the 429 storm. The rejections are not diffuse contention: +1000 nodes want the same layer at the same moment, which saturates whichever few +nodes hold it. Concentrated demand on a narrow wavefront is the cause, and the +serve cap is what makes it visible. + +The journal capture is truncated to 7 distinct layers per phase, and the 2000 +events per layer indicate some line duplication in the capture, so the ordering +and the 7 second step are measured while the 40 layer figure is arithmetic. +Capturing the full journal, or timestamping per-digest mirror advertisements, +would close that gap. + +## Byte reduction is unaffected and remains the headline + +| Run | ACR bytes | Byte reduction | Pulls B/G | Peer bytes served | Fallbacks | +| --- | ---: | ---: | ---: | ---: | ---: | +| downloads=3 | 45.008 TB to 154.787 GB | 99.656% | 1000 / 2 | 42.817 TB | 0 | +| downloads=6 | 47.165 TB to 256.512 GB | 99.456% | 1000 / 4 | 42.734 TB | 0 | + +Both runs reported `FAIL` only because they were configured with a maximum +Gantry-to-baseline P95 ratio of 1.0. Every other sample in `RESULTS.md` used +3.0, which both runs pass. No pod in either run reported `ErrImagePull` or +`ImagePullBackOff`, and neither run recorded an origin fallback. + +## Conclusions + +1. Pod startup on this workload is almost entirely image pull. Scheduling and + container creation are negligible. +2. `max_concurrent_downloads = 6` is worth taking. It cut P95 by 15-20% for + both phases at negligible resource cost. +3. `max_concurrent_unpacks = 1` leaves roughly 333s of strictly serialized work + per node, now 36-44% of the pull. Overlayfs supports the `rebase` capability + needed to parallelize it, so this is the largest untested lever. +4. The Gantry phase is slower than baseline because of its cold start. Delivery + takes four minutes to reach full rate while baseline is there in one, costing + about 2.6 minutes. Neither the 429 storm nor the historical 60s stalls are + the cost: the first is a startup transient at 1.5ms each, and the second + preserves the delivered prefix and occurs while delivery is at peak rate. +5. The ramp exists because every node walks the manifest in the same order, so + the swarm seeds one layer position at a time instead of all 40 at once. Layer + completions arrive in strict waves about 7 seconds apart. Baseline shows the + same ordering and no ramp, which isolates ordering as costly only when supply + must be built rather than already existing at the origin. +6. Once warm, Gantry delivers faster than pulling from the registry, peaking + near 350 MB/s per node against about 182 MB/s for baseline. +7. `PeerFetchTimeout` is a total request deadline rather than a no-progress + deadline. The 60s setting used by these runs imposed a size-dependent rate + floor: 17.9 MB/s for a 1 GiB layer and 716 MB/s for a 40 GiB one. The default + is now 15m, while the transport retains its 10s idle-health probe. +8. Gantry's value on this workload is the 99.5% reduction in registry egress + and origin pulls, not pod startup latency, which stays 15-22% above baseline. + +## Suggested next experiments + +- `max_concurrent_unpacks = 4` at `max_concurrent_downloads = 6`, changing one + variable from the run above. Watch disk busy, which is already at 73.7% P95. +- Desynchronize layer acquisition order across nodes so the swarm seeds every + layer position at once instead of advancing a single wavefront. With 1000 + nodes and 40 layers, starting nodes at staggered offsets would put a seed on + every position within roughly one layer-time rather than 40. This targets the + ramp, which is the entire Gantry penalty. +- Do not raise `max_concurrent_downloads` past 6 until unpack concurrency is + addressed, since byte-waiting is no longer the majority of baseline pull time. + +## Reproducing + +Per-layer and per-image unpack durations, with the parallel flag: + +```bash +r=/var/lib/gantry-benchmark/artifacts/ +grep -ao 'msg=\\"layer unpacked\\" duration=[0-9.]*[a-z]*' "$r/baseline-performance.json" +grep -ao 'msg=\\"image unpacked\\"[^|]\{0,400\}' "$r/baseline-performance.json" +grep -ao 'parallel=[a-z]*' "$r/baseline-performance.json" | sort | uniq -c +``` + +Per-layer completion order, which shows the wavefront: + +```bash +grep -ao 'time=\\"[0-9T:.Z-]*\\" level=debug msg=\\"layer unpacked\\" duration=[0-9.a-z]* layer=\\"sha256:[0-9a-f]*' \ + "$r/gantry_cold-performance.json" +``` + +Audit-derived latency decomposition: + +```bash +jq -c '{startup: .baseline.azure.audit.pod_startup_latency, + scheduling: .baseline.azure.audit.scheduling_latency, + post_bind: .baseline.azure.audit.post_bind_startup_latency}' "$r/comparison.json" +``` + +Node resource utilization is under `.prometheus[] | select(.name == "") +| .response.data.result` in the phase performance artifacts, using capture names +`node_cpu_busy_ratio`, `node_disk_busy_ratio`, and +`node_network_receive_utilization_ratio`. + +Peer fetch outcomes, DHT results, and layer bytes served use the same path with +capture names `gantry_peer_outcomes`, `gantry_peer_duration`, +`gantry_dht_outcomes`, `gantry_dht_duration`, and `gantry_mirror_bytes`. These +are counters, so a phase total is the last sample minus the first, summed across +pods; bin those per-sample increases by minute to recover the timelines above. diff --git a/hack/gantry-benchmark/README.md b/hack/gantry-benchmark/README.md index 0e25dcaf4..576abf21e 100644 --- a/hack/gantry-benchmark/README.md +++ b/hack/gantry-benchmark/README.md @@ -1,19 +1,57 @@ # Gantry ACR benchmark -This workflow compares direct ACR image distribution with Gantry on an -existing 300-node test cluster. It is adapted from the standalone Gantry demo -that produced the project's published 300-node results. +## Repeatable full-stack deployment -The workflow does not provision AKS, create ACR, or install Gantry. It expects: +Use `deploy.sh` as the only entrypoint for creating or reconciling the Azure and +Kubernetes infrastructure needed by this benchmark. Do not recreate the setup +from commands copied out of the playbook or shell history. -- Exactly 300 Ready, schedulable `linux/amd64` nodes. -- A Ready `gantry-system/gantry` DaemonSet on all 300 nodes. +```bash +cp hack/gantry-benchmark/deploy.env.example hack/gantry-benchmark/deploy.env +# Edit the subscription, deployment name, and globally unique ACR names. + +make -C hack/gantry-benchmark deploy-plan +make -C hack/gantry-benchmark deploy +make -C hack/gantry-benchmark deploy-status +``` + +The script is idempotent and rejects existing resources whose topology differs +from the config. It owns the VNet/subnets, 1000-node AKS shape, two Premium ACRs, +dedicated data endpoints, Private Endpoints/DNS, diagnostics, immutable branch +images, containerd settings, deterministic node-side ACR routing, bounded +Prometheus discovery, Gantry, and the private operator VM. It leaves the stack +preflight-ready by default; set `START_BENCHMARK=true` in `deploy.env` only when +the same invocation should start the benchmark after every deployment gate +passes. + +The deployment config contains names and topology only. Credentials remain in +Azure managed identities and short-lived ACR tokens. + +The workstation needs one valid Azure management-plane login before invoking +`deploy.sh`; the script never invokes `az login`, `az acr login`, or workstation +Podman. It publishes only the revision-labelled source carrier through an ACR +Task, creates Private Endpoints and disables public registry access, then +bootstraps the private operator VM. Gantry and pull-probe images are built and +pushed from that VM with its managed identity over Private Link. + +The sections below document benchmark behavior and direct lifecycle control. +They do not replace `deploy.sh`; commands that assume an existing cluster are +for diagnosis or manual operation after full-stack deployment succeeds. + +When invoking the benchmark tool directly, it expects: + +- Exactly `BENCHMARK_NODE_COUNT` Ready, schedulable `linux/amd64` nodes. +- A Ready `gantry-system/gantry` DaemonSet on every benchmark node. - A dedicated Gantry ACR listed exactly once in Gantry's `upstream_registries` configuration, plus a different baseline ACR. - kube-prometheus-stack, the Prometheus Operator CRDs, kube-state-metrics, and a Grafana dashboard sidecar. The workflow installs benchmark-owned PodMonitors for Gantry and, in proxy mode, the proxy. - Containerd configured to read `/etc/containerd/certs.d`. +- Containerd metrics listening on `0.0.0.0:10257` and debug logging enabled. + The managed node template in this repository configures both. Preflight + refuses to run without a containerd scrape from every target node, and the + node-observer DaemonSet fails if effective containerd log level is not debug. - A private operator VM in the AKS VNet. The VM runs every benchmark command, builds and pushes both images, queries Azure telemetry, and stores artifacts. - Cluster permission to create privileged hostPath DaemonSets. Proxy mode also @@ -22,8 +60,9 @@ The workflow does not provision AKS, create ACR, or install Gantry. It expects: For source-authoritative Azure measurements, set `BENCHMARK_AZURE_TELEMETRY=true`. This additionally requires: -- Both ACRs reachable only through their own approved Private Endpoints, with - public access disabled throughout. The operator VM reaches both ACRs over +- Both ACRs reachable only through their own approved Private Endpoints before + operator and benchmark validation, with public access disabled throughout + image preparation and measurement. The operator VM reaches both ACRs over Private Link while preparing and measuring the images. - A Log Analytics workspace receiving `ContainerRegistryRepositoryEvents` and `AKSAuditAdmin` in resource-specific tables. @@ -87,6 +126,44 @@ hostnames alone do not isolate containerd's digest-addressed cache; the phase-specific layer paths provide that isolation while preserving identical payload bytes. +## Performance attribution artifacts + +`enable` installs a benchmark-owned node-observer DaemonSet on every target +node. It exposes node-exporter metrics, provides a Prometheus target for the +host containerd metrics endpoint, and streams a filtered subset of the host +containerd journal. Preflight requires both `node_uname_info` and +`containerd_build_info_total` from every observer pod. + +Each phase writes `-performance.json` with the unmodified Prometheus +range-query envelopes at 10-second resolution for: + +- host disk bytes, I/O busy time, CPU, memory, and network bytes/errors; +- containerd process and built-in metrics; +- per-Gantry-pod peer outcomes and durations; +- exact latest peer busy/stall event timestamps plus interval counts; +- per-Gantry-pod DHT outcomes and durations; +- mirror bytes and final response-completion timestamps; and +- stream-completion-to-containerd-inventory observation distributions and + latest measured durations. + +The same artifact includes the observer-pod-to-node map, raw filtered +containerd journal, and phase-bounded structured events for `PullImage`, +successful pull completion, no-progress cancellation, `layer unpacked`, and +`image unpacked`. Capture fails unless every observer pod has an event in the +phase window. Containerd's +`layer unpacked` duration spans fetch, apply, and snapshot commit; it is not a +pure filesystem-write duration. Host disk metrics provide the independent +filesystem pressure signal during that span. + +Phase JSON also includes: + +- exact per-workload-pod container start/finish timestamps and node names; and +- per-Gantry-pod counter deltas and phase-local timestamp gauges. + +These two node maps are the supported join key for correlating peer busy/stall +events, DHT latency, final layer-response time, containerd commit observation, +containerd unpack logs, host resource use, and workload startup latency. + ## Lifecycle All lifecycle commands run on the operator VM under its system-assigned managed @@ -120,21 +197,156 @@ az vm run-command invoke -g "$AZURE_RESOURCE_GROUP" \ Follow progress from the workstation in a separate terminal: ```bash -export OPERATOR_SSH_HOST="" -export OPERATOR_SSH_KEY="tmp/gantry-benchmark-ssh-key" +export AZURE_RESOURCE_GROUP="" +export OPERATOR_VM_NAME="gantry-benchmark-operator" make -C hack/gantry-benchmark operator-vm-watch ``` -SSH mode is preferred because each refresh is immediate. The operator VM SSH -NSG rule must allow TCP/22 only from the current workstation `/32`. If -`OPERATOR_SSH_HOST` is unset, the watcher falls back to Azure Run Command using -`AZURE_RESOURCE_GROUP` and `OPERATOR_VM_NAME`. +The full-stack deployment gives the operator VM no public IP. Azure Run Command +is the default status transport. SSH is optional only when the operator has +deliberately provided private network connectivity to the VM. The live view reports the lifecycle stage and start time, immutable run shape, -payload files/bytes/percentage, active Podman build or push, VM disk usage, -Kubernetes Job completion, Gantry readiness, recent logs, and the final report. -Use `operator-vm-status` for a single snapshot. Override the refresh cadence -with `WATCH_INTERVAL_SECONDS` (default 30). +payload files/bytes/percentage, each baseline and Gantry-cold image reference, +image size, layer count, build/push state, completed digest, active image +operation and elapsed time, per-phase Kubernetes Job completion, Gantry +readiness, recent logs, and the final report. Podman 4.9 does not expose a +machine-readable live push byte percentage, so the view reports that limitation +instead of estimating it. Use `operator-vm-status` for a single snapshot. +Override the refresh cadence with `WATCH_INTERVAL_SECONDS` (default 30). + +For the Gantry pull phase, use the dedicated live monitor from the workstation: + +```bash +make -C hack/gantry-benchmark monitor +``` + +It redraws every second and shows per-phase-minute peer outcomes (`busy`, +`hit`, `stall`, `notfound`, `unavailable`) alongside layer bytes, aggregate +and per-node throughput, cumulative payload percentage, and live Pod counts for +completed, running, creating, image-pull failures, and failed Pods. Pod counts +come from one Kubernetes list followed by watch events rather than polling all +1000 Pod objects. + +Two node-paged grids show the current image in detail. The layer-by-node grid +uses `.` for pending and `0-9/A-Z` for the phase minute when Gantry finished +writing that layer to the node's containerd client (`Z` means minute 35 or +later). The image-by-node grid uses `.` for not started, `0` for started, +`1-9` for unpacked-layer deciles, and `#` after containerd reports the image +unpacked. + +The same node page shows one-minute CPU utilization averaged across cores and +current memory-used percentage (`1 - MemAvailable / MemTotal`) for every node +as `0-9` deciles, plus fleet p50, p95, and maximum values. These resource +values advance at the 10-second Prometheus scrape cadence. Node columns are +sorted and identified by their three-character +suffix. The default page width follows the terminal, bounded to 16-96 nodes. +Select another page or a fixed width with, for example: + +```bash +make -C hack/gantry-benchmark monitor \ + MONITOR_ARGS="--node-page 2 --nodes-per-page 64" +``` + +The header uses ANSI emphasis on a TTY and remains plain when redirected or +piped. The monitor uses one server-side aggregated Prometheus range query per +display refresh and one exact-image progress query every 10 seconds. Prometheus +scrapes at 10-second cadence, so the screen updates each second while grid and +counter values advance at scrape cadence. +Use `MONITOR_ARGS="--once --no-clear"` for a single non-interactive snapshot. + +### Gantry CPU profiles + +Benchmark deployments enable Go pprof on each Gantry pod at the loopback-only +address `127.0.0.1:6060`. It is not declared as a pod port and is reachable +from the workstation only through `kubectl port-forward`. During an active +Gantry-cold phase, capture concurrent CPU profiles from the three nodes with +the highest one-minute CPU utilization: + +```bash +make -C hack/gantry-benchmark profile-gantry +``` + +Override the sample duration and node count with `GANTRY_PPROF_SECONDS` and +`GANTRY_PPROF_COUNT`. The command stores individual and merged protobuf +profiles plus text reports under `tmp/gantry-pprof/-/` and +prints the merged top functions. Open the merged profile interactively with +the `go tool pprof -http=...` command printed at completion. + +CPU profiling adds runtime overhead to the selected pods. Treat a profiled +run as diagnostic and do not use it for benchmark comparisons. The sampler +annotates the active Job with the capture timestamp, duration, requested pod +count, and successfully captured pod count so the diagnostic status remains +visible after the run finishes. If one target fails, two or more valid profiles +are still merged and the failed target's port-forward log is retained. + +## Reusable Gantry image pool + +For a one-off Gantry-only run that creates a brand-new random 40 GiB image +inside the lifecycle, use the retained baseline without involving the image +pool: + +```bash +AZURE_RESOURCE_GROUP=vapa-gantry-benchmark1 \ +OPERATOR_VM_NAME=gantry-benchmark-operator \ +GANTRY_ONLY_BASELINE_RUN_ID=run-20260806-205719-51c38730 \ +make -C hack/gantry-benchmark operator-vm-run-fresh +``` + +Gantry-only benchmarks can consume prebuilt images instead of generating, +building, and pushing 40 GiB during every benchmark lifecycle. Start a batch +on the operator VM from the workstation: + +```bash +AZURE_RESOURCE_GROUP=vapa-gantry-benchmark1 \ +OPERATOR_VM_NAME=gantry-benchmark-operator \ +GANTRY_IMAGE_POOL_COUNT=10 \ +make -C hack/gantry-benchmark operator-vm-prebuild +``` + +The command starts `gantry-benchmark-image-builder.service` asynchronously. +The builder authenticates its managed identity once, generates each random +payload sequentially, pushes each image to the Gantry ACR, records its +immutable digest and payload SHA-256, removes the local image, and deletes the +40 GiB build context before starting the next image. Durable pool metadata +lives under `/var/lib/gantry-benchmark/image-pool`; transient build data lives +on the `/opt/gantry-benchmark` build disk. + +Use the bounded status view while it runs: + +```bash +AZURE_RESOURCE_GROUP=vapa-gantry-benchmark1 \ +OPERATOR_VM_NAME=gantry-benchmark-operator \ +make -C hack/gantry-benchmark operator-vm-image-pool-status +``` + +Start a Gantry-only benchmark with the oldest compatible ready image: + +```bash +AZURE_RESOURCE_GROUP=vapa-gantry-benchmark1 \ +OPERATOR_VM_NAME=gantry-benchmark-operator \ +GANTRY_ONLY_BASELINE_RUN_ID=run-20260806-205719-51c38730 \ +make -C hack/gantry-benchmark operator-vm-run-pool +``` + +The benchmark restores the retained baseline metadata automatically, atomically +moves one image from `ready/` to `claimed/`, and adopts its immutable digest. +Claimed images are never offered to a later run, preserving the cache-cold +contract. Pool adoption performs no image build, push, or registry credential +exchange inside the benchmark lifecycle. + +Pool building and benchmark execution are mutually exclusive on one operator +VM. Both services hold the same lifecycle lock, and the builder also refuses +to run while a benchmark state exists. This is required for measurement +correctness: pushing another image to the Gantry ACR during a measured phase +would produce unrelated repository events and invalidate the Azure telemetry +completeness gate. Build the 5-10 image batch before starting benchmark runs, +not concurrently with them. + +For direct CLI use with an already-authenticated container engine, the +equivalent targets are `prebuild-gantry`, `image-pool-status`, and +`prepare-gantry-pool`. Configure metadata and scratch locations with +`BENCHMARK_IMAGE_POOL_ROOT` and `BENCHMARK_IMAGE_POOL_BUILD_ROOT`. Artifacts persist on the VM under `/var/lib/gantry-benchmark/artifacts//`; `latest` points at the newest diff --git a/hack/gantry-benchmark/RESULTS.md b/hack/gantry-benchmark/RESULTS.md index 3e40a8ecb..764c8133a 100644 --- a/hack/gantry-benchmark/RESULTS.md +++ b/hack/gantry-benchmark/RESULTS.md @@ -16,6 +16,10 @@ We removed prior benchmark images before repeat samples, measured registry traffic at each Azure Private Endpoint, measured pod startup from AKS audit logs, and measured peer traffic from Gantry metrics. +For a breakdown of where pod startup time is actually spent, and the effect of +containerd download and unpack concurrency, see +[PULL-LATENCY-ANALYSIS.md](PULL-LATENCY-ANALYSIS.md). + ### ACR traffic @@ -23,6 +27,10 @@ logs, and measured peer traffic from Gantry metrics. | --- | ---: | ---: | ---: | ---: | ---: | ---: | | 1000 nodes - sample 1 | 40 GiB | 47.296 TB | 174.773 GB | 99.630% | 1002 / 4 | 99.601% | | 1000 nodes - sample 2 | 40 GiB | 47.566 TB | 174.781 GB | 99.633% | 1008 / 5 | 99.504% | +| **1000 nodes - Canada Central ACR** | **40 GiB** | **47.178 TB** | **245.878 GB** | **99.479%** | **1000 / 2** | **99.800%** | +| **1000 nodes - Canada Central ACR rerun** 3 | **40 GiB** | **45.008 TB** | **154.787 GB** | **99.656%** | **1000 / 2** | **99.800%** | +| **1000 nodes - Canada Central ACR, 6 downloads** 4 | **40 GiB** | **47.165 TB** | **256.512 GB** | **99.456%** | **1000 / 4** | **99.600%** | +| **1000 nodes - Canada Central, bounded coordination** 5 | **40 GiB** | **47.165 TB** | **945.400 GB** | **97.996%** | **1000 / 1** | **99.900%** | | **1000 nodes - UK South ACR** | **40 GiB** | **53.369 TB** | **219.262 GB** | **99.589%** | **1254 / 5** | **99.601%** | | **1000 nodes - East US ACR** | **40 GiB** | **47.562 TB** | **182.317 GB** | **99.617%** | **1004 / 4** | **99.602%** | | **1000 nodes - Central India ACR** 1 | **40 GiB** | **97.115 TB** | **803.184 GB** | **99.173%** | **2287 / 6** | **99.738%** | @@ -46,6 +54,10 @@ aggregates below. | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | | 1000 nodes - sample 1 | 40 GiB | 682.725s | 1099.629s | 821.209s | 1171.863s | 1422.461s | 1885.385s | 42.700% slower | | 1000 nodes - sample 2 | 40 GiB | 894.597s | 1087.179s | 1065.724s | 1169.448s | 1652.704s | 1885.011s | 9.733% slower | +| **1000 nodes - Canada Central ACR** | **40 GiB** | **1862.580s** | **774.028s** | **2030.636s** | **817.139s** | **2149.535s** | **891.766s** | **59.759% faster** | +| **1000 nodes - Canada Central ACR rerun** 3 | **40 GiB** | **939.368s** | **1105.766s** | **1091.173s** | **1180.970s** | **1180.111s** | **1239.429s** | **8.229% slower** | +| **1000 nodes - Canada Central ACR, 6 downloads** 4 | **40 GiB** | **759.746s** | **923.473s** | **867.850s** | **994.468s** | **956.492s** | **1058.555s** | **14.590% slower** | +| **1000 nodes - Canada Central, bounded coordination** 5 | **40 GiB** | **759.746s** | **917.602s** | **867.850s** | **1001.609s** | **956.492s** | **1132.420s** | **15.413% slower** | | **1000 nodes - UK South ACR** | **40 GiB** | **3561.000s** | **1064.557s** | **3953.000s** | **1146.557s** | **5399.000s** | **1815.557s** | **70.995% faster** | | **1000 nodes - East US ACR** | **40 GiB** | **1401.026s** | **1065.950s** | **1655.894s** | **1144.771s** | **2351.081s** | **1831.832s** | **30.867% faster** | | **1000 nodes - Central India ACR** 2 | **40 GiB** | **3184.649s** | **1065.570s** | **4851.649s** | **1155.570s** | **5351.649s** | **1865.570s** | **76.182% faster** | @@ -53,16 +65,73 @@ aggregates below. | 2000 nodes - sample 2 | 40 GiB | 939.834s | 1093.091s | 1141.022s | 1177.821s | 1724.219s | 1856.380s | 3.225% slower | | 2000 nodes - sample 3 | 40 GiB | 1241.331s | 1096.589s | 1472.041s | 1184.248s | 2131.053s | 1821.000s | 19.551% faster | -Positive improvement means Gantry started pods faster. The configured gate was -a maximum Gantry-to-baseline P95 ratio of 3.0, so all six audit-complete runs -and the cross-region performance-only samples passed even when an unusually -fast baseline made Gantry slower. The UK South and Central India rows use -retained Kubernetes pod status timestamps; every other row, including East US, -uses AKS audit timestamps. +Positive improvement means Gantry started pods faster. Most rows used a maximum +Gantry-to-baseline P95 ratio of 3.0. The three Canada Central follow-ups marked +3 through 5 used a tightened ratio of 1.0 and reported FAIL on latency while +passing their traffic and integrity gates. The UK South and Central India rows +use retained Kubernetes pod status timestamps; every other row, including +East US, uses AKS audit timestamps. 2 Central India latency uses retained Kubernetes pod status because the telemetry timeout occurred before the runner wrote its audit measurement. +3 Run `run-20260806-185139-9995e167`, reported **FAIL**. Byte and +pull reduction were the strongest Canada Central results recorded, but the +baseline was unusually fast: its P95 of 1091.173s is roughly half the 2030.636s +of the earlier Canada Central sample, while Gantry landed at 1180.970s, close to +the 1146.557s to 1184.248s that Gantry produces on nearly every run. The +resulting P95 ratio of 1.0823 exceeded the 1.0 gate configured for this run, +which the 3.0 gate used elsewhere would have passed. All 2000 pods across both +phases succeeded with no image-pull backoff and no origin fallbacks. This +sample also ran with the containerd transfer-service download concurrency +override removed, so both phases used the containerd default of 3. + +4 Run `run-20260806-205719-51c38730`, reported **FAIL**. Same cluster +as footnote 3 with the containerd transfer-service override restored to 6 +concurrent downloads, isolating that one variable. Both phases got faster: +baseline P95 improved 20.5% and Gantry P95 improved 15.8%. The remaining gap is +Gantry's cold start: layer delivery takes four minutes to reach full rate while +baseline is at full rate within one, costing about 2.6 minutes. Once warm, +Gantry delivers faster than the registry. The P95 ratio was 1.1459 and exceeded +the 1.0 gate this run used. All 2000 pods succeeded with no image-pull backoff +and no origin fallbacks. See +[PULL-LATENCY-ANALYSIS.md](PULL-LATENCY-ANALYSIS.md). + +5 Gantry-only run `run-20260807-035224-b5d10f22`, reported +**FAIL** against the retained baseline from footnote 4 because its P95 ratio +was 1.154, above the strict 1.0 gate. All 1000 pods succeeded, no Gantry pod +restarted, peer delivery reached 100% in minute 11, and no direct-origin +fallback occurred. This run deployed the 15-minute absolute peer-fetch +ceiling, three deterministic remote-prefetch coordinators, bounded 8/4/32 +bootstrap dialing with peer-ID address deduplication, authoritative Pod-IP +replacement before coordination RPCs, and context-aware containerd storage +error classification. + +The fixes removed the targeted failure signatures. Peer stalls fell from +30,829 to zero; remote prefetch groups fell from 575,623 to 1,138 (880 +successful and 258 failed); bootstrap dial failures fell from 1,099,355 to +2,088; and the 1,614 former `storage unavailable` warnings fell to zero. The +configured 2% fanout remained intact: 801 layer origin pulls completed for 40 +layers and 20 selected seeds per layer, plus one extra completed layer copy. +The higher 945.400 GB ACR figure is therefore expected seed traffic, not peer +fallback. Compared with the immediately preceding Gantry-only run using the +60-second ceiling, P50 increased 4.566%, P95 increased 5.794%, and ACR traffic +changed by only +0.011%. The 15-minute ceiling removed false stall +classification, but did not improve startup latency in this sample. + +Two issues remain visible in the complete all-pod log scan. One remote +prefetch coordinator hit libp2p's default connection resource limit for 117 +groups, while another had 141 transient refusal/backoff failures; redundant +dispatch still started every intended seed. Also, one canceled transfer-store +open was logged at WARN and returned HTTP 500, so the transfer endpoint still +needs the same cancellation handling already applied to the mirror endpoint. +The 6,311 peer `notfound` outcomes were confined to the first 44 seconds and +came from current healthy pods whose stale provider records preceded +containerd inventory reconciliation. None of these observations caused a +failed workload pod, origin fallback, digest mismatch, protocol error, or +server-error peer outcome. This Gantry-only validation reuses the footnote 4 +baseline and is excluded from the aggregate independent-run statistics below. + #### Latency excluding image-pull backoff AKS audit logs retained the pod status patches containing `ErrImagePull` and @@ -89,6 +158,10 @@ The unfiltered table remains the primary end-to-end result because | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | | 1000 nodes - sample 1 | 40 GiB | 43.566 TB | 158.928 GB | 157 | 153 | 42,597 | 0 | | 1000 nodes - sample 2 | 40 GiB | 43.438 TB | 160.002 GB | 159 | 154 | 42,472 | 0 | +| **1000 nodes - Canada Central ACR** | **40 GiB** | **42.735 TB** | **223.358 GB** | **214** | **212** | **41,791** | **0** | +| **1000 nodes - Canada Central ACR rerun** 3 | **40 GiB** | **42.817 TB** | **140.672 GB** | **134** | **132** | **41,870** | **0** | +| **1000 nodes - Canada Central ACR, 6 downloads** 4 | **40 GiB** | **42.734 TB** | **233.022 GB** | **223** | **219** | **41,789** | **0** | +| **1000 nodes - Canada Central, bounded coordination** 5 | **40 GiB** | **42.153 TB** | **859.069 GB** | **806** | **801** | **41,250** | **0** | | **1000 nodes - East US ACR** | **40 GiB** | **43.137 TB** | **161.075 GB** | **159** | **155** | **42,179** | **0** | | **1000 nodes - Central India ACR** | **40 GiB** | **43.070 TB** | **709.766 GB** | **682** | **662** | **42,129** | **0** | | 2000 nodes - sample 1 | 40 GiB | 86.971 TB | 221.210 GB | 213 | 210 | 85,044 | 0 | diff --git a/hack/gantry-benchmark/RUNBOOK.md b/hack/gantry-benchmark/RUNBOOK.md index 9705ee702..fe9b728cc 100644 --- a/hack/gantry-benchmark/RUNBOOK.md +++ b/hack/gantry-benchmark/RUNBOOK.md @@ -1,5 +1,10 @@ # Gantry ACR benchmark runbook +`deploy.sh` is the authoritative infrastructure and installation entrypoint. +Use this runbook only for manual benchmark lifecycle control or diagnosis after +`make -C hack/gantry-benchmark deploy` succeeds. Do not reconstruct deployment +from the individual commands below. + This workflow is for a dedicated test cluster. The benchmark itself runs only on a private VM in the AKS VNet. Run the provisioning and Azure Run Command commands below from the Unbounded repository root on an admin workstation. @@ -18,10 +23,11 @@ export AZURE_GANTRY_ACR_PRIVATE_ENDPOINT_RESOURCE_ID=" [config-file] + +One idempotent entrypoint for the complete Gantry benchmark stack. The config +file is a shell environment file. No credentials are stored in it. + + plan validate inputs and print the complete deployment contract + deploy create or validate every resource and leave the benchmark ready + status report Azure and Kubernetes readiness without mutation +USAGE +} + +action=${1:-plan} +config_file=${2:-${GANTRY_BENCHMARK_DEPLOY_CONFIG:-$script_dir/deploy.env}} + +case "$action" in +plan | deploy | status) ;; +-h | --help | help) + usage + exit 0 + ;; +*) + usage >&2 + exit 2 + ;; +esac + +[[ -f "$config_file" ]] || { + echo "missing deployment config: $config_file" >&2 + echo "copy $script_dir/deploy.env.example and set the required values" >&2 + exit 2 +} + +set -a +# shellcheck source=/dev/null +. "$config_file" +set +a + +: "${AZURE_SUBSCRIPTION_ID:?Set AZURE_SUBSCRIPTION_ID}" +: "${DEPLOYMENT_NAME:?Set DEPLOYMENT_NAME}" +: "${BASELINE_ACR_NAME:?Set globally unique BASELINE_ACR_NAME}" +: "${GANTRY_ACR_NAME:?Set globally unique GANTRY_ACR_NAME}" + +AZURE_LOCATION=${AZURE_LOCATION:-canadacentral} +AZURE_RESOURCE_GROUP=${AZURE_RESOURCE_GROUP:-$DEPLOYMENT_NAME} +AZURE_AKS_CLUSTER_NAME=${AZURE_AKS_CLUSTER_NAME:-${DEPLOYMENT_NAME}-aks} +AZURE_NODE_RESOURCE_GROUP=${AZURE_NODE_RESOURCE_GROUP:-${DEPLOYMENT_NAME}-nodes-rg} +AZURE_LOG_ANALYTICS_WORKSPACE_NAME=${AZURE_LOG_ANALYTICS_WORKSPACE_NAME:-${DEPLOYMENT_NAME}-law} + +VNET_NAME=${VNET_NAME:-${DEPLOYMENT_NAME}-vnet} +VNET_CIDR=${VNET_CIDR:-10.224.0.0/12} +AKS_SUBNET_NAME=${AKS_SUBNET_NAME:-aks-nodes} +AKS_SUBNET_CIDR=${AKS_SUBNET_CIDR:-10.224.0.0/20} +PRIVATE_ENDPOINT_SUBNET_NAME=${PRIVATE_ENDPOINT_SUBNET_NAME:-acr-private-endpoints} +PRIVATE_ENDPOINT_SUBNET_CIDR=${PRIVATE_ENDPOINT_SUBNET_CIDR:-10.225.0.0/27} +OPERATOR_SUBNET_NAME=${OPERATOR_SUBNET_NAME:-gantry-benchmark-operator} +OPERATOR_SUBNET_CIDR=${OPERATOR_SUBNET_CIDR:-10.236.0.0/24} + +POD_CIDR=${POD_CIDR:-10.64.0.0/12} +SERVICE_CIDR=${SERVICE_CIDR:-10.0.0.0/16} +DNS_SERVICE_IP=${DNS_SERVICE_IP:-10.0.0.10} +AKS_KUBERNETES_VERSION=${AKS_KUBERNETES_VERSION:-1.35} +AKS_NODE_POOL_NAME=${AKS_NODE_POOL_NAME:-system} +AKS_NODE_COUNT=${AKS_NODE_COUNT:-1000} +AKS_NODE_VM_SIZE=${AKS_NODE_VM_SIZE:-Standard_D8s_v3} +AKS_NODE_OS_DISK_GB=${AKS_NODE_OS_DISK_GB:-512} +AKS_MAX_PODS=${AKS_MAX_PODS:-250} + +BENCHMARK_NODE_COUNT=${BENCHMARK_NODE_COUNT:-$AKS_NODE_COUNT} +BENCHMARK_IMAGE_SIZE_MIB=${BENCHMARK_IMAGE_SIZE_MIB:-40960} +BENCHMARK_IMAGE_LAYERS=${BENCHMARK_IMAGE_LAYERS:-40} +BENCHMARK_MINIMUM_BYTE_REDUCTION=${BENCHMARK_MINIMUM_BYTE_REDUCTION:-0.90} +BENCHMARK_MAXIMUM_LATENCY_RATIO=${BENCHMARK_MAXIMUM_LATENCY_RATIO:-1.0} +ADOPT_BASELINE_IMAGE=${ADOPT_BASELINE_IMAGE:-} +ADOPT_GANTRY_IMAGE=${ADOPT_GANTRY_IMAGE:-} +ADOPT_PAYLOAD_SHA256=${ADOPT_PAYLOAD_SHA256:-} +BASELINE_PULL_MAX_NODE_REPLACEMENTS=${BASELINE_PULL_MAX_NODE_REPLACEMENTS:-5} + +GANTRY_NAMESPACE=${GANTRY_NAMESPACE:-gantry-system} +BENCHMARK_NAMESPACE=${BENCHMARK_NAMESPACE:-gantry-benchmark} +MONITORING_NAMESPACE=${MONITORING_NAMESPACE:-monitoring} +KPS_RELEASE=${KPS_RELEASE:-kps} +KPS_CHART_VERSION=${KPS_CHART_VERSION:-87.21.0} +PROMETHEUS_SERVICE=${PROMETHEUS_SERVICE:-kps-kube-prometheus-stack-prometheus} + +OPERATOR_VM_NAME=${OPERATOR_VM_NAME:-gantry-benchmark-operator} +OPERATOR_VM_SIZE=${OPERATOR_VM_SIZE:-Standard_D32ds_v5} +OPERATOR_VM_ZONE=${OPERATOR_VM_ZONE:-1} +OPERATOR_OS_DISK_GB=${OPERATOR_OS_DISK_GB:-128} +OPERATOR_BUILD_DISK_GB=${OPERATOR_BUILD_DISK_GB:-512} +OPERATOR_BUILD_DISK_SKU=${OPERATOR_BUILD_DISK_SKU:-PremiumV2_LRS} +OPERATOR_BUILD_DISK_IOPS=${OPERATOR_BUILD_DISK_IOPS:-20000} +OPERATOR_BUILD_DISK_MBPS=${OPERATOR_BUILD_DISK_MBPS:-750} + +START_BENCHMARK=${START_BENCHMARK:-false} +DEPLOY_CONFIRM=${DEPLOY_CONFIRM:-} +DEPLOY_STATE_DIR=${DEPLOY_STATE_DIR:-$repo_root/tmp/$DEPLOYMENT_NAME} +KUBECONFIG=${DEPLOY_KUBECONFIG:-$DEPLOY_STATE_DIR/kubeconfig} +OPERATOR_RUN_COMMAND_LOCK=${OPERATOR_RUN_COMMAND_LOCK:-${TMPDIR:-/tmp}/gantry-benchmark-${AZURE_RESOURCE_GROUP}-${OPERATOR_VM_NAME}.run-command.lock} + +BASELINE_PRIVATE_ENDPOINT_NAME=${BASELINE_PRIVATE_ENDPOINT_NAME:-${DEPLOYMENT_NAME}-baseline-acr-pe} +GANTRY_PRIVATE_ENDPOINT_NAME=${GANTRY_PRIVATE_ENDPOINT_NAME:-${DEPLOYMENT_NAME}-gantry-acr-pe} +PRIVATE_DNS_ZONE=privatelink.azurecr.io +PRIVATE_DNS_LINK_NAME=${PRIVATE_DNS_LINK_NAME:-${DEPLOYMENT_NAME}-acr-link} + +BASELINE_ACR_LOGIN_SERVER=${BASELINE_ACR_NAME}.azurecr.io +GANTRY_ACR_LOGIN_SERVER=${GANTRY_ACR_NAME}.azurecr.io +BASELINE_ACR_DATA_HOST=${BASELINE_ACR_NAME}.${AZURE_LOCATION}.data.azurecr.io +GANTRY_ACR_DATA_HOST=${GANTRY_ACR_NAME}.${AZURE_LOCATION}.data.azurecr.io + +[[ "$AKS_NODE_COUNT" =~ ^[1-9][0-9]*$ ]] || { echo "AKS_NODE_COUNT must be positive" >&2; exit 2; } +[[ "$BENCHMARK_NODE_COUNT" == "$AKS_NODE_COUNT" ]] || { + echo "BENCHMARK_NODE_COUNT must equal AKS_NODE_COUNT for this topology" >&2 + exit 2 +} +[[ "$BENCHMARK_IMAGE_LAYERS" =~ ^[1-9][0-9]*$ ]] || { echo "BENCHMARK_IMAGE_LAYERS must be positive" >&2; exit 2; } +((BENCHMARK_IMAGE_LAYERS <= BENCHMARK_IMAGE_SIZE_MIB)) || { + echo "BENCHMARK_IMAGE_LAYERS cannot exceed BENCHMARK_IMAGE_SIZE_MIB" >&2 + exit 2 +} +[[ "$BASELINE_PULL_MAX_NODE_REPLACEMENTS" =~ ^[1-9][0-9]*$ ]] || { + echo "BASELINE_PULL_MAX_NODE_REPLACEMENTS must be positive" >&2 + exit 2 +} +for acr_name in "$BASELINE_ACR_NAME" "$GANTRY_ACR_NAME"; do + [[ "$acr_name" =~ ^[a-z0-9]{5,50}$ ]] || { + echo "invalid ACR name $acr_name: use 5-50 lowercase alphanumeric characters" >&2 + exit 2 + } +done +[[ "$START_BENCHMARK" == true || "$START_BENCHMARK" == false ]] || { + echo "START_BENCHMARK must be true or false" >&2 + exit 2 +} +valid_adopted_image() { + local image=$1 + local login_server=$2 + local prefix="$login_server/gantry-benchmark-pull@" + [[ "$image" == "$prefix"* && "${image#"$prefix"}" =~ ^sha256:[0-9a-f]{64}$ ]] +} +adoption_values=0 +for value in "$ADOPT_BASELINE_IMAGE" "$ADOPT_GANTRY_IMAGE" "$ADOPT_PAYLOAD_SHA256"; do + [[ -z "$value" ]] || adoption_values=$((adoption_values + 1)) +done +if ((adoption_values != 0 && adoption_values != 3)); then + echo "ADOPT_BASELINE_IMAGE, ADOPT_GANTRY_IMAGE, and ADOPT_PAYLOAD_SHA256 must be set together" >&2 + exit 2 +fi +if ((adoption_values == 3)); then + valid_adopted_image "$ADOPT_BASELINE_IMAGE" "$BASELINE_ACR_LOGIN_SERVER" || { + echo "ADOPT_BASELINE_IMAGE must be an immutable gantry-benchmark-pull image in $BASELINE_ACR_LOGIN_SERVER" >&2 + exit 2 + } + valid_adopted_image "$ADOPT_GANTRY_IMAGE" "$GANTRY_ACR_LOGIN_SERVER" || { + echo "ADOPT_GANTRY_IMAGE must be an immutable gantry-benchmark-pull image in $GANTRY_ACR_LOGIN_SERVER" >&2 + exit 2 + } + [[ "$ADOPT_PAYLOAD_SHA256" =~ ^sha256:[0-9a-f]{64}$ ]] || { + echo "ADOPT_PAYLOAD_SHA256 must be a sha256 digest" >&2 + exit 2 + } +fi +assert_default() { + local name=$1 + local actual=$2 + local expected=$3 + [[ "$actual" == "$expected" ]] || { + echo "$name=$actual is unsupported; the operator contract requires $expected" >&2 + exit 2 + } +} +assert_default GANTRY_NAMESPACE "$GANTRY_NAMESPACE" gantry-system +assert_default BENCHMARK_NAMESPACE "$BENCHMARK_NAMESPACE" gantry-benchmark +assert_default MONITORING_NAMESPACE "$MONITORING_NAMESPACE" monitoring +assert_default KPS_RELEASE "$KPS_RELEASE" kps +assert_default PROMETHEUS_SERVICE "$PROMETHEUS_SERVICE" kps-kube-prometheus-stack-prometheus + +log() { printf '%s [deploy] %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*"; } + +require_command() { + command -v "$1" >/dev/null 2>&1 || { echo "required command not found: $1" >&2; exit 1; } +} + +retry_command() { + local attempts=$1 + local delay=$2 + shift 2 + local attempt + for attempt in $(seq 1 "$attempts"); do + if "$@"; then + return 0 + fi + if ((attempt == attempts)); then + return 1 + fi + log "attempt $attempt/$attempts failed; retrying in ${delay}s" + sleep "$delay" + done +} + +print_plan() { + local image_preparation="build and push fresh workload images" + if ((adoption_values == 3)); then + image_preparation="adopt existing immutable workload images" + fi + cat </dev/null || echo unknown) + source carrier: ACR Task before registry privatization + runtime images: managed-identity operator over Private Link + +Azure + subscription: $AZURE_SUBSCRIPTION_ID + location: $AZURE_LOCATION + resource group: $AZURE_RESOURCE_GROUP + AKS: $AZURE_AKS_CLUSTER_NAME + node resource group: $AZURE_NODE_RESOURCE_GROUP + node pool: $AKS_NODE_POOL_NAME ($AKS_NODE_COUNT x $AKS_NODE_VM_SIZE) + node OS disk: ${AKS_NODE_OS_DISK_GB} GiB managed + Kubernetes: $AKS_KUBERNETES_VERSION + +Network + VNet: $VNET_NAME $VNET_CIDR + AKS subnet: $AKS_SUBNET_NAME $AKS_SUBNET_CIDR + pod CIDR: $POD_CIDR + service CIDR: $SERVICE_CIDR + PE subnet: $PRIVATE_ENDPOINT_SUBNET_NAME $PRIVATE_ENDPOINT_SUBNET_CIDR + operator subnet: $OPERATOR_SUBNET_NAME $OPERATOR_SUBNET_CIDR + +Registries + baseline: $BASELINE_ACR_LOGIN_SERVER + Gantry: $GANTRY_ACR_LOGIN_SERVER + access: Premium, dedicated data endpoint, Private Endpoint, public disabled at completion + +Benchmark + nodes: $BENCHMARK_NODE_COUNT + payload: ${BENCHMARK_IMAGE_SIZE_MIB} MiB in $BENCHMARK_IMAGE_LAYERS layers + image preparation: $image_preparation + adopted baseline: ${ADOPT_BASELINE_IMAGE:-none} + adopted Gantry: ${ADOPT_GANTRY_IMAGE:-none} + adopted payload: ${ADOPT_PAYLOAD_SHA256:-none} + monitoring: kube-prometheus-stack $KPS_CHART_VERSION with benchmark-only discovery + operator: $OPERATOR_VM_SIZE with ${OPERATOR_BUILD_DISK_GB} GiB $OPERATOR_BUILD_DISK_SKU + start benchmark: $START_BENCHMARK +PLAN +} + +if [[ "$action" == plan ]]; then + print_plan + exit 0 +fi + +for command in az jq kubectl helm git make sha256sum timeout; do + require_command "$command" +done + +az account set --subscription "$AZURE_SUBSCRIPTION_ID" +if ! timeout 60s az account get-access-token --resource https://management.core.windows.net/ --output none; then + echo "Azure management authentication is unavailable; run az login once before deploy.sh" >&2 + exit 1 +fi + +if [[ "$action" == status ]]; then + az group show -g "$AZURE_RESOURCE_GROUP" --query '{name:name,location:location,state:properties.provisioningState}' -o json + az aks show -g "$AZURE_RESOURCE_GROUP" -n "$AZURE_AKS_CLUSTER_NAME" \ + --query '{state:provisioningState,version:kubernetesVersion,nodeResourceGroup:nodeResourceGroup}' -o json + az acr list -g "$AZURE_RESOURCE_GROUP" \ + --query '[].{name:name,publicNetworkAccess:publicNetworkAccess,dataEndpointEnabled:dataEndpointEnabled}' -o json + mkdir -p "$(dirname "$KUBECONFIG")" + az aks get-credentials -g "$AZURE_RESOURCE_GROUP" -n "$AZURE_AKS_CLUSTER_NAME" \ + --admin --file "$KUBECONFIG" --overwrite-existing --only-show-errors + chmod 0600 "$KUBECONFIG" + export KUBECONFIG + kubectl get nodes -o json | jq '{total:(.items|length),ready:([.items[]|select(any(.status.conditions[];.type=="Ready" and .status=="True"))]|length),unschedulable:([.items[]|select(.spec.unschedulable==true)]|length)}' + kubectl get daemonset -A -o json | jq '[.items[]|select(.metadata.name|test("gantry|benchmark"))|{namespace:.metadata.namespace,name:.metadata.name,desired:.status.desiredNumberScheduled,ready:.status.numberReady}]' + exit 0 +fi + +[[ "$DEPLOY_CONFIRM" == "$AZURE_RESOURCE_GROUP" ]] || { + echo "set DEPLOY_CONFIRM=$AZURE_RESOURCE_GROUP to authorize deployment" >&2 + exit 2 +} + +[[ -z "$(git -C "$repo_root" status --porcelain)" ]] || { + echo "deployment requires a clean Git worktree" >&2 + exit 1 +} + +source_revision=$(git -C "$repo_root" rev-parse HEAD) +source_short=$(git -C "$repo_root" rev-parse --short=12 HEAD) + +mkdir -p "$DEPLOY_STATE_DIR" +chmod 0700 "$DEPLOY_STATE_DIR" + +public_restore_needed=false +set_acrs_private() { + for acr in "$BASELINE_ACR_NAME" "$GANTRY_ACR_NAME"; do + if az acr show -g "$AZURE_RESOURCE_GROUP" -n "$acr" --output none >/dev/null 2>&1; then + az acr update -g "$AZURE_RESOURCE_GROUP" -n "$acr" \ + --data-endpoint-enabled true --default-action Deny --public-network-enabled false \ + --only-show-errors -o none + fi + done +} + +restore_private_access() { + local status=$? + if [[ "$public_restore_needed" == true ]]; then + set_acrs_private || true + fi + exit "$status" +} +trap restore_private_access EXIT INT TERM + +assert_equal() { + local description=$1 + local actual=$2 + local expected=$3 + [[ "$actual" == "$expected" ]] || { + echo "$description is $actual, want $expected" >&2 + exit 1 + } +} + +guard_active_benchmark() { + if ! az aks show -g "$AZURE_RESOURCE_GROUP" -n "$AZURE_AKS_CLUSTER_NAME" --output none 2>/dev/null; then + return + fi + + mkdir -p "$(dirname "$KUBECONFIG")" + az aks get-credentials -g "$AZURE_RESOURCE_GROUP" -n "$AZURE_AKS_CLUSTER_NAME" \ + --admin --file "$KUBECONFIG" --overwrite-existing --only-show-errors + chmod 0600 "$KUBECONFIG" + export KUBECONFIG + if kubectl -n "$BENCHMARK_NAMESPACE" get configmap gantry-benchmark-state >/dev/null 2>&1 || + kubectl -n "$GANTRY_NAMESPACE" get configmap gantry-benchmark-lock >/dev/null 2>&1; then + echo "an active benchmark state or lock exists; finish or disable it before deployment" >&2 + exit 1 + fi +} + +ensure_group() { + if [[ $(az group exists -n "$AZURE_RESOURCE_GROUP") == false ]]; then + log "creating resource group $AZURE_RESOURCE_GROUP" + az group create -n "$AZURE_RESOURCE_GROUP" -l "$AZURE_LOCATION" --only-show-errors -o none + fi + assert_equal "resource group location" \ + "$(az group show -n "$AZURE_RESOURCE_GROUP" --query location -o tsv)" "$AZURE_LOCATION" +} + +ensure_vnet() { + if ! az network vnet show -g "$AZURE_RESOURCE_GROUP" -n "$VNET_NAME" --output none 2>/dev/null; then + log "creating VNet $VNET_NAME" + az network vnet create -g "$AZURE_RESOURCE_GROUP" -n "$VNET_NAME" -l "$AZURE_LOCATION" \ + --address-prefixes "$VNET_CIDR" --subnet-name "$AKS_SUBNET_NAME" \ + --subnet-prefixes "$AKS_SUBNET_CIDR" --only-show-errors -o none + fi + local actual_prefix + actual_prefix=$(az network vnet show -g "$AZURE_RESOURCE_GROUP" -n "$VNET_NAME" --query 'addressSpace.addressPrefixes[0]' -o tsv) + assert_equal "VNet prefix" "$actual_prefix" "$VNET_CIDR" + + if ! az network vnet subnet show -g "$AZURE_RESOURCE_GROUP" --vnet-name "$VNET_NAME" -n "$AKS_SUBNET_NAME" --output none 2>/dev/null; then + az network vnet subnet create -g "$AZURE_RESOURCE_GROUP" --vnet-name "$VNET_NAME" \ + -n "$AKS_SUBNET_NAME" --address-prefixes "$AKS_SUBNET_CIDR" --only-show-errors -o none + fi + assert_equal "AKS subnet prefix" \ + "$(az network vnet subnet show -g "$AZURE_RESOURCE_GROUP" --vnet-name "$VNET_NAME" -n "$AKS_SUBNET_NAME" --query addressPrefix -o tsv)" \ + "$AKS_SUBNET_CIDR" + + if ! az network vnet subnet show -g "$AZURE_RESOURCE_GROUP" --vnet-name "$VNET_NAME" -n "$PRIVATE_ENDPOINT_SUBNET_NAME" --output none 2>/dev/null; then + az network vnet subnet create -g "$AZURE_RESOURCE_GROUP" --vnet-name "$VNET_NAME" \ + -n "$PRIVATE_ENDPOINT_SUBNET_NAME" --address-prefixes "$PRIVATE_ENDPOINT_SUBNET_CIDR" \ + --disable-private-endpoint-network-policies true --only-show-errors -o none + fi + assert_equal "Private Endpoint subnet prefix" \ + "$(az network vnet subnet show -g "$AZURE_RESOURCE_GROUP" --vnet-name "$VNET_NAME" -n "$PRIVATE_ENDPOINT_SUBNET_NAME" --query addressPrefix -o tsv)" \ + "$PRIVATE_ENDPOINT_SUBNET_CIDR" + + if ! az network vnet subnet show -g "$AZURE_RESOURCE_GROUP" --vnet-name "$VNET_NAME" -n "$OPERATOR_SUBNET_NAME" --output none 2>/dev/null; then + az network vnet subnet create -g "$AZURE_RESOURCE_GROUP" --vnet-name "$VNET_NAME" \ + -n "$OPERATOR_SUBNET_NAME" --address-prefixes "$OPERATOR_SUBNET_CIDR" \ + --only-show-errors -o none + fi + assert_equal "operator subnet prefix" \ + "$(az network vnet subnet show -g "$AZURE_RESOURCE_GROUP" --vnet-name "$VNET_NAME" -n "$OPERATOR_SUBNET_NAME" --query addressPrefix -o tsv)" \ + "$OPERATOR_SUBNET_CIDR" +} + +ensure_acr() { + local name=$1 + if ! az acr show -g "$AZURE_RESOURCE_GROUP" -n "$name" --output none 2>/dev/null; then + log "creating Premium ACR $name" + az acr create -g "$AZURE_RESOURCE_GROUP" -n "$name" -l "$AZURE_LOCATION" \ + --sku Premium --public-network-enabled false --only-show-errors -o none + fi + assert_equal "$name SKU" "$(az acr show -g "$AZURE_RESOURCE_GROUP" -n "$name" --query sku.name -o tsv)" Premium + assert_equal "$name location" "$(az acr show -g "$AZURE_RESOURCE_GROUP" -n "$name" --query location -o tsv)" "$AZURE_LOCATION" + az acr update -g "$AZURE_RESOURCE_GROUP" -n "$name" \ + --data-endpoint-enabled true --only-show-errors -o none +} + +acr_public_access_enabled() { + local state + state=$(az acr show -g "$AZURE_RESOURCE_GROUP" -n "$GANTRY_ACR_NAME" -o json) + [[ $(jq -r .publicNetworkAccess <<<"$state") == Enabled && + $(jq -r .networkRuleSet.defaultAction <<<"$state") == Allow ]] +} + +build_source_image() { + log "publishing private source carrier from $source_revision" + SOURCE_IMAGE=$GANTRY_ACR_LOGIN_SERVER/gantry-benchmark-source:$source_revision + + public_restore_needed=true + az acr update -g "$AZURE_RESOURCE_GROUP" -n "$GANTRY_ACR_NAME" \ + --default-action Allow --public-network-enabled true --only-show-errors -o none + + retry_command 30 10 acr_public_access_enabled + + local build_log=$DEPLOY_STATE_DIR/source-carrier-build.log + local built=false + local attempt + for attempt in $(seq 1 18); do + if az acr build \ + --registry "$GANTRY_ACR_NAME" \ + --image "gantry-benchmark-source:$source_revision" \ + --file "$repo_root/images/gantry-benchmark-source/Containerfile" \ + --build-arg "SOURCE_REVISION=$source_revision" \ + "$repo_root" --only-show-errors -o none >"$build_log" 2>&1; then + built=true + break + fi + log "source-carrier ACR Task is waiting for firewall propagation ($attempt/18)" + sleep 30 + done + if [[ "$built" != true ]]; then + cat "$build_log" >&2 + return 1 + fi + cat "$build_log" + + set_acrs_private + public_restore_needed=false +} + +ensure_aks() { + local subnet_id + subnet_id=$(az network vnet subnet show -g "$AZURE_RESOURCE_GROUP" --vnet-name "$VNET_NAME" \ + -n "$AKS_SUBNET_NAME" --query id -o tsv) + if ! az aks show -g "$AZURE_RESOURCE_GROUP" -n "$AZURE_AKS_CLUSTER_NAME" --output none 2>/dev/null; then + log "creating AKS cluster $AZURE_AKS_CLUSTER_NAME" + az aks create -g "$AZURE_RESOURCE_GROUP" -n "$AZURE_AKS_CLUSTER_NAME" -l "$AZURE_LOCATION" \ + --tier standard --enable-managed-identity --node-resource-group "$AZURE_NODE_RESOURCE_GROUP" \ + --nodepool-name "$AKS_NODE_POOL_NAME" --node-count "$AKS_NODE_COUNT" \ + --node-vm-size "$AKS_NODE_VM_SIZE" --node-osdisk-type Managed \ + --node-osdisk-size "$AKS_NODE_OS_DISK_GB" --max-pods "$AKS_MAX_PODS" \ + --os-sku Ubuntu --network-plugin azure --network-plugin-mode overlay \ + --network-dataplane azure --pod-cidr "$POD_CIDR" --service-cidr "$SERVICE_CIDR" \ + --dns-service-ip "$DNS_SERVICE_IP" --vnet-subnet-id "$subnet_id" \ + --load-balancer-sku standard --outbound-type loadBalancer \ + --kubernetes-version "$AKS_KUBERNETES_VERSION" --no-ssh-key --only-show-errors -o none + fi + + az aks wait -g "$AZURE_RESOURCE_GROUP" -n "$AZURE_AKS_CLUSTER_NAME" \ + --created --interval 30 --timeout 7200 + + local cluster_json pool_json + cluster_json=$(az aks show -g "$AZURE_RESOURCE_GROUP" -n "$AZURE_AKS_CLUSTER_NAME" -o json) + assert_equal "AKS location" "$(jq -r .location <<<"$cluster_json")" "$AZURE_LOCATION" + assert_equal "AKS Kubernetes version" "$(jq -r .kubernetesVersion <<<"$cluster_json")" "$AKS_KUBERNETES_VERSION" + assert_equal "AKS pod CIDR" "$(jq -r .networkProfile.podCidr <<<"$cluster_json")" "$POD_CIDR" + assert_equal "AKS service CIDR" "$(jq -r .networkProfile.serviceCidr <<<"$cluster_json")" "$SERVICE_CIDR" + assert_equal "AKS node resource group" "$(jq -r .nodeResourceGroup <<<"$cluster_json")" "$AZURE_NODE_RESOURCE_GROUP" + + pool_json=$(az aks nodepool show -g "$AZURE_RESOURCE_GROUP" --cluster-name "$AZURE_AKS_CLUSTER_NAME" \ + -n "$AKS_NODE_POOL_NAME" -o json) + assert_equal "AKS node count" "$(jq -r .count <<<"$pool_json")" "$AKS_NODE_COUNT" + assert_equal "AKS node VM size" "$(jq -r .vmSize <<<"$pool_json")" "$AKS_NODE_VM_SIZE" + assert_equal "AKS max pods" "$(jq -r .maxPods <<<"$pool_json")" "$AKS_MAX_PODS" + assert_equal "AKS node OS disk" "$(jq -r .osDiskSizeGb <<<"$pool_json")" "$AKS_NODE_OS_DISK_GB" + assert_equal "AKS node OS SKU" "$(jq -r .osSku <<<"$pool_json")" Ubuntu + assert_equal "AKS node-pool mode" "$(jq -r .mode <<<"$pool_json")" System + assert_equal "AKS node subnet" "$(jq -r .vnetSubnetId <<<"$pool_json")" "$subnet_id" +} + +ensure_role() { + local principal=$1 + local role=$2 + local scope=$3 + if [[ $(az role assignment list --assignee-object-id "$principal" --scope "$scope" --role "$role" --query 'length(@)' -o tsv) == 0 ]]; then + az role assignment create --assignee-object-id "$principal" \ + --assignee-principal-type ServicePrincipal --role "$role" --scope "$scope" \ + --only-show-errors -o none + fi +} + +ensure_diagnostics() { + local law_id aks_id baseline_id gantry_id + if ! az monitor log-analytics workspace show -g "$AZURE_RESOURCE_GROUP" -n "$AZURE_LOG_ANALYTICS_WORKSPACE_NAME" --output none 2>/dev/null; then + az monitor log-analytics workspace create -g "$AZURE_RESOURCE_GROUP" \ + -n "$AZURE_LOG_ANALYTICS_WORKSPACE_NAME" -l "$AZURE_LOCATION" \ + --retention-time 30 --only-show-errors -o none + fi + assert_equal "Log Analytics location" \ + "$(az monitor log-analytics workspace show -g "$AZURE_RESOURCE_GROUP" -n "$AZURE_LOG_ANALYTICS_WORKSPACE_NAME" --query location -o tsv)" \ + "$AZURE_LOCATION" + law_id=$(az monitor log-analytics workspace show -g "$AZURE_RESOURCE_GROUP" \ + -n "$AZURE_LOG_ANALYTICS_WORKSPACE_NAME" --query id -o tsv) + aks_id=$(az aks show -g "$AZURE_RESOURCE_GROUP" -n "$AZURE_AKS_CLUSTER_NAME" --query id -o tsv) + baseline_id=$(az acr show -g "$AZURE_RESOURCE_GROUP" -n "$BASELINE_ACR_NAME" --query id -o tsv) + gantry_id=$(az acr show -g "$AZURE_RESOURCE_GROUP" -n "$GANTRY_ACR_NAME" --query id -o tsv) + + az monitor diagnostic-settings create --name "${DEPLOYMENT_NAME}-baseline-acr-diag" \ + --resource "$baseline_id" --workspace "$law_id" --export-to-resource-specific true \ + --logs '[{"category":"ContainerRegistryRepositoryEvents","enabled":true},{"category":"ContainerRegistryLoginEvents","enabled":true}]' \ + --metrics '[{"category":"AllMetrics","enabled":true}]' --only-show-errors -o none + az monitor diagnostic-settings create --name "${DEPLOYMENT_NAME}-gantry-acr-diag" \ + --resource "$gantry_id" --workspace "$law_id" --export-to-resource-specific true \ + --logs '[{"category":"ContainerRegistryRepositoryEvents","enabled":true},{"category":"ContainerRegistryLoginEvents","enabled":true}]' \ + --metrics '[{"category":"AllMetrics","enabled":true}]' --only-show-errors -o none + az monitor diagnostic-settings create --name "${DEPLOYMENT_NAME}-aks-diag" \ + --resource "$aks_id" --workspace "$law_id" --export-to-resource-specific true \ + --logs '[{"category":"kube-audit-admin","enabled":true},{"category":"kube-apiserver","enabled":true},{"category":"kube-scheduler","enabled":true}]' \ + --metrics '[{"category":"AllMetrics","enabled":true}]' --only-show-errors -o none +} + +ensure_private_endpoint() { + local name=$1 + local acr_id=$2 + local connection_name=$3 + local subnet_id zone_id zone_group_count + subnet_id=$(az network vnet subnet show -g "$AZURE_RESOURCE_GROUP" --vnet-name "$VNET_NAME" \ + -n "$PRIVATE_ENDPOINT_SUBNET_NAME" --query id -o tsv) + zone_id=$(az network private-dns zone show -g "$AZURE_RESOURCE_GROUP" -n "$PRIVATE_DNS_ZONE" --query id -o tsv) + + if ! az network private-endpoint show -g "$AZURE_RESOURCE_GROUP" -n "$name" --output none 2>/dev/null; then + az network private-endpoint create -g "$AZURE_RESOURCE_GROUP" -n "$name" -l "$AZURE_LOCATION" \ + --subnet "$subnet_id" --private-connection-resource-id "$acr_id" \ + --group-ids registry --connection-name "$connection_name" --only-show-errors -o none + fi + zone_group_count=$(az network private-endpoint dns-zone-group list -g "$AZURE_RESOURCE_GROUP" \ + --endpoint-name "$name" --query 'length(@)' -o tsv) + if [[ "$zone_group_count" == 0 ]]; then + az network private-endpoint dns-zone-group create -g "$AZURE_RESOURCE_GROUP" \ + --endpoint-name "$name" -n acr --private-dns-zone "$zone_id" \ + --zone-name "$PRIVATE_DNS_ZONE" --only-show-errors -o none + fi + assert_equal "$name private DNS zone" \ + "$(az network private-endpoint dns-zone-group show -g "$AZURE_RESOURCE_GROUP" \ + --endpoint-name "$name" -n acr --query 'privateDnsZoneConfigs[0].privateDnsZoneId' -o tsv)" \ + "$zone_id" + assert_equal "$name connection state" \ + "$(az network private-endpoint show -g "$AZURE_RESOURCE_GROUP" -n "$name" --query 'privateLinkServiceConnections[0].privateLinkServiceConnectionState.status' -o tsv)" \ + Approved + assert_equal "$name target resource" \ + "$(az network private-endpoint show -g "$AZURE_RESOURCE_GROUP" -n "$name" --query 'privateLinkServiceConnections[0].privateLinkServiceId' -o tsv)" \ + "$acr_id" + assert_equal "$name subnet" \ + "$(az network private-endpoint show -g "$AZURE_RESOURCE_GROUP" -n "$name" --query subnet.id -o tsv)" \ + "$subnet_id" +} + +ensure_private_network() { + if ! az network private-dns zone show -g "$AZURE_RESOURCE_GROUP" -n "$PRIVATE_DNS_ZONE" --output none 2>/dev/null; then + az network private-dns zone create -g "$AZURE_RESOURCE_GROUP" -n "$PRIVATE_DNS_ZONE" --only-show-errors -o none + fi + local vnet_id + vnet_id=$(az network vnet show -g "$AZURE_RESOURCE_GROUP" -n "$VNET_NAME" --query id -o tsv) + if ! az network private-dns link vnet show -g "$AZURE_RESOURCE_GROUP" -z "$PRIVATE_DNS_ZONE" \ + -n "$PRIVATE_DNS_LINK_NAME" --output none 2>/dev/null; then + az network private-dns link vnet create -g "$AZURE_RESOURCE_GROUP" -z "$PRIVATE_DNS_ZONE" \ + -n "$PRIVATE_DNS_LINK_NAME" -v "$vnet_id" -e false --only-show-errors -o none + fi + assert_equal "private DNS VNet link" \ + "$(az network private-dns link vnet show -g "$AZURE_RESOURCE_GROUP" -z "$PRIVATE_DNS_ZONE" -n "$PRIVATE_DNS_LINK_NAME" --query virtualNetwork.id -o tsv)" \ + "$vnet_id" + + local baseline_id gantry_id + baseline_id=$(az acr show -g "$AZURE_RESOURCE_GROUP" -n "$BASELINE_ACR_NAME" --query id -o tsv) + gantry_id=$(az acr show -g "$AZURE_RESOURCE_GROUP" -n "$GANTRY_ACR_NAME" --query id -o tsv) + ensure_private_endpoint "$BASELINE_PRIVATE_ENDPOINT_NAME" "$baseline_id" "${DEPLOYMENT_NAME}-baseline-acr" + ensure_private_endpoint "$GANTRY_PRIVATE_ENDPOINT_NAME" "$gantry_id" "${DEPLOYMENT_NAME}-gantry-acr" +} + +wait_for_nodes() { + mkdir -p "$(dirname "$KUBECONFIG")" + rm -f "$KUBECONFIG" + az aks get-credentials -g "$AZURE_RESOURCE_GROUP" -n "$AZURE_AKS_CLUSTER_NAME" \ + --admin --file "$KUBECONFIG" --overwrite-existing --only-show-errors + chmod 0600 "$KUBECONFIG" + export KUBECONFIG + + if kubectl -n "$BENCHMARK_NAMESPACE" get configmap gantry-benchmark-state >/dev/null 2>&1 || + kubectl -n "$GANTRY_NAMESPACE" get configmap gantry-benchmark-lock >/dev/null 2>&1; then + echo "an active benchmark state or lock exists; finish or disable it before deployment" >&2 + exit 1 + fi + + local attempt total ready unschedulable + for attempt in $(seq 1 120); do + local nodes + nodes=$(kubectl get nodes -o json) + total=$(jq '.items|length' <<<"$nodes") + ready=$(jq '[.items[]|select(any(.status.conditions[];.type=="Ready" and .status=="True"))]|length' <<<"$nodes") + unschedulable=$(jq '[.items[]|select(.spec.unschedulable==true)]|length' <<<"$nodes") + if [[ "$total" == "$AKS_NODE_COUNT" && "$ready" == "$AKS_NODE_COUNT" && "$unschedulable" == 0 ]]; then + log "AKS nodes ready: $ready/$total" + return + fi + log "waiting for nodes: total=$total ready=$ready unschedulable=$unschedulable" + sleep 30 + done + echo "AKS did not reach $AKS_NODE_COUNT Ready schedulable nodes" >&2 + exit 1 +} + +install_monitoring() { + export KUBECONFIG + local values=$DEPLOY_STATE_DIR/kps-values.yaml + cat >"$values" </dev/null || true) + if [[ -n "$value" ]]; then + printf '%s' "$value" + return + fi + sleep 10 + done + echo "private DNS record $record did not appear" >&2 + exit 1 +} + +install_node_configuration() { + export KUBECONFIG + kubectl create namespace "$GANTRY_NAMESPACE" --dry-run=client -o yaml | kubectl apply -f - + kubectl apply -f "$repo_root/hack/gantry-benchmark/manifests/containerd.yaml" + kubectl -n "$GANTRY_NAMESPACE" rollout status \ + daemonset/gantry-benchmark-containerd-config --timeout=45m + + local baseline_login_ip baseline_data_ip gantry_login_ip gantry_data_ip + baseline_login_ip=$(private_dns_ip "$BASELINE_ACR_NAME") + baseline_data_ip=$(private_dns_ip "$BASELINE_ACR_NAME.$AZURE_LOCATION.data") + gantry_login_ip=$(private_dns_ip "$GANTRY_ACR_NAME") + gantry_data_ip=$(private_dns_ip "$GANTRY_ACR_NAME.$AZURE_LOCATION.data") + + local guard=$DEPLOY_STATE_DIR/acr-private-dns-guard.yaml + cat >"$guard" <"\$temp" + cat >>"\$temp" <<'HOSTS' + # BEGIN GANTRY BENCHMARK ACR PRIVATE DNS + $baseline_login_ip $BASELINE_ACR_LOGIN_SERVER + $baseline_data_ip $BASELINE_ACR_DATA_HOST + $gantry_login_ip $GANTRY_ACR_LOGIN_SERVER + $gantry_data_ip $GANTRY_ACR_DATA_HOST + # END GANTRY BENCHMARK ACR PRIVATE DNS + HOSTS + cat "\$temp" >"\$hosts" + rm "\$temp" + resolvectl flush-caches + exec sleep 2147483647 + readinessProbe: + exec: + command: + - chroot + - /host + - sh + - -c + - | + set -eu + check() { + resolved=\$(getent ahostsv4 "\$1" | awk '{print \$1}' | sort -u) + test "\$resolved" = "\$2" + } + check $BASELINE_ACR_LOGIN_SERVER $baseline_login_ip + check $BASELINE_ACR_DATA_HOST $baseline_data_ip + check $GANTRY_ACR_LOGIN_SERVER $gantry_login_ip + check $GANTRY_ACR_DATA_HOST $gantry_data_ip + initialDelaySeconds: 2 + timeoutSeconds: 10 + periodSeconds: 15 + failureThreshold: 3 + resources: + requests: {cpu: 1m, memory: 4Mi} + limits: {cpu: 20m, memory: 16Mi} + securityContext: + privileged: true + runAsUser: 0 + volumeMounts: + - name: host + mountPath: /host + volumes: + - name: host + hostPath: + path: / + type: Directory +GUARD + kubectl apply -f "$guard" + kubectl -n "$GANTRY_NAMESPACE" rollout status daemonset/gantry-acr-private-dns-guard --timeout=30m +} + +replace_private_pull_tls_nodes() { + local pods_json node provider_id machine_name current_count + local -a nodes + pods_json=$(kubectl -n "$GANTRY_NAMESPACE" get pods \ + -l app.kubernetes.io/name=gantry-baseline-acr-pull-probe -o json) + mapfile -t nodes < <(jq -r '.items[] | + select(any(.status.containerStatuses[]?; ((.state.waiting.message? // "") | contains("TLS handshake timeout")))) | + .spec.nodeName' <<<"$pods_json" | sort -u) + ((${#nodes[@]} > 0)) || return 1 + ((${#nodes[@]} <= BASELINE_PULL_MAX_NODE_REPLACEMENTS)) || { + echo "refusing to replace ${#nodes[@]} nodes with ACR TLS handshake timeouts; limit is $BASELINE_PULL_MAX_NODE_REPLACEMENTS" >&2 + return 1 + } + + for node in "${nodes[@]}"; do + provider_id=$(kubectl get node "$node" -o jsonpath='{.spec.providerID}') + machine_name=$(az aks machine list -g "$AZURE_RESOURCE_GROUP" \ + --cluster-name "$AZURE_AKS_CLUSTER_NAME" --nodepool-name "$AKS_NODE_POOL_NAME" -o json | \ + jq -r --arg resource_id "${provider_id#azure://}" \ + '.[] | select((.properties.resourceId | ascii_downcase) == ($resource_id | ascii_downcase)) | .name') + [[ -n "$machine_name" ]] || { + echo "cannot resolve AKS machine for $node provider ID $provider_id" >&2 + return 1 + } + log "replacing $node (AKS machine $machine_name) after persistent ACR Private Endpoint TLS timeouts" + az aks nodepool delete-machines -g "$AZURE_RESOURCE_GROUP" \ + --cluster-name "$AZURE_AKS_CLUSTER_NAME" -n "$AKS_NODE_POOL_NAME" \ + --machine-names "$machine_name" --only-show-errors -o none + done + + current_count=$(az aks nodepool show -g "$AZURE_RESOURCE_GROUP" \ + --cluster-name "$AZURE_AKS_CLUSTER_NAME" -n "$AKS_NODE_POOL_NAME" --query count -o tsv) + if [[ "$current_count" != "$AKS_NODE_COUNT" ]]; then + log "restoring AKS node pool count from $current_count to $AKS_NODE_COUNT" + az aks nodepool scale -g "$AZURE_RESOURCE_GROUP" --cluster-name "$AZURE_AKS_CLUSTER_NAME" \ + -n "$AKS_NODE_POOL_NAME" --node-count "$AKS_NODE_COUNT" --only-show-errors -o none + fi + + local attempt total ready old_nodes_remaining + for attempt in $(seq 1 180); do + total=$(kubectl get nodes -l "agentpool=$AKS_NODE_POOL_NAME" -o json | jq '.items | length') + ready=$(kubectl get nodes -l "agentpool=$AKS_NODE_POOL_NAME" -o json | \ + jq '[.items[] | select(any(.status.conditions[]; .type == "Ready" and .status == "True"))] | length') + old_nodes_remaining=0 + for node in "${nodes[@]}"; do + kubectl get node "$node" >/dev/null 2>&1 && ((old_nodes_remaining += 1)) + done + if [[ "$total" == "$AKS_NODE_COUNT" && "$ready" == "$AKS_NODE_COUNT" && "$old_nodes_remaining" == 0 ]]; then + break + fi + sleep 10 + done + [[ "$total" == "$AKS_NODE_COUNT" && "$ready" == "$AKS_NODE_COUNT" && "$old_nodes_remaining" == 0 ]] || { + echo "AKS node replacement did not restore $AKS_NODE_COUNT Ready nodes or remove all failed nodes" >&2 + return 1 + } + kubectl -n "$GANTRY_NAMESPACE" rollout status \ + daemonset/gantry-benchmark-containerd-config --timeout=30m + kubectl -n "$GANTRY_NAMESPACE" rollout status \ + daemonset/gantry-acr-private-dns-guard --timeout=30m +} + +verify_private_baseline_pull() { + export KUBECONFIG + local manifest=$DEPLOY_STATE_DIR/baseline-private-pull-probe.yaml + cat >"$manifest" <&2 + return 1 + fi + kubectl -n "$GANTRY_NAMESPACE" delete daemonset gantry-baseline-acr-pull-probe --wait=true +} + +deploy_gantry() { + export KUBECONFIG + local rendered=$DEPLOY_STATE_DIR/gantry-rendered + rm -rf "$rendered" + GOTOOLCHAIN=auto go run "$repo_root/hack/cmd/render-manifests" \ + --templates-dir "$repo_root/deploy/gantry" --output-dir "$rendered" \ + --set "Namespace=$GANTRY_NAMESPACE" --set "Image=$GANTRY_IMAGE" \ + --set "PprofListen=127.0.0.1:6060" + sed -i "s/registry\.example\.com/$GANTRY_ACR_LOGIN_SERVER/g" "$rendered/configmap.yaml" + + kubectl apply -f "$rendered/serviceaccount.yaml" + kubectl apply -f "$rendered/configmap.yaml" + kubectl apply -f "$rendered/node-config.yaml" + kubectl apply -f "$rendered/daemonset.yaml" + kubectl -n "$GANTRY_NAMESPACE" rollout status daemonset/gantry-containerd-config --timeout=30m + kubectl -n "$GANTRY_NAMESPACE" rollout status daemonset/gantry --timeout=45m +} + +provision_operator() { + export AZURE_SUBSCRIPTION_ID AZURE_RESOURCE_GROUP AZURE_AKS_CLUSTER_NAME + export BASELINE_ACR_NAME GANTRY_ACR_NAME AZURE_LOG_ANALYTICS_WORKSPACE_NAME + export OPERATOR_VNET_RESOURCE_GROUP=$AZURE_RESOURCE_GROUP OPERATOR_VNET_NAME=$VNET_NAME + export AZURE_LOCATION OPERATOR_VM_NAME OPERATOR_VM_SIZE OPERATOR_VM_ZONE + export OPERATOR_OS_DISK_GB OPERATOR_BUILD_DISK_GB OPERATOR_BUILD_DISK_SKU + export OPERATOR_BUILD_DISK_IOPS OPERATOR_BUILD_DISK_MBPS OPERATOR_SUBNET_NAME OPERATOR_SUBNET_CIDR + export BENCHMARK_SOURCE_IMAGE=$SOURCE_IMAGE BENCHMARK_SOURCE_REVISION=$source_revision + export BENCHMARK_NODE_COUNT BENCHMARK_IMAGE_SIZE_MIB BENCHMARK_IMAGE_LAYERS + export BENCHMARK_AZURE_TELEMETRY=true BENCHMARK_MINIMUM_BYTE_REDUCTION BENCHMARK_MAXIMUM_LATENCY_RATIO + export ADOPT_BASELINE_IMAGE ADOPT_GANTRY_IMAGE ADOPT_PAYLOAD_SHA256 + AZURE_BASELINE_ACR_PRIVATE_ENDPOINT_RESOURCE_ID=$(az network private-endpoint show \ + -g "$AZURE_RESOURCE_GROUP" -n "$BASELINE_PRIVATE_ENDPOINT_NAME" --query id -o tsv) + AZURE_GANTRY_ACR_PRIVATE_ENDPOINT_RESOURCE_ID=$(az network private-endpoint show \ + -g "$AZURE_RESOURCE_GROUP" -n "$GANTRY_PRIVATE_ENDPOINT_NAME" --query id -o tsv) + export AZURE_BASELINE_ACR_PRIVATE_ENDPOINT_RESOURCE_ID AZURE_GANTRY_ACR_PRIVATE_ENDPOINT_RESOURCE_ID + START_BENCHMARK=false "$repo_root/hack/gantry-benchmark/operator-vm-provision.sh" + + assert_equal "operator VM size" \ + "$(az vm show -g "$AZURE_RESOURCE_GROUP" -n "$OPERATOR_VM_NAME" --query hardwareProfile.vmSize -o tsv)" \ + "$OPERATOR_VM_SIZE" + local public_ip_id + public_ip_id=$(az vm nic list -g "$AZURE_RESOURCE_GROUP" --vm-name "$OPERATOR_VM_NAME" \ + --query '[].ipConfigurations[].publicIPAddress.id | [0]' -o tsv) + [[ -z "$public_ip_id" ]] || { echo "operator VM unexpectedly has public IP resource $public_ip_id" >&2; exit 1; } + local build_disk_name + build_disk_name=${OPERATOR_BUILD_DISK_NAME:-${OPERATOR_VM_NAME}-build} + assert_equal "operator build disk size" \ + "$(az disk show -g "$AZURE_RESOURCE_GROUP" -n "$build_disk_name" --query diskSizeGB -o tsv)" \ + "$OPERATOR_BUILD_DISK_GB" + assert_equal "operator build disk SKU" \ + "$(az disk show -g "$AZURE_RESOURCE_GROUP" -n "$build_disk_name" --query sku.name -o tsv)" \ + "$OPERATOR_BUILD_DISK_SKU" +} + +build_operator_images() { + log "building Gantry and pull-probe images inside the private operator VM" + local output + output=$(az vm run-command invoke -g "$AZURE_RESOURCE_GROUP" -n "$OPERATOR_VM_NAME" \ + --command-id RunShellScript \ + --scripts @"$repo_root/hack/gantry-benchmark/operator-vm-build-images.sh" \ + --parameters "$AZURE_SUBSCRIPTION_ID" "$BASELINE_ACR_NAME" "$GANTRY_ACR_NAME" \ + "$source_revision" "$source_short" \ + --query 'value[0].message' -o tsv) + local result_json + result_json=$(tr -d '\r' <<<"$output" | sed -n 's/^DEPLOYMENT_IMAGES_JSON=//p' | tail -1) + jq -e 'type == "object" and (.gantry_image | type == "string") and (.baseline_probe_image | type == "string")' \ + <<<"$result_json" >/dev/null || { + echo "operator did not return valid deployment image JSON" >&2 + return 1 + } + GANTRY_IMAGE=$(jq -r .gantry_image <<<"$result_json") + BASELINE_PROBE_IMAGE=$(jq -r .baseline_probe_image <<<"$result_json") + [[ "$GANTRY_IMAGE" == "$GANTRY_ACR_LOGIN_SERVER/gantry@sha256:"* ]] || { + echo "operator did not return an immutable Gantry image" >&2 + return 1 + } + [[ "$BASELINE_PROBE_IMAGE" == "$BASELINE_ACR_LOGIN_SERVER/gantry-deploy-probe@sha256:"* ]] || { + echo "operator did not return an immutable baseline probe image" >&2 + return 1 + } +} + +acquire_operator_run_command_lock() { + log "waiting for exclusive operator VM Run Command access" + exec {operator_run_command_lock_fd}>"$OPERATOR_RUN_COMMAND_LOCK" + flock "$operator_run_command_lock_fd" +} + +release_operator_run_command_lock() { + flock -u "$operator_run_command_lock_fd" + exec {operator_run_command_lock_fd}>&- +} + +guard_active_benchmark +ensure_group +ensure_vnet +ensure_acr "$BASELINE_ACR_NAME" +ensure_acr "$GANTRY_ACR_NAME" +build_source_image +ensure_private_network +ensure_aks +ensure_diagnostics + +kubelet_object_id=$(az aks show -g "$AZURE_RESOURCE_GROUP" -n "$AZURE_AKS_CLUSTER_NAME" \ + --query identityProfile.kubeletidentity.objectId -o tsv) +ensure_role "$kubelet_object_id" AcrPull \ + "$(az acr show -g "$AZURE_RESOURCE_GROUP" -n "$BASELINE_ACR_NAME" --query id -o tsv)" +ensure_role "$kubelet_object_id" AcrPull \ + "$(az acr show -g "$AZURE_RESOURCE_GROUP" -n "$GANTRY_ACR_NAME" --query id -o tsv)" + +wait_for_nodes +install_node_configuration +install_monitoring + +set_acrs_private + +acquire_operator_run_command_lock +provision_operator +build_operator_images +release_operator_run_command_lock +verify_private_baseline_pull +deploy_gantry + +log "validating final deployment" +assert_equal "baseline ACR public access" \ + "$(az acr show -g "$AZURE_RESOURCE_GROUP" -n "$BASELINE_ACR_NAME" --query publicNetworkAccess -o tsv)" Disabled +assert_equal "Gantry ACR public access" \ + "$(az acr show -g "$AZURE_RESOURCE_GROUP" -n "$GANTRY_ACR_NAME" --query publicNetworkAccess -o tsv)" Disabled + +kubectl -n "$MONITORING_NAMESPACE" get endpoints "$PROMETHEUS_SERVICE" -o json | \ + jq -e '.subsets | any(.addresses | length > 0)' >/dev/null +for daemonset in gantry-benchmark-containerd-config gantry-acr-private-dns-guard gantry-containerd-config gantry; do + namespace=$GANTRY_NAMESPACE + desired=$(kubectl -n "$namespace" get daemonset "$daemonset" -o jsonpath='{.status.desiredNumberScheduled}') + ready=$(kubectl -n "$namespace" get daemonset "$daemonset" -o jsonpath='{.status.numberReady}') + assert_equal "$daemonset readiness" "$ready" "$desired" +done + +if [[ "$START_BENCHMARK" == true ]]; then + log "starting benchmark operator service" + acquire_operator_run_command_lock + az vm run-command invoke -g "$AZURE_RESOURCE_GROUP" -n "$OPERATOR_VM_NAME" \ + --command-id RunShellScript \ + --scripts 'systemctl reset-failed gantry-benchmark-operator.service; systemctl start --no-block gantry-benchmark-operator.service' \ + --only-show-errors -o none + release_operator_run_command_lock +fi + +trap - EXIT INT TERM +log "deployment complete" +print_plan +cat <&2 + exit 1 + fi + else + [ -e "$main_config.gantry-benchmark-backup" ] || \ + cp "$main_config" "$main_config.gantry-benchmark-backup" + config_temp=/host/etc/containerd/.config.toml.gantry-benchmark.tmp + { + echo '# Added by the Gantry benchmark.' + echo 'imports = ["/etc/containerd/conf.d/*.toml"]' + cat "$main_config" + } >"$config_temp" + chmod 0644 "$config_temp" + mv "$config_temp" "$main_config" + fi + + if [ ! -f "$marker" ] || [ "$(cat "$marker")" != "$desired_sha" ]; then + echo "$desired_sha" >"$marker" + chroot /host systemd-run \ + --unit="gantry-benchmark-containerd-restart-$(date +%s)" \ + --no-block systemctl restart containerd + fi + + exec sleep 2147483647 + readinessProbe: + exec: + command: + - sh + - -c + - | + set -eu + cmp -s /config/benchmark.toml /host/etc/containerd/conf.d/97-gantry-benchmark.toml + config_dump="$(chroot /host sh -c ' + pid=$(systemctl show --property MainPID --value containerd) + bin=$(readlink -f "/proc/${pid}/exe") + "$bin" config dump + ')" + printf '%s\n' "$config_dump" | grep -Eq "^[[:space:]]*level = ['\"]debug['\"]$" + printf '%s\n' "$config_dump" | grep -Eq "^[[:space:]]*image_pull_progress_timeout = ['\"]15m(0s)?['\"]$" + printf '%s\n' "$config_dump" | grep -Eq '^[[:space:]]*max_concurrent_downloads = 6$' + initialDelaySeconds: 10 + periodSeconds: 10 + failureThreshold: 30 + resources: + requests: + cpu: 5m + memory: 8Mi + limits: + cpu: 100m + memory: 32Mi + securityContext: + privileged: true + runAsUser: 0 + volumeMounts: + - name: host + mountPath: /host + - name: config + mountPath: /config + readOnly: true + volumes: + - name: host + hostPath: + path: / + type: Directory + - name: config + configMap: + name: gantry-benchmark-containerd-config \ No newline at end of file diff --git a/hack/gantry-benchmark/manifests/monitoring.yaml.tmpl b/hack/gantry-benchmark/manifests/monitoring.yaml.tmpl index b1f121124..41bc0370a 100644 --- a/hack/gantry-benchmark/manifests/monitoring.yaml.tmpl +++ b/hack/gantry-benchmark/manifests/monitoring.yaml.tmpl @@ -10,6 +10,7 @@ metadata: namespace: {{ .Namespace }} labels: release: {{ .MonitoringLabel }} + gantry_benchmark: "true" app.kubernetes.io/part-of: gantry-benchmark spec: namespaceSelector: @@ -28,7 +29,245 @@ spec: - action: keep sourceLabels: - __name__ - regex: gantry_storage_mode_info|p2p_dht_health_score|gantry_peer_serve_bytes_total|gantry_origin_bytes_total|p2p_origin_pull_total|p2p_peer_fetch_total|p2p_origin_pull_success_total|p2p_origin_fallback_total|p2p_in_flight_pulls|gantry_peer_fetch_bytes_total|gantry_mirror_bytes_served_total + regex: process_cpu_seconds_total|gantry_storage_mode_info|p2p_dht_health_score|p2p_dht_lookup_total|p2p_dht_lookup_duration_seconds_(bucket|sum|count)|gantry_peer_serve_bytes_total|gantry_origin_bytes_total|p2p_origin_pull_total|p2p_peer_fetch_total|gantry_peer_fetch_last_timestamp_seconds|p2p_peer_fetch_duration_seconds_(bucket|sum|count)|p2p_origin_pull_success_total|p2p_origin_fallback_total|p2p_in_flight_pulls|gantry_peer_fetch_bytes_total|gantry_mirror_bytes_served_total|gantry_mirror_response_completed_timestamp_seconds|gantry_layer_download_completed_timestamp_seconds|gantry_containerd_commit_observed_total|gantry_containerd_commit_observed_timestamp_seconds|gantry_containerd_commit_observation_duration_seconds_(bucket|sum|count)|gantry_containerd_commit_latest_observation_duration_seconds|gantry_containerd_commit_missing_after_stream_total|p2p_coord_please_pull_(served|started|declined)_total|p2p_coord_pull_intent_served_total|p2p_prefetch_(batches|digests|groups)_total|p2p_prefetch_pullers_per_manifest_(bucket|sum|count) + - action: replace + targetLabel: gantry_benchmark + replacement: "true" +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: gantry-benchmark-node-observer + namespace: {{ .Namespace }} + labels: + app.kubernetes.io/name: gantry-benchmark-node-observer + app.kubernetes.io/part-of: gantry-benchmark +spec: + selector: + matchLabels: + app.kubernetes.io/name: gantry-benchmark-node-observer + template: + metadata: + labels: + app.kubernetes.io/name: gantry-benchmark-node-observer + app.kubernetes.io/part-of: gantry-benchmark + spec: + hostNetwork: true + hostPID: true + dnsPolicy: ClusterFirstWithHostNet + nodeSelector: + kubernetes.io/os: {{ .NodeOS }} + kubernetes.io/arch: {{ .NodeArch }} + tolerations: + - operator: Exists + containers: + - name: node-exporter + image: quay.io/prometheus/node-exporter:v1.9.1 + args: + - --path.procfs=/host/proc + - --path.sysfs=/host/sys + - --path.rootfs=/host/root + - --web.listen-address=:29100 + - --collector.filesystem.mount-points-exclude=^/(dev|proc|sys|var/lib/kubelet/pods/.+)($|/) + - --collector.textfile.directory=/textfile + ports: + - name: node-metrics + containerPort: 29100 + resources: + requests: + cpu: 10m + memory: 24Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + runAsNonRoot: true + runAsUser: 65534 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumeMounts: + - name: proc + mountPath: /host/proc + readOnly: true + - name: sys + mountPath: /host/sys + readOnly: true + - name: root + mountPath: /host/root + readOnly: true + - name: progress + mountPath: /textfile + readOnly: true + - name: containerd-metrics-target + image: mcr.microsoft.com/cbl-mariner/busybox:2.0 + command: ["sh", "-c", "exec sleep 2147483647"] + ports: + - name: ctr-metrics + containerPort: 10257 + resources: + requests: + cpu: 1m + memory: 4Mi + limits: + cpu: 20m + memory: 16Mi + securityContext: + runAsNonRoot: true + runAsUser: 65532 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + - name: containerd-journal + image: mcr.microsoft.com/cbl-mariner/busybox:2.0 + command: + - sh + - -c + - | + set -eu + + textfile="${TEXTFILE_DIR:-/textfile}" + current_image_file="${textfile}/current-image" + mkdir -p "${textfile}" + rm -f "${textfile}"/*.tmp + + containerd_pid="$(chroot /host systemctl show --property MainPID --value containerd)" + containerd_bin="$(chroot /host readlink -f "/proc/${containerd_pid}/exe")" + if [ -z "${containerd_bin}" ] || ! chroot /host "${containerd_bin}" config dump 2>/dev/null | grep -Eq "^[[:space:]]*level = ['\"]debug['\"]$"; then + echo "containerd debug logging is required for unpack timing capture" >&2 + exit 1 + fi + + chroot /host journalctl -f -n 0 -u containerd -o cat | while IFS= read -r line; do + case "${line}" in + *PullImage*gantry-benchmark-pull*) + image="$(printf '%s\n' "${line}" | sed -n 's/.*image="\([^"]*\)".*/\1/p')" + if [ -n "${image}" ]; then + previous="$(cat "${current_image_file}" 2>/dev/null || true)" + if [ "${previous}" != "${image}" ]; then + rm -f "${textfile}"/layer-*.prom "${textfile}"/image-*.prom + printf '%s\n' "${image}" >"${current_image_file}.tmp" + mv "${current_image_file}.tmp" "${current_image_file}" + fi + + output="${textfile}/image-start.prom" + if [ ! -f "${output}" ]; then + timestamp="$(date +%s)" + { + echo '# TYPE gantry_benchmark_image_unpack_started_timestamp_seconds gauge' + printf 'gantry_benchmark_image_unpack_started_timestamp_seconds{node="%s",image="%s"} %s\n' "${NODE_NAME}" "${image}" "${timestamp}" + } >"${output}.tmp" + mv "${output}.tmp" "${output}" + fi + fi + printf '%s\n' "${line}" + ;; + *'layer unpacked'*) + digest="$(printf '%s\n' "${line}" | sed -n 's/.*layer=\(sha256:[0-9a-f]*\).*/\1/p')" + image="$(cat "${current_image_file}" 2>/dev/null || true)" + if [ -n "${digest}" ] && [ -n "${image}" ]; then + timestamp="$(date +%s)" + output="${textfile}/layer-${digest#sha256:}.prom" + { + echo '# TYPE gantry_benchmark_layer_unpacked_timestamp_seconds gauge' + printf 'gantry_benchmark_layer_unpacked_timestamp_seconds{node="%s",image="%s",layer_digest="%s"} %s\n' "${NODE_NAME}" "${image}" "${digest}" "${timestamp}" + } >"${output}.tmp" + mv "${output}.tmp" "${output}" + fi + printf '%s\n' "${line}" + ;; + *'image unpacked'*) + image="$(cat "${current_image_file}" 2>/dev/null || true)" + if [ -n "${image}" ]; then + timestamp="$(date +%s)" + output="${textfile}/image-complete.prom" + { + echo '# TYPE gantry_benchmark_image_unpacked_timestamp_seconds gauge' + printf 'gantry_benchmark_image_unpacked_timestamp_seconds{node="%s",image="%s"} %s\n' "${NODE_NAME}" "${image}" "${timestamp}" + } >"${output}.tmp" + mv "${output}.tmp" "${output}" + fi + printf '%s\n' "${line}" + ;; + *'Pulled image'*|*'cancel pulling image'*) + printf '%s\n' "${line}" + ;; + esac + done + resources: + requests: + cpu: 2m + memory: 8Mi + limits: + cpu: 50m + memory: 32Mi + securityContext: + privileged: true + runAsUser: 0 + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: host + mountPath: /host + readOnly: true + - name: progress + mountPath: /textfile + volumes: + - name: proc + hostPath: + path: /proc + type: Directory + - name: sys + hostPath: + path: /sys + type: Directory + - name: root + hostPath: + path: / + type: Directory + - name: host + hostPath: + path: / + type: Directory + - name: progress + emptyDir: {} +--- +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + name: gantry-benchmark-node-observer + namespace: {{ .Namespace }} + labels: + release: {{ .MonitoringLabel }} + gantry_benchmark: "true" + app.kubernetes.io/part-of: gantry-benchmark +spec: + selector: + matchLabels: + app.kubernetes.io/name: gantry-benchmark-node-observer + podMetricsEndpoints: + - port: node-metrics + path: /metrics + interval: 10s + metricRelabelings: + - action: keep + sourceLabels: [__name__] + regex: node_uname_info|node_cpu_seconds_total|node_memory_(MemAvailable|MemTotal)_bytes|node_disk_(read|written)_bytes_total|node_disk_io_time_seconds_total|node_disk_io_time_weighted_seconds_total|node_disk_reads_completed_total|node_disk_writes_completed_total|node_filesystem_(avail|size)_bytes|node_network_speed_bytes|node_network_(receive|transmit)_(bytes|drop|errs)_total|gantry_benchmark_(image_unpack_started|image_unpacked|layer_unpacked)_timestamp_seconds + - action: replace + targetLabel: gantry_benchmark + replacement: "true" + - port: ctr-metrics + path: /v1/metrics + interval: 10s + metricRelabelings: + - action: keep + sourceLabels: [__name__] + regex: containerd_.*|grpc_server_.*|process_(cpu_seconds_total|resident_memory_bytes|virtual_memory_bytes) - action: replace targetLabel: gantry_benchmark replacement: "true" diff --git a/hack/gantry-benchmark/operator-vm-bootstrap.sh b/hack/gantry-benchmark/operator-vm-bootstrap.sh index 1cc7b0d4c..829e1fed1 100755 --- a/hack/gantry-benchmark/operator-vm-bootstrap.sh +++ b/hack/gantry-benchmark/operator-vm-bootstrap.sh @@ -10,11 +10,12 @@ Usage: operator-vm-bootstrap.sh \ \ \ \ - + \ + USAGE } -[[ $# -eq 18 ]] || { usage >&2; exit 2; } +[[ $# -eq 23 ]] || { usage >&2; exit 2; } subscription_id=$1 resource_group=$2 @@ -34,6 +35,23 @@ minimum_byte_reduction=${15} maximum_latency_ratio=${16} build_disk_lun=${17} build_mount=${18} +source_image=${19} +source_revision=${20} +adopt_baseline_image=${21} +adopt_gantry_image=${22} +adopt_payload_sha256=${23} +[[ "$adopt_baseline_image" != - ]] || adopt_baseline_image="" +[[ "$adopt_gantry_image" != - ]] || adopt_gantry_image="" +[[ "$adopt_payload_sha256" != - ]] || adopt_payload_sha256="" + +adoption_values=0 +for value in "$adopt_baseline_image" "$adopt_gantry_image" "$adopt_payload_sha256"; do + [[ -z "$value" ]] || adoption_values=$((adoption_values + 1)) +done +if ((adoption_values != 0 && adoption_values != 3)); then + echo "adopted baseline image, Gantry image, and payload digest must be set together" >&2 + exit 2 +fi retry() { local attempts=0 @@ -50,6 +68,64 @@ retry() { done } +acr_access_token() { + local acr_name=$1 + local attempts=0 + local maximum=60 + local token + + until token=$(az acr login --name "$acr_name" --expose-token --query accessToken -o tsv); do + attempts=$((attempts + 1)) + if ((attempts >= maximum)); then + echo "failed to obtain managed-identity ACR token for $acr_name after $attempts attempts" >&2 + return 1 + fi + sleep 10 + done + + printf '%s' "$token" +} + +private_dns_ip() { + local record_name=$1 + local attempts=0 + local maximum=60 + local ip + + until ip=$(az network private-dns record-set a show \ + --resource-group "$resource_group" \ + --zone-name privatelink.azurecr.io \ + --name "$record_name" \ + --query 'aRecords[0].ipv4Address' \ + --output tsv 2>/dev/null) && [[ -n "$ip" ]]; do + attempts=$((attempts + 1)) + if ((attempts >= maximum)); then + echo "private DNS record $record_name did not become readable" >&2 + return 1 + fi + sleep 10 + done + + printf '%s' "$ip" +} + +require_private_resolution() { + local host=$1 + local expected_ip=$2 + local attempts=0 + local maximum=60 + + local resolved + until resolved=$(getent ahostsv4 "$host" | awk '{print $1}' | sort -u) && [[ "$resolved" == "$expected_ip" ]]; do + attempts=$((attempts + 1)) + if ((attempts >= maximum)); then + echo "$host did not resolve to private IP $expected_ip" >&2 + return 1 + fi + sleep 10 + done +} + export DEBIAN_FRONTEND=noninteractive apt-get update apt-get install -y ca-certificates curl e2fsprogs git gnupg jq make podman golang-go @@ -62,6 +138,9 @@ if ! command -v kubectl >/dev/null 2>&1; then az aks install-cli --install-location /usr/local/bin/kubectl fi +retry az login --identity --allow-no-subscriptions --output none +az account set --subscription "$subscription_id" + build_device="/dev/disk/azure/scsi1/lun${build_disk_lun}" retry test -b "$build_device" @@ -92,8 +171,58 @@ install -d -m 0700 /var/lib/gantry-benchmark install -d -m 0750 /etc/gantry-benchmark install -d -m 0750 /var/log/gantry-benchmark +retry az acr show -g "$resource_group" -n "$baseline_acr_name" --output none +retry az acr show -g "$resource_group" -n "$gantry_acr_name" --output none +for acr_name in "$baseline_acr_name" "$gantry_acr_name"; do + public_access=$(az acr show -g "$resource_group" -n "$acr_name" --query publicNetworkAccess -o tsv) + data_endpoint=$(az acr show -g "$resource_group" -n "$acr_name" --query dataEndpointEnabled -o tsv) + [[ "$public_access" == Disabled ]] || { echo "$acr_name public access is $public_access, want Disabled" >&2; exit 1; } + [[ "$data_endpoint" == true ]] || { echo "$acr_name dedicated data endpoint is not enabled" >&2; exit 1; } +done + +baseline_location=$(az acr show -g "$resource_group" -n "$baseline_acr_name" --query location -o tsv) +gantry_location=$(az acr show -g "$resource_group" -n "$gantry_acr_name" --query location -o tsv) +baseline_login_ip=$(private_dns_ip "$baseline_acr_name") +baseline_data_ip=$(private_dns_ip "$baseline_acr_name.$baseline_location.data") +gantry_login_ip=$(private_dns_ip "$gantry_acr_name") +gantry_data_ip=$(private_dns_ip "$gantry_acr_name.$gantry_location.data") +require_private_resolution "$baseline_acr_name.azurecr.io" "$baseline_login_ip" +require_private_resolution "$baseline_acr_name.$baseline_location.data.azurecr.io" "$baseline_data_ip" +require_private_resolution "$gantry_acr_name.azurecr.io" "$gantry_login_ip" +require_private_resolution "$gantry_acr_name.$gantry_location.data.azurecr.io" "$gantry_data_ip" + repo_root="$build_mount/unbounded" -if [[ -d "$repo_root/.git" ]]; then +source_description="$repo_url ($repo_branch)" +for lifecycle_service in gantry-benchmark-operator.service gantry-benchmark-image-builder.service; do + if systemctl is-active --quiet "$lifecycle_service"; then + echo "$lifecycle_service is active; finish it before refreshing the operator checkout" >&2 + exit 1 + fi +done +if [[ -n "$source_image" ]]; then + gantry_login_server=$(az acr show -g "$resource_group" -n "$gantry_acr_name" --query loginServer -o tsv) + source_token=$(acr_access_token "$gantry_acr_name") + printf '%s' "$source_token" | podman login "$gantry_login_server" \ + --username 00000000-0000-0000-0000-000000000000 \ + --password-stdin + unset source_token + + podman pull "$source_image" + actual_source_revision=$(podman image inspect \ + --format '{{ index .Labels "org.opencontainers.image.revision" }}' \ + "$source_image") + if [[ -n "$source_revision" && "$actual_source_revision" != "$source_revision" ]]; then + echo "source image revision $actual_source_revision, want $source_revision" >&2 + exit 1 + fi + source_container=$(podman create "$source_image") + rm -rf "$repo_root" + install -d -m 0755 "$repo_root" + podman cp "$source_container:/workspace/." "$repo_root/" + podman rm "$source_container" + podman logout "$gantry_login_server" + source_description="$source_image ($actual_source_revision)" +elif [[ -d "$repo_root/.git" ]]; then git -C "$repo_root" fetch origin "$repo_branch" git -C "$repo_root" checkout -B "$repo_branch" "origin/$repo_branch" else @@ -109,9 +238,6 @@ podman_graph_root=$(podman info --format '{{.Store.GraphRoot}}') exit 1 } -retry az login --identity --allow-no-subscriptions --output none -az account set --subscription "$subscription_id" - retry az acr show -g "$resource_group" -n "$baseline_acr_name" --output none retry az acr show -g "$resource_group" -n "$gantry_acr_name" --output none retry az aks show -g "$resource_group" -n "$aks_cluster" --output none @@ -122,6 +248,27 @@ aks_id=$(az aks show -g "$resource_group" -n "$aks_cluster" --query id -o tsv) baseline_login_server=$(az acr show -g "$resource_group" -n "$baseline_acr_name" --query loginServer -o tsv) gantry_login_server=$(az acr show -g "$resource_group" -n "$gantry_acr_name" --query loginServer -o tsv) +valid_adopted_image() { + local image=$1 + local login_server=$2 + local prefix="$login_server/gantry-benchmark-pull@" + [[ "$image" == "$prefix"* && "${image#"$prefix"}" =~ ^sha256:[0-9a-f]{64}$ ]] +} +if ((adoption_values == 3)); then + valid_adopted_image "$adopt_baseline_image" "$baseline_login_server" || { + echo "adopted baseline image is not an immutable gantry-benchmark-pull image in $baseline_login_server" >&2 + exit 2 + } + valid_adopted_image "$adopt_gantry_image" "$gantry_login_server" || { + echo "adopted Gantry image is not an immutable gantry-benchmark-pull image in $gantry_login_server" >&2 + exit 2 + } + [[ "$adopt_payload_sha256" =~ ^sha256:[0-9a-f]{64}$ ]] || { + echo "adopted payload fingerprint must be a sha256 digest" >&2 + exit 2 + } +fi + cat >/etc/gantry-benchmark/env </etc/systemd/system/gantry-benchmark-image-builder.service </etc/gantry-benchmark/image-pool.env <<'ENV' +GANTRY_IMAGE_POOL_COUNT="1" +ENV +fi + systemctl daemon-reload retry az aks get-credentials \ @@ -214,7 +397,7 @@ echo "Gantry ACR status: $gantry_status" cat < " >&2 + exit 2 +fi + +subscription_id=$1 +baseline_acr=$2 +gantry_acr=$3 +source_revision=$4 +source_short=$5 +baseline_login="${baseline_acr}.azurecr.io" +gantry_login="${gantry_acr}.azurecr.io" +log_file=/var/log/gantry-benchmark/deployment-images.log + +. /etc/gantry-benchmark/env +export HOME="${BENCHMARK_OPERATOR_HOME:-/var/lib/gantry-benchmark}" +cd "$BENCHMARK_REPO_ROOT" + +az login --identity --allow-no-subscriptions --output none >>"$log_file" 2>&1 +az account set --subscription "$subscription_id" + +image_state=/var/lib/gantry-benchmark/deployment-images.env +if [[ -f "$image_state" ]]; then + recorded_revision=$(sed -n "s/^SOURCE_REVISION='\([^']*\)'$/\1/p" "$image_state") + if [[ "$recorded_revision" == "$source_revision" ]]; then + # shellcheck source=/dev/null + . "$image_state" + if [[ "$GANTRY_IMAGE" == "$gantry_login/gantry@sha256:"* && + "$BASELINE_PROBE_IMAGE" == "$baseline_login/gantry-deploy-probe@sha256:"* ]]; then + jq -cn --arg gantry_image "$GANTRY_IMAGE" --arg baseline_probe_image "$BASELINE_PROBE_IMAGE" \ + '{gantry_image:$gantry_image,baseline_probe_image:$baseline_probe_image}' | \ + sed 's/^/DEPLOYMENT_IMAGES_JSON=/' + exit 0 + fi + fi +fi + +registry_login() { + local acr=$1 + local login=$2 + local token + local attempt + + for attempt in $(seq 1 18); do + if token=$(az acr login --name "$acr" --expose-token --query accessToken -o tsv 2>>"$log_file"); then + printf '%s' "$token" | podman login "$login" \ + --username 00000000-0000-0000-0000-000000000000 \ + --password-stdin >>"$log_file" 2>&1 + unset token + return + fi + sleep 10 + done + + echo "failed to authenticate to $login" >&2 + return 1 +} + +registry_login "$gantry_acr" "$gantry_login" +gantry_tag="$gantry_login/gantry:benchmark-$source_short" +podman build --isolation chroot --platform linux/amd64 \ + --build-arg "VERSION=benchmark-$source_short" \ + --build-arg "GIT_COMMIT=$source_revision" \ + --tag "$gantry_tag" --file images/gantry/Containerfile . >>"$log_file" 2>&1 +gantry_digest_file=/var/lib/gantry-benchmark/gantry-deploy.digest +podman push --digestfile "$gantry_digest_file" "$gantry_tag" >>"$log_file" 2>&1 +gantry_digest=$(tr -d '[:space:]' <"$gantry_digest_file") +podman logout "$gantry_login" >>"$log_file" 2>&1 + +registry_login "$baseline_acr" "$baseline_login" +podman pull mcr.microsoft.com/cbl-mariner/busybox:2.0 >>"$log_file" 2>&1 +probe_tag="$baseline_login/gantry-deploy-probe:$source_revision" +podman tag mcr.microsoft.com/cbl-mariner/busybox:2.0 "$probe_tag" +probe_digest_file=/var/lib/gantry-benchmark/baseline-probe.digest +podman push --digestfile "$probe_digest_file" "$probe_tag" >>"$log_file" 2>&1 +probe_digest=$(tr -d '[:space:]' <"$probe_digest_file") +podman logout "$baseline_login" >>"$log_file" 2>&1 + +GANTRY_IMAGE="$gantry_login/gantry@$gantry_digest" +BASELINE_PROBE_IMAGE="$baseline_login/gantry-deploy-probe@$probe_digest" +cat >"$image_state" <&2 + return 1 + fi + + if ((remote_status != 0)); then + echo "operator VM command failed with exit code $remote_status" >&2 + return "$remote_status" + fi +} + +(($# >= 1)) || { usage >&2; exit 2; } +action=$1 +shift + +: "${AZURE_RESOURCE_GROUP:?Set AZURE_RESOURCE_GROUP}" + +exec {run_command_lock_fd}>"$OPERATOR_RUN_COMMAND_LOCK" +flock "$run_command_lock_fd" +trap 'flock -u "$run_command_lock_fd"' EXIT + +case "$action" in + start) + (($# == 1)) || { usage >&2; exit 2; } + count=$1 + [[ "$count" =~ ^[1-9][0-9]*$ ]] || { echo "COUNT must be a positive integer" >&2; exit 2; } + ((count <= 100)) || { echo "COUNT must not exceed 100" >&2; exit 2; } + + script=$(cat <