diff --git a/agent/cmd/main.go b/agent/cmd/main.go index 3900c18f2..805b26e93 100644 --- a/agent/cmd/main.go +++ b/agent/cmd/main.go @@ -244,6 +244,13 @@ func main() { ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) defer cancel() + usageReader := usage.New(usage.Config{ + DataDir: dataRoot, + ProcMode: envOr("GAMEPLANE_USAGE_PROC", "") == "1", + CPULimitMillicores: envInt("GAMEPLANE_CPU_LIMIT_MILLICORES"), + MemLimitBytes: envInt("GAMEPLANE_MEM_LIMIT_BYTES"), + }) + go usageReader.MonitorDisk(ctx) // Prometheus metrics for per-server resource usage and player counts. // Registered with the default registry so promhttp.Handler() picks it up. @@ -268,12 +275,7 @@ func main() { } return nil }(), - Usage: usage.New(usage.Config{ - DataDir: dataRoot, - ProcMode: envOr("GAMEPLANE_USAGE_PROC", "") == "1", - CPULimitMillicores: envInt("GAMEPLANE_CPU_LIMIT_MILLICORES"), - MemLimitBytes: envInt("GAMEPLANE_MEM_LIMIT_BYTES"), - }), + Usage: usageReader, Metrics: m, }) diff --git a/agent/internal/heartbeat/heartbeat.go b/agent/internal/heartbeat/heartbeat.go index 5419dd6ff..3de616703 100644 --- a/agent/internal/heartbeat/heartbeat.go +++ b/agent/internal/heartbeat/heartbeat.go @@ -187,7 +187,7 @@ func sendOnce(ctx context.Context, dyn dynamic.Interface, cfg Config) error { agent["memoryBytes"] = nullable(s.MemoryBytes, s.MemoryKnown) agent["memoryLimitBytes"] = nullable(s.MemoryLimitBytes, s.MemoryLimitKnown) agent["diskUsedBytes"] = nullable(s.DiskUsedBytes, s.DiskKnown) - agent["diskTotalBytes"] = nullable(s.DiskTotalBytes, s.DiskKnown) + agent["diskTotalBytes"] = nullable(s.DiskTotalBytes, s.DiskTotalKnown) // Update metrics with usage data. if cfg.Metrics != nil { metricsSnap.CPUMillicores = s.CPUMillicores @@ -199,8 +199,9 @@ func sendOnce(ctx context.Context, dyn dynamic.Interface, cfg Config) error { metricsSnap.MemoryLimitBytes = s.MemoryLimitBytes metricsSnap.MemoryLimitKnown = s.MemoryLimitKnown metricsSnap.DiskUsedBytes = s.DiskUsedBytes - metricsSnap.DiskTotalBytes = s.DiskTotalBytes metricsSnap.DiskKnown = s.DiskKnown + metricsSnap.DiskTotalKnown = s.DiskTotalKnown + metricsSnap.DiskTotalBytes = s.DiskTotalBytes } } diff --git a/agent/internal/heartbeat/heartbeat_test.go b/agent/internal/heartbeat/heartbeat_test.go index 7322f87f2..531075f07 100644 --- a/agent/internal/heartbeat/heartbeat_test.go +++ b/agent/internal/heartbeat/heartbeat_test.go @@ -155,7 +155,7 @@ func TestSendOnce_EmitsKnownUsage(t *testing.T) { CPULimitMillicores: 2000, CPULimitKnown: true, MemoryBytes: 1024, MemoryKnown: true, MemoryLimitBytes: 4096, MemoryLimitKnown: true, - DiskUsedBytes: 10, DiskTotalBytes: 100, DiskKnown: true, + DiskUsedBytes: 10, DiskKnown: true, DiskTotalBytes: 100, DiskTotalKnown: true, }}, }) if err != nil { @@ -452,8 +452,8 @@ func TestSendOnce_PopulatesMetricsSnapshot_PlayersAndUsage(t *testing.T) { MemoryLimitBytes: 4096, MemoryLimitKnown: true, DiskUsedBytes: 100, - DiskTotalBytes: 1000, - DiskKnown: true, + DiskTotalBytes: 1000, DiskTotalKnown: true, + DiskKnown: true, }}, Metrics: metricsObj, } diff --git a/agent/internal/metrics/metrics.go b/agent/internal/metrics/metrics.go index e8ed49142..15ad3d46e 100644 --- a/agent/internal/metrics/metrics.go +++ b/agent/internal/metrics/metrics.go @@ -28,6 +28,7 @@ type Snapshot struct { // Disk bytes: used and total. Only emitted when known=true. DiskUsedBytes int64 DiskTotalBytes int64 + DiskTotalKnown bool DiskKnown bool // Player counts. PlayersOnline is emitted when known=true. @@ -111,6 +112,8 @@ func (m *Metrics) Collect(ch chan<- prometheus.Metric) { // Disk: emit only when known. if snap.DiskKnown { ch <- prometheus.MustNewConstMetric(diskUsedDesc, prometheus.GaugeValue, float64(snap.DiskUsedBytes), labels...) + } + if snap.DiskTotalKnown { ch <- prometheus.MustNewConstMetric(diskTotalDesc, prometheus.GaugeValue, float64(snap.DiskTotalBytes), labels...) } @@ -155,7 +158,7 @@ var ( ) diskTotalDesc = prometheus.NewDesc( "gameplane_agent_disk_total_bytes", - "Total disk capacity of the game's data directory, in bytes. Emitted only when readable.", + "Observed capacity scoped to the game's data directory, in bytes. Emitted only when available.", []string{"server", "namespace", "template", "game"}, nil, ) playersOnlineDesc = prometheus.NewDesc( diff --git a/agent/internal/metrics/metrics_test.go b/agent/internal/metrics/metrics_test.go index dc94e8dbe..829a39237 100644 --- a/agent/internal/metrics/metrics_test.go +++ b/agent/internal/metrics/metrics_test.go @@ -24,6 +24,7 @@ func TestMetricsEmitsWhenKnown(t *testing.T) { MemoryLimitKnown: true, DiskUsedBytes: 1024 * 1024 * 100, // 100 MiB DiskTotalBytes: 1024 * 1024 * 1024, // 1 GiB + DiskTotalKnown: true, DiskKnown: true, PlayersOnline: 5, PlayersOnlineKnown: true, @@ -257,6 +258,7 @@ func TestMetricsZeroValuesEmitted(t *testing.T) { MemoryBytes: 0, MemoryKnown: true, DiskUsedBytes: 0, + DiskTotalKnown: true, DiskKnown: true, PlayersOnline: 0, PlayersOnlineKnown: true, @@ -438,6 +440,7 @@ func TestMetricsDiskZeroTotal(t *testing.T) { snap := Snapshot{ DiskUsedBytes: 0, DiskTotalBytes: 0, // Edge case: zero total (e.g., mount failed) + DiskTotalKnown: true, DiskKnown: true, ServerName: "test-server", Namespace: "games", @@ -592,6 +595,7 @@ func TestMetricsAllMetricsIndependent(t *testing.T) { // Disk: known DiskUsedBytes: 500 * 1024 * 1024, DiskTotalBytes: 1024 * 1024 * 1024, + DiskTotalKnown: true, DiskKnown: true, // Players: online known, max unknown PlayersOnline: 8, diff --git a/agent/internal/usage/disk.go b/agent/internal/usage/disk.go new file mode 100644 index 000000000..c2444931b --- /dev/null +++ b/agent/internal/usage/disk.go @@ -0,0 +1,205 @@ +package usage + +import ( + "context" + "errors" + "fmt" + "io" + "math" + "os" + "strconv" + "time" + + "golang.org/x/sys/unix" +) + +const ( + diskInterval = 5 * time.Minute + diskTimeout = time.Minute + maxDiskEntries = 2_000_000 + maxDiskDepth = 128 +) + +// MonitorDisk runs one scan at a time until ctx is canceled. A shared +// filesystem has no per-directory capacity. The reader publishes allocated +// bytes only after the entire scan completes, and clears a failed sample. +func (r *Reader) MonitorDisk(ctx context.Context) { r.monitorDisk(ctx, volumeUsage) } + +func (r *Reader) monitorDisk(ctx context.Context, scan func(context.Context, string, int) (diskSample, error)) { + if r.cfg.DataDir == "" { + return + } + for ctx.Err() == nil { + scanCtx, cancel := context.WithTimeout(ctx, diskTimeout) + sample, err := scan(scanCtx, r.cfg.DataDir, maxDiskEntries) + // A scan and its timer can become runnable together. Observe the + // deadline itself even if cancellation has not been delivered yet. + deadline, _ := scanCtx.Deadline() + completed := err == nil && scanCtx.Err() == nil && time.Now().Before(deadline) + cancel() + r.mu.Lock() + r.diskMeasuredAt = time.Time{} + if completed { + r.diskBytes = sample.used + r.diskCapacity = sample.capacity + r.diskMeasuredAt = r.cfg.now() + } + r.mu.Unlock() + select { + case <-ctx.Done(): + return + case <-time.After(diskInterval): + } + } +} + +func (r *Reader) readDisk(s *Sample) { + r.mu.Lock() + defer r.mu.Unlock() + if !r.diskMeasuredAt.IsZero() && r.cfg.now().Sub(r.diskMeasuredAt) < diskInterval+diskTimeout { + s.DiskUsedBytes = r.diskBytes + s.DiskKnown = true + s.DiskTotalBytes = r.diskCapacity + s.DiskTotalKnown = r.diskCapacity > 0 + } +} + +type diskSample struct{ used, capacity int64 } + +func volumeUsage(ctx context.Context, root string, limit int) (diskSample, error) { + fd, err := unix.Open(root, unix.O_RDONLY|unix.O_DIRECTORY|unix.O_NOFOLLOW|unix.O_CLOEXEC, 0) + if err != nil { + return diskSample{}, err + } + dir := os.NewFile(uintptr(fd), root) + defer func() { _ = dir.Close() }() + var fs unix.Statfs_t + if err := unix.Fstatfs(fd, &fs); err == nil && fs.Type == unix.CEPH_SUPER_MAGIC { + // Kernel CephFS exposes recursive usage and quotas as virtual xattrs on + // this directory. An inherited/absent quota cannot establish capacity. + if s, ok := cephDiskUsage(func(key string) (int64, error) { return diskAttribute(fd, key) }); ok { + return s, nil + } + } + used, err := allocatedDiskUsage(ctx, dir, limit) + return diskSample{used: used}, err +} + +// CephFS exposes the counters as decimal integers on the opened directory. +func diskAttribute(fd int, key string) (int64, error) { + var value [32]byte + n, err := unix.Fgetxattr(fd, key, value[:]) + if err != nil { + return 0, err + } + return strconv.ParseInt(string(value[:n]), 10, 64) +} + +func cephDiskUsage(read func(string) (int64, error)) (diskSample, bool) { + used, err := read("ceph.dir.rbytes") + if err != nil || used < 0 { + return diskSample{}, false + } + quota, err := read("ceph.quota.max_bytes") + if err != nil || quota < 0 { + quota = 0 + } + return diskSample{used: used, capacity: quota}, true +} + +type diskWalk struct { + ctx context.Context + remaining int + // Single-link files need no entry. NO_XDEV confines inode numbers to one mount. + hardlinks map[uint64]struct{} + bytes int64 +} + +func allocatedDiskUsage(ctx context.Context, dir *os.File, limit int) (int64, error) { + fd := int(dir.Fd()) + walk := diskWalk{ctx: ctx, remaining: limit, hardlinks: make(map[uint64]struct{})} + if err := walk.count(fd); err != nil { + return 0, err + } + if err := walk.dir(dir, 0); err != nil { + return 0, err + } + return walk.bytes, nil +} + +func (w *diskWalk) count(fd int) error { + var st unix.Stat_t + if err := unix.Fstat(fd, &st); err != nil { + return err + } + if st.Mode&unix.S_IFMT != unix.S_IFDIR && st.Nlink > 1 { + if _, ok := w.hardlinks[st.Ino]; ok { + return nil + } + w.hardlinks[st.Ino] = struct{}{} + } + if st.Blocks < 0 || st.Blocks > (math.MaxInt64-w.bytes)/512 { + return errors.New("allocated byte count overflow") + } + w.bytes += st.Blocks * 512 + return nil +} + +func (w *diskWalk) dir(dir *os.File, depth int) error { + if depth > maxDiskDepth { + return errors.New("disk scan depth exceeded") + } + for { + if err := w.ctx.Err(); err != nil { + return err + } + names, err := dir.Readdirnames(128) + if err != nil && !errors.Is(err, io.EOF) { + return err + } + for _, name := range names { + if err := w.ctx.Err(); err != nil { + return err + } + w.remaining-- + if w.remaining < 0 { + return errors.New("disk scan entry limit exceeded") + } + if err := w.entry(int(dir.Fd()), name, depth); err != nil { + return err + } + } + if errors.Is(err, io.EOF) { + return nil + } + } +} + +func (w *diskWalk) entry(parent int, name string, depth int) error { + // O_PATH avoids opening FIFOs or devices for I/O. NO_XDEV also excludes + // bind mounts on the same device; NOFOLLOW counts the link itself. + fd, err := unix.Openat2(parent, name, &unix.OpenHow{Flags: unix.O_PATH | unix.O_CLOEXEC | unix.O_NOFOLLOW, Resolve: unix.RESOLVE_BENEATH | unix.RESOLVE_NO_XDEV | unix.RESOLVE_NO_MAGICLINKS}) + // Entries that disappeared after readdir no longer occupy this path. + if errors.Is(err, unix.EXDEV) || errors.Is(err, unix.ENOENT) { + return nil + } + if err != nil { + return fmt.Errorf("open disk entry: %w", err) + } + defer func() { _ = unix.Close(fd) }() + if err := w.count(fd); err != nil { + return err + } + // Reopen the pinned inode as a readable directory. ENOTDIR identifies + // leaves without trusting the directory entry type or reopening its name. + child, err := unix.Openat(fd, ".", unix.O_RDONLY|unix.O_DIRECTORY|unix.O_CLOEXEC, 0) + if errors.Is(err, unix.ENOTDIR) { + return nil + } + if err != nil { + return err + } + dir := os.NewFile(uintptr(child), name) + defer func() { _ = dir.Close() }() + return w.dir(dir, depth+1) +} diff --git a/agent/internal/usage/disk_test.go b/agent/internal/usage/disk_test.go new file mode 100644 index 000000000..8de9c06d7 --- /dev/null +++ b/agent/internal/usage/disk_test.go @@ -0,0 +1,370 @@ +package usage + +import ( + "context" + "errors" + "math" + "os" + "path/filepath" + "testing" + "testing/synctest" + "time" + + "golang.org/x/sys/unix" +) + +func TestAllocatedDiskUsage(t *testing.T) { + parent := t.TempDir() + root := filepath.Join(parent, "server") + if err := os.Mkdir(root, 0700); err != nil { + t.Fatal(err) + } + file := filepath.Join(root, "region") + if err := os.WriteFile(file, []byte("world"), 0600); err != nil { + t.Fatal(err) + } + if err := os.Truncate(file, 64<<20); err != nil { + t.Fatal(err) + } + if err := os.Link(file, file+"-link"); err != nil { + t.Fatal(err) + } + outside := filepath.Join(parent, "sibling") + if err := os.WriteFile(outside, make([]byte, 1<<20), 0600); err != nil { + t.Fatal(err) + } + link := filepath.Join(root, "external") + if err := os.Symlink(outside, link); err != nil { + t.Fatal(err) + } + var want int64 + for _, path := range []string{root, file, link} { + var st unix.Stat_t + if err := unix.Lstat(path, &st); err != nil { + t.Fatal(err) + } + want += st.Blocks * 512 + } + got, err := volumeUsage(context.Background(), root, 100) + if err != nil || got.used != want { + t.Fatalf("size=%v, want %d; err=%v", got, want, err) + } + if _, err = volumeUsage(context.Background(), root, 2); err == nil { + t.Fatal("published incomplete scan") + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err = volumeUsage(ctx, root, 100); !errors.Is(err, context.Canceled) { + t.Fatalf("cancellation: %v", err) + } + if _, err = volumeUsage(context.Background(), outside+"-missing", 100); err == nil { + t.Fatal("missing directory returned a sample") + } +} +func TestDiskScanDoesNotBlockHeartbeat(t *testing.T) { + started := make(chan struct{}) + stopped := make(chan struct{}) + r := New(Config{Root: t.TempDir(), DataDir: "/data"}) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go r.monitorDisk(ctx, func(ctx context.Context, _ string, _ int) (diskSample, error) { + close(started) + <-ctx.Done() + close(stopped) + return diskSample{}, ctx.Err() + }) + select { + case <-started: + case <-time.After(time.Second): + t.Fatal("worker did not start") + } + result := make(chan Sample, 1) + go func() { result <- r.Read() }() + select { + case s := <-result: + if s.DiskKnown { + t.Fatal("partial scan published") + } + case <-time.After(time.Second): + t.Fatal("heartbeat blocked by disk scan") + } + cancel() + select { + case <-stopped: + case <-time.After(time.Second): + t.Fatal("worker did not stop") + } +} + +func TestCephDirectoryCounters(t *testing.T) { + for _, tc := range []struct { + name string + values map[string]int64 + want diskSample + ok bool + }{ + {"quota", map[string]int64{"ceph.dir.rbytes": 0, "ceph.quota.max_bytes": 9000}, diskSample{0, 9000}, true}, + {"no local quota", map[string]int64{"ceph.dir.rbytes": 20}, diskSample{20, 0}, true}, + {"missing counters", nil, diskSample{}, false}, + } { + t.Run(tc.name, func(t *testing.T) { + got, ok := cephDiskUsage(func(key string) (int64, error) { + v, exists := tc.values[key] + if !exists { + return 0, unix.ENODATA + } + return v, nil + }) + if ok != tc.ok || got != tc.want { + t.Fatalf("%+v %v", got, ok) + } + }) + } +} + +func TestDiskWalkDoesNotDescendIntoProcMount(t *testing.T) { + root, err := os.Open("/") + if err != nil { + t.Fatal(err) + } + defer root.Close() + var proc unix.Statfs_t + if unix.Statfs("/proc", &proc) != nil || proc.Type != unix.PROC_SUPER_MAGIC { + t.Skip("procfs not mounted") + } + walk := diskWalk{ctx: context.Background(), remaining: 0} + if err := walk.entry(int(root.Fd()), "proc", 0); err != nil || walk.bytes != 0 { + t.Fatalf("walk crossed mount: bytes=%d, err=%v", walk.bytes, err) + } +} +func TestDiskWalkExcludesSameDeviceBind(t *testing.T) { + root := t.TempDir() + outside := t.TempDir() + mount := filepath.Join(root, "nested") + if err := os.Mkdir(mount, 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(outside, "foreign"), make([]byte, 1<<20), 0600); err != nil { + t.Fatal(err) + } + if err := unix.Mount(outside, mount, "", unix.MS_BIND, ""); err != nil { + if errors.Is(err, unix.EPERM) { + t.Skip("bind mount needs a private privileged test namespace") + } + t.Fatal(err) + } + defer unix.Unmount(mount, 0) + var st unix.Stat_t + if err := unix.Stat(root, &st); err != nil { + t.Fatal(err) + } + got, err := volumeUsage(context.Background(), root, 100) + if err != nil || got.used != st.Blocks*512 { + t.Fatalf("nested bind was counted: %+v %v", got, err) + } +} + +func TestUnlinkDuringDirectoryScan(t *testing.T) { + root, err := os.Open(t.TempDir()) + if err != nil { + t.Fatal(err) + } + defer root.Close() + walk := diskWalk{ctx: context.Background(), remaining: 1} + if err := walk.entry(int(root.Fd()), "already-removed", 0); err != nil || walk.bytes != 0 { + t.Fatalf("removed entry: %d %v", walk.bytes, err) + } +} + +func TestDiskMonitorPublishesOnlyFreshCompleteScans(t *testing.T) { + root := t.TempDir() + file := filepath.Join(root, "region") + if err := os.WriteFile(file, []byte("world"), 0o600); err != nil { + t.Fatal(err) + } + synctest.Test(t, func(t *testing.T) { + r := New(Config{Root: root, DataDir: root}) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + finish := make(chan struct{}) + var starts int + go r.monitorDisk(ctx, func(ctx context.Context, root string, limit int) (diskSample, error) { + starts++ + s, err := volumeUsage(ctx, root, limit) + <-finish + return s, err + }) + synctest.Wait() + if starts != 1 || r.Read().DiskKnown { + t.Fatal("published first scan before completion") + } + time.Sleep(diskTimeout / 2) + finish <- struct{}{} + synctest.Wait() + if s := r.Read(); !s.DiskKnown || s.DiskUsedBytes <= 0 || s.DiskTotalKnown { + t.Fatalf("complete scan: %+v", s) + } + time.Sleep(diskInterval) + synctest.Wait() + if starts != 2 || !r.Read().DiskKnown { + t.Fatal("fresh prior measurement disappeared during second scan") + } + time.Sleep(diskTimeout) + if r.Read().DiskKnown { + t.Fatal("published expired sample while the second scan was blocked") + } + finish <- struct{}{} + synctest.Wait() + if r.Read().DiskKnown { + t.Fatal("late scan result resurrected a stale measurement") + } + }) +} + +func TestPublicDiskMonitorAndFailedRefresh(t *testing.T) { + root := t.TempDir() + synctest.Test(t, func(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + New(Config{}).MonitorDisk(ctx) + r := New(Config{Root: root, DataDir: root}) + go r.MonitorDisk(ctx) + synctest.Wait() + if !r.Read().DiskKnown { + t.Fatal("missing initial measurement") + } + if err := os.Remove(root); err != nil { + t.Fatal(err) + } + time.Sleep(diskInterval) + synctest.Wait() + if r.Read().DiskKnown { + t.Fatal("measurement from a removed data directory remained visible") + } + }) +} + +func TestDiskUsageRecursesAndRejectsDeepTrees(t *testing.T) { + root := t.TempDir() + where := root + var want int64 + for i := 0; i < 4; i++ { + var st unix.Stat_t + if err := unix.Stat(where, &st); err != nil { + t.Fatal(err) + } + want += st.Blocks * 512 + child := filepath.Join(where, "child") + if err := os.Mkdir(child, 0o700); err != nil { + t.Fatal(err) + } + where = child + } + file := filepath.Join(where, "data") + if err := os.WriteFile(file, make([]byte, 8192), 0o600); err != nil { + t.Fatal(err) + } + for _, path := range []string{where, file} { + var st unix.Stat_t + if err := unix.Stat(path, &st); err != nil { + t.Fatal(err) + } + want += st.Blocks * 512 + } + if got, err := volumeUsage(t.Context(), root, maxDiskEntries); err != nil || got.used != want { + t.Fatalf("recursive size: %+v want %d, err=%v", got, want, err) + } + for i := 4; i <= maxDiskDepth; i++ { + where = filepath.Join(where, "d") + if err := os.Mkdir(where, 0o700); err != nil { + t.Fatal(err) + } + } + if got, err := volumeUsage(t.Context(), root, maxDiskEntries); err == nil || got.used != 0 { + t.Fatalf("deep scan published %+v (%v)", got, err) + } +} + +func TestDiskUsageReadFailures(t *testing.T) { + root := t.TempDir() + file := filepath.Join(root, "file") + if err := os.WriteFile(file, []byte("allocated"), 0o600); err != nil { + t.Fatal(err) + } + f, err := os.Open(file) + if err != nil { + t.Fatal(err) + } + defer f.Close() + if bytes, err := allocatedDiskUsage(t.Context(), f, 50); !errors.Is(err, unix.ENOTDIR) || bytes != 0 { + t.Fatalf("non-directory produced %d bytes, %v", bytes, err) + } + w := diskWalk{ctx: t.Context(), bytes: math.MaxInt64} + if err := w.count(int(f.Fd())); err == nil { + t.Fatal("overflow produced a finite measurement") + } + dir, err := os.Open(root) + if err != nil { + t.Fatal(err) + } + if err := w.entry(int(dir.Fd()), "file", 0); err == nil { + t.Fatal("entry overflow did not abort scan") + } + if err := dir.Close(); err != nil { + t.Fatal(err) + } + if bytes, err := allocatedDiskUsage(t.Context(), dir, 0); !errors.Is(err, unix.EBADF) || bytes != 0 { + t.Fatalf("closed root produced %d bytes, %v", bytes, err) + } + if err := w.entry(-1, "child", 0); !errors.Is(err, unix.EBADF) { + t.Fatalf("invalid parent: %v", err) + } +} + +func TestDiskUsageUnreadableSubtree(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("root bypasses directory mode checks") + } + root := t.TempDir() + child := filepath.Join(root, "save") + if err := os.Mkdir(child, 0); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(child, 0o700) }) + got, err := volumeUsage(t.Context(), root, 100) + if !errors.Is(err, unix.EACCES) || got.used != 0 { + t.Fatalf("unreadable subtree produced %+v (%v)", got, err) + } +} + +func TestDiskAttributesFromPinnedDirectory(t *testing.T) { + dir, err := os.Open(t.TempDir()) + if err != nil { + t.Fatal(err) + } + defer dir.Close() + fd := int(dir.Fd()) + set := func(k, v string) { + t.Helper() + if err := unix.Fsetxattr(fd, "user."+k, []byte(v), 0); errors.Is(err, unix.ENOTSUP) { + t.Skip("filesystem does not support user xattrs") + } else if err != nil { + t.Fatal(err) + } + } + // The same descriptor-based getxattr and parser used on kernel CephFS + // can be exercised on an ordinary Linux filesystem with user attributes. + read := func(key string) (int64, error) { return diskAttribute(fd, "user."+key) } + set("ceph.dir.rbytes", "123456") + if got, ok := cephDiskUsage(read); !ok || got != (diskSample{used: 123456}) { + t.Fatalf("missing quota: %+v %t", got, ok) + } + set("ceph.quota.max_bytes", "204800") + if got, ok := cephDiskUsage(read); !ok || got != (diskSample{used: 123456, capacity: 204800}) { + t.Fatalf("measured quota: %+v %t", got, ok) + } + set("ceph.dir.rbytes", "invalid") + if _, ok := cephDiskUsage(read); ok { + t.Fatal("malformed counter accepted") + } +} diff --git a/agent/internal/usage/usage.go b/agent/internal/usage/usage.go index c653bfa21..cda2657e1 100644 --- a/agent/internal/usage/usage.go +++ b/agent/internal/usage/usage.go @@ -15,7 +15,7 @@ // pod's cgroup v2 files. Inside a per-container cgroup namespace this // reflects only the agent container, so it's a fallback. // -// Disk is a statfs of the game data volume in both modes. Each value carries +// Disk is sampled asynchronously within the game data directory. Each value carries // a "known" flag so callers can patch null ("unknown") rather than a // misleading zero when a source is unreadable. package usage @@ -28,8 +28,6 @@ import ( "strings" "sync" "time" - - "golang.org/x/sys/unix" ) // defaultCgroupRoot is the cgroup v2 unified-hierarchy mount present in @@ -49,7 +47,7 @@ const userHZ = 100 type Config struct { // Root is the cgroup v2 mount point. Empty means defaultCgroupRoot. Root string - // DataDir is the game data volume to statfs for disk usage. Empty + // DataDir is the game data volume to measure. Empty // disables the disk reading. DataDir string @@ -65,11 +63,10 @@ type Config struct { // ProcRoot is the procfs mount; empty means defaultProcRoot. ProcRoot string - // selfPID, clkTck, now and statfs are injected only by tests. + // selfPID, clkTck and now are injected only by tests. selfPID int clkTck int64 now func() time.Time - statfs func(string, *unix.Statfs_t) error } // Sample is one point-in-time reading. Each *Known flag reports whether @@ -88,6 +85,7 @@ type Sample struct { DiskUsedBytes int64 DiskTotalBytes int64 + DiskTotalKnown bool DiskKnown bool } @@ -98,10 +96,13 @@ type Sample struct { type Reader struct { cfg Config - mu sync.Mutex - prevUsageUsec uint64 - prevAt time.Time - havePrev bool + mu sync.Mutex + prevUsageUsec uint64 + prevAt time.Time + havePrev bool + diskBytes int64 + diskCapacity int64 + diskMeasuredAt time.Time } // New builds a Reader, applying defaults for any unset Config field. @@ -123,9 +124,6 @@ func New(cfg Config) *Reader { if cfg.now == nil { cfg.now = time.Now } - if cfg.statfs == nil { - cfg.statfs = unix.Statfs - } return &Reader{cfg: cfg} } @@ -392,35 +390,6 @@ func readProcStatmRSS(path string, procRoot string) int64 { return rss } -func (r *Reader) readDisk(s *Sample) { - if r.cfg.DataDir == "" { - return - } - var st unix.Statfs_t - if err := r.cfg.statfs(r.cfg.DataDir, &st); err != nil { - return - } - // Bsize's Go type varies by arch (int64 on amd64/arm64); convert via - // uint64 so the arithmetic is correct everywhere and no conversion is - // redundant. - if st.Bsize <= 0 { - return - } - bsize := uint64(st.Bsize) - // Safe: block counts and sizes are always positive and bounded by filesystem limits. - total := st.Blocks * bsize - if total > uint64(math.MaxInt64) { - return - } - s.DiskTotalBytes = int64(total) - used := (st.Blocks - st.Bfree) * bsize - if used > uint64(math.MaxInt64) { - return - } - s.DiskUsedBytes = int64(used) - s.DiskKnown = true -} - // readUintFile parses a cgroup file holding a single unsigned integer. func readUintFile(path string) (uint64, bool) { b, err := os.ReadFile(filepath.Clean(path)) diff --git a/agent/internal/usage/usage_test.go b/agent/internal/usage/usage_test.go index a18d8d5c7..a7f797a3c 100644 --- a/agent/internal/usage/usage_test.go +++ b/agent/internal/usage/usage_test.go @@ -1,15 +1,12 @@ package usage import ( - "errors" "fmt" "os" "path/filepath" "strconv" "testing" "time" - - "golang.org/x/sys/unix" ) func writeFile(t *testing.T, dir, name, body string) { @@ -121,32 +118,6 @@ func TestRead_MemoryMaxUnlimited(t *testing.T) { } } -func TestRead_Disk(t *testing.T) { - r := New(Config{ - Root: t.TempDir(), - DataDir: "/data", - statfs: func(path string, st *unix.Statfs_t) error { - if path != "/data" { - t.Fatalf("statfs path = %q", path) - } - st.Bsize = 4096 - st.Blocks = 1000 - st.Bfree = 250 - return nil - }, - }) - s := r.Read() - if !s.DiskKnown { - t.Fatalf("disk should be known") - } - if s.DiskTotalBytes != 4096*1000 { - t.Fatalf("disk total = %d", s.DiskTotalBytes) - } - if s.DiskUsedBytes != 4096*750 { - t.Fatalf("disk used = %d", s.DiskUsedBytes) - } -} - func TestRead_DiskDisabledWhenNoDataDir(t *testing.T) { s := New(Config{Root: t.TempDir()}).Read() if s.DiskKnown { @@ -154,31 +125,6 @@ func TestRead_DiskDisabledWhenNoDataDir(t *testing.T) { } } -func TestRead_DiskStatfsError(t *testing.T) { - r := New(Config{ - Root: t.TempDir(), - DataDir: "/data", - statfs: func(string, *unix.Statfs_t) error { return errors.New("boom") }, - }) - if r.Read().DiskKnown { - t.Fatalf("disk should be unknown on statfs error") - } -} - -func TestRead_DiskZeroBlockSize(t *testing.T) { - r := New(Config{ - Root: t.TempDir(), - DataDir: "/data", - statfs: func(_ string, st *unix.Statfs_t) error { - st.Bsize = 0 - return nil - }, - }) - if r.Read().DiskKnown { - t.Fatalf("disk should be unknown when block size is zero") - } -} - func TestRead_AllUnknownWhenAbsent(t *testing.T) { s := New(Config{Root: t.TempDir()}).Read() if s.CPUKnown || s.CPULimitKnown || s.MemoryKnown || s.MemoryLimitKnown || s.DiskKnown { @@ -229,8 +175,8 @@ func TestNew_DefaultsRoot(t *testing.T) { if r.cfg.Root != defaultCgroupRoot { t.Fatalf("default root = %q, want %q", r.cfg.Root, defaultCgroupRoot) } - if r.cfg.now == nil || r.cfg.statfs == nil { - t.Fatalf("now/statfs should default to non-nil") + if r.cfg.now == nil { + t.Fatalf("clock should default to non-nil") } } diff --git a/agent/specs.md b/agent/specs.md index 40935f565..55cdfa100 100644 --- a/agent/specs.md +++ b/agent/specs.md @@ -63,7 +63,7 @@ agent/ │ ├── quiesce/ # Pause auto-saves and flush state before snapshots │ ├── rcon/ # RCON wire protocols (source, telnet, websocket, battleye, satisfactory, palworld, nuclearoption, rest, cli) │ ├── status/ # Live metrics extraction via RCON + regex -│ └── usage/ # Resource usage reader (/proc or cgroups; disk via statfs) +│ └── usage/ # Resource usage reader (/proc or cgroups; asynchronous per-directory disk usage) ├── openapi.yaml # Partial machine-readable HTTP contract (subset of routes) └── .testcoverage.yml # Coverage gate (90% total, unit-only) ``` @@ -98,7 +98,7 @@ Per-package roles: - **`quiesce`**: Runs module-declared sequences (e.g., Minecraft's `save-off` + `save-all flush`) over RCON; responds `quiesced: false` + reason when unsupported (not an error). - **`rcon`**: Factory pattern for wire-protocol clients (`Valve/Source`, `Telnet`, `WebSocket`, `BattlEye`, `Satisfactory`, `Palworld`, `NuclearOption`, `REST`, `CLI`, `Disabled`); `Exec(cmd) (string, error)` interface. RCON reply packets are accepted up to 16394 bytes (a 4096-character Minecraft chunk at its worst-case UTF-8 byte length, plus the 10-byte packet header) and rejected as malformed outside `[10, 16394]` (F-104). Source RCON also caps aggregate response text at 8 MiB and enforces an absolute 30-second command exchange deadline after connection/authentication; each 400-ms fragment idle window is clamped to that deadline. Exceeding the aggregate cap or hard deadline drops the connection and returns an error without partial output. The status collector separately accepts at most 64 KiB. - **`status`**: Runs module-declared metrics queries over RCON; each metric specifies a command and a regex with named group `"value"` for extraction. -- **`usage`**: Reads CPU/memory from `/proc` (proc mode, default) or cgroup v2; disk via `statfs`; exposes `Sample` with `Known` flags so callers distinguish "unknown" from "zero". +- **`usage`**: Reads CPU/memory from `/proc` (proc mode, default) or cgroup v2; disk from kernel CephFS recursive logical byte counts and the local directory quota, or a bounded asynchronous allocated-block scan on Linux 5.6+; exposes `Sample` with `Known` flags so callers distinguish "unknown" from "zero". ## External interface / contracts @@ -303,3 +303,5 @@ actual server incarnation even if the name-based agent Service changes between the gateway's Kubernetes lookup and connection. Existing operator-owned game Pod templates gain the UID environment variable on reconciliation, which can roll existing game workloads during an operator upgrade. + +Disk scans run every five minutes, with a one-minute budget, a two-million-entry ceiling, and at most 128 nested directories. A failed or incomplete scan clears the reading. A blocked filesystem call can delay cancellation of the single reader. The heartbeat continues and expires stale usage. On shared filesystems a scan counts allocated blocks, so sparse files can use less space than their logical length. diff --git a/assets/design-export/MANIFEST.md b/assets/design-export/MANIFEST.md index b7e0ccd34..ae8211611 100644 --- a/assets/design-export/MANIFEST.md +++ b/assets/design-export/MANIFEST.md @@ -2874,3 +2874,6 @@ Nodes now follows the Backups toolbar: a plain search input, capped at 320px on All seven roots have JSON from Pencil `Get` and 2× PNG from `Export`. Toolbar, popover, desktop, fixed-cluster and mobile layouts were inspected using `TakeScreenshot`; the full screens retain their previous dimensions and content. JSON exports retain component references without elided children or editing placeholders. No React source, tests or lint suites were changed or run for this design update. +## Server volume measurements + +The running Overview (`EZFW0`) shows measured data usage alongside the requested PVC capacity when a filesystem total is unavailable. The sleeping Overview (`TE2jI`) shows the request and a dash while usage is unknown. diff --git a/assets/design-export/json/EZFW0.json b/assets/design-export/json/EZFW0.json index 251cb37c0..78c345a67 100644 --- a/assets/design-export/json/EZFW0.json +++ b/assets/design-export/json/EZFW0.json @@ -325,6 +325,7 @@ "width": "fill_container", "gap": 0, "padding": 0, + "height": "fill_container", "children": [ { "type": "frame", @@ -366,8 +367,9 @@ { "type": "text", "id": "QLkWj", + "name": "Measured disk usage", "fill": "$foreground/foreground", - "content": "13%", + "content": "3.7 GB", "fontFamily": "$typography/font-mono", "fontSize": 24, "fontWeight": "700" @@ -375,6 +377,9 @@ { "type": "path", "id": "vdDJJ", + "x": 16, + "y": 80, + "enabled": false, "geometry": "M0 21l12-1 13 1 12-2 13 1 12-1 13 2 12-1 13 0", "viewBox": [ 0, @@ -383,7 +388,7 @@ 24 ], "fill": "#00000000", - "width": "fill_container", + "width": "fill_container(201.33333333333334)", "height": 24, "stroke": "$success/soft-foreground", "strokeWidth": 1.5 @@ -391,8 +396,11 @@ { "type": "frame", "id": "JpkaE", + "x": 16, + "y": 112, + "enabled": false, "clip": true, - "width": "fill_container", + "width": "fill_container(201.33333333333334)", "height": 6, "fill": "$default/default", "cornerRadius": 999, @@ -410,8 +418,9 @@ { "type": "text", "id": "gijqx", + "name": "Requested PVC capacity", "fill": "$muted", - "content": "3.7 GB / 29 GB", + "content": "30Gi requested", "fontFamily": "$typography/font-sans", "fontSize": 12, "fontWeight": "normal" diff --git a/assets/design-export/json/TE2jI.json b/assets/design-export/json/TE2jI.json index 35a059723..7c70eb0bf 100644 --- a/assets/design-export/json/TE2jI.json +++ b/assets/design-export/json/TE2jI.json @@ -153,6 +153,7 @@ "id": "Mcf1C", "name": "Metric CPU", "width": "fill_container", + "height": "fill_container", "fill": "$surface/surface", "stroke": "$border/border", "strokeWidth": 1, @@ -280,6 +281,7 @@ "id": "QGBuR", "name": "Metric Memory", "width": "fill_container", + "height": "fill_container", "fill": "$surface/surface", "stroke": "$border/border", "strokeWidth": 1, @@ -516,11 +518,9 @@ { "type": "text", "id": "bsk1A", - "x": 16, - "y": 126, - "enabled": false, + "name": "Requested PVC capacity", "fill": "$muted", - "content": "3.7 GB / 29 GB", + "content": "30Gi requested", "fontFamily": "$typography/font-sans", "fontSize": 12, "fontWeight": "normal" diff --git a/assets/design-export/screenshots/EZFW0.png b/assets/design-export/screenshots/EZFW0.png index 5f9181e76..e86e24170 100644 Binary files a/assets/design-export/screenshots/EZFW0.png and b/assets/design-export/screenshots/EZFW0.png differ diff --git a/assets/design-export/screenshots/TE2jI.png b/assets/design-export/screenshots/TE2jI.png index 18499a61a..1544684c7 100644 Binary files a/assets/design-export/screenshots/TE2jI.png and b/assets/design-export/screenshots/TE2jI.png differ diff --git a/changelog/unreleased/937-server-volume-usage.md b/changelog/unreleased/937-server-volume-usage.md new file mode 100644 index 000000000..c488e600f --- /dev/null +++ b/changelog/unreleased/937-server-volume-usage.md @@ -0,0 +1,3 @@ +# Per-server storage usage + +The agent reports storage scoped to the server data directory. It uses native directory counters on kernel CephFS and a bounded background scan on other filesystems. The dashboard labels requested storage separately when measured capacity is unavailable. diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backups.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backups.yaml index 63d29c4f4..2d02e8e80 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backups.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backups.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 3b2e34cfb50a4b2312502522937d45442c012cb313195cd2a556f69af01f3a96 + gameplane.local/crd-bundle-sha256: 9ffb2bdfc486b12cb927fed97d0ce3f185b9cb06d7875092aa4c28b16490d752 controller-gen.kubebuilder.io/version: v0.22.0 name: backups.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backupschedules.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backupschedules.yaml index 4beb31166..78cd38439 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backupschedules.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backupschedules.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 3b2e34cfb50a4b2312502522937d45442c012cb313195cd2a556f69af01f3a96 + gameplane.local/crd-bundle-sha256: 9ffb2bdfc486b12cb927fed97d0ce3f185b9cb06d7875092aa4c28b16490d752 controller-gen.kubebuilder.io/version: v0.22.0 name: backupschedules.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_clusters.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_clusters.yaml index 18b3b8f55..755773aff 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_clusters.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_clusters.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 3b2e34cfb50a4b2312502522937d45442c012cb313195cd2a556f69af01f3a96 + gameplane.local/crd-bundle-sha256: 9ffb2bdfc486b12cb927fed97d0ce3f185b9cb06d7875092aa4c28b16490d752 controller-gen.kubebuilder.io/version: v0.22.0 name: clusters.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gameservers.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gameservers.yaml index c9f4e670b..77f0f5626 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gameservers.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gameservers.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 3b2e34cfb50a4b2312502522937d45442c012cb313195cd2a556f69af01f3a96 + gameplane.local/crd-bundle-sha256: 9ffb2bdfc486b12cb927fed97d0ce3f185b9cb06d7875092aa4c28b16490d752 controller-gen.kubebuilder.io/version: v0.22.0 name: gameservers.gameplane.local spec: @@ -2434,13 +2434,13 @@ spec: type: integer diskTotalBytes: description: |- - DiskTotalBytes is the total size of the game data volume (statfs). + DiskTotalBytes is the observed capacity scoped to the game data volume. nil means "unknown". format: int64 type: integer diskUsedBytes: description: |- - DiskUsedBytes is the used space on the game data volume (statfs). + DiskUsedBytes is observed usage scoped to the game data directory. nil means "unknown". format: int64 type: integer diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gametemplates.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gametemplates.yaml index fb1cca85b..457defab0 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gametemplates.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gametemplates.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 3b2e34cfb50a4b2312502522937d45442c012cb313195cd2a556f69af01f3a96 + gameplane.local/crd-bundle-sha256: 9ffb2bdfc486b12cb927fed97d0ce3f185b9cb06d7875092aa4c28b16490d752 controller-gen.kubebuilder.io/version: v0.22.0 name: gametemplates.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modules.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modules.yaml index 5d74582b1..e5510694d 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modules.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modules.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 3b2e34cfb50a4b2312502522937d45442c012cb313195cd2a556f69af01f3a96 + gameplane.local/crd-bundle-sha256: 9ffb2bdfc486b12cb927fed97d0ce3f185b9cb06d7875092aa4c28b16490d752 controller-gen.kubebuilder.io/version: v0.22.0 name: modules.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modulesources.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modulesources.yaml index 8cf33a5ec..7e91cae57 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modulesources.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modulesources.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 3b2e34cfb50a4b2312502522937d45442c012cb313195cd2a556f69af01f3a96 + gameplane.local/crd-bundle-sha256: 9ffb2bdfc486b12cb927fed97d0ce3f185b9cb06d7875092aa4c28b16490d752 controller-gen.kubebuilder.io/version: v0.22.0 name: modulesources.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_networkcaptures.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_networkcaptures.yaml index f2cd36c44..50f36554c 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_networkcaptures.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_networkcaptures.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 3b2e34cfb50a4b2312502522937d45442c012cb313195cd2a556f69af01f3a96 + gameplane.local/crd-bundle-sha256: 9ffb2bdfc486b12cb927fed97d0ce3f185b9cb06d7875092aa4c28b16490d752 controller-gen.kubebuilder.io/version: v0.22.0 name: networkcaptures.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_restores.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_restores.yaml index d2f47f0ec..9a6a6aa88 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_restores.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_restores.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 3b2e34cfb50a4b2312502522937d45442c012cb313195cd2a556f69af01f3a96 + gameplane.local/crd-bundle-sha256: 9ffb2bdfc486b12cb927fed97d0ce3f185b9cb06d7875092aa4c28b16490d752 controller-gen.kubebuilder.io/version: v0.22.0 name: restores.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_backups.yaml b/charts/gameplane/crd-manifests/gameplane.local_backups.yaml index 63d29c4f4..2d02e8e80 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_backups.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_backups.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 3b2e34cfb50a4b2312502522937d45442c012cb313195cd2a556f69af01f3a96 + gameplane.local/crd-bundle-sha256: 9ffb2bdfc486b12cb927fed97d0ce3f185b9cb06d7875092aa4c28b16490d752 controller-gen.kubebuilder.io/version: v0.22.0 name: backups.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_backupschedules.yaml b/charts/gameplane/crd-manifests/gameplane.local_backupschedules.yaml index 4beb31166..78cd38439 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_backupschedules.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_backupschedules.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 3b2e34cfb50a4b2312502522937d45442c012cb313195cd2a556f69af01f3a96 + gameplane.local/crd-bundle-sha256: 9ffb2bdfc486b12cb927fed97d0ce3f185b9cb06d7875092aa4c28b16490d752 controller-gen.kubebuilder.io/version: v0.22.0 name: backupschedules.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_clusters.yaml b/charts/gameplane/crd-manifests/gameplane.local_clusters.yaml index 18b3b8f55..755773aff 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_clusters.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_clusters.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 3b2e34cfb50a4b2312502522937d45442c012cb313195cd2a556f69af01f3a96 + gameplane.local/crd-bundle-sha256: 9ffb2bdfc486b12cb927fed97d0ce3f185b9cb06d7875092aa4c28b16490d752 controller-gen.kubebuilder.io/version: v0.22.0 name: clusters.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_gameservers.yaml b/charts/gameplane/crd-manifests/gameplane.local_gameservers.yaml index c9f4e670b..77f0f5626 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_gameservers.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_gameservers.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 3b2e34cfb50a4b2312502522937d45442c012cb313195cd2a556f69af01f3a96 + gameplane.local/crd-bundle-sha256: 9ffb2bdfc486b12cb927fed97d0ce3f185b9cb06d7875092aa4c28b16490d752 controller-gen.kubebuilder.io/version: v0.22.0 name: gameservers.gameplane.local spec: @@ -2434,13 +2434,13 @@ spec: type: integer diskTotalBytes: description: |- - DiskTotalBytes is the total size of the game data volume (statfs). + DiskTotalBytes is the observed capacity scoped to the game data volume. nil means "unknown". format: int64 type: integer diskUsedBytes: description: |- - DiskUsedBytes is the used space on the game data volume (statfs). + DiskUsedBytes is observed usage scoped to the game data directory. nil means "unknown". format: int64 type: integer diff --git a/charts/gameplane/crd-manifests/gameplane.local_gametemplates.yaml b/charts/gameplane/crd-manifests/gameplane.local_gametemplates.yaml index fb1cca85b..457defab0 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_gametemplates.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_gametemplates.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 3b2e34cfb50a4b2312502522937d45442c012cb313195cd2a556f69af01f3a96 + gameplane.local/crd-bundle-sha256: 9ffb2bdfc486b12cb927fed97d0ce3f185b9cb06d7875092aa4c28b16490d752 controller-gen.kubebuilder.io/version: v0.22.0 name: gametemplates.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_modules.yaml b/charts/gameplane/crd-manifests/gameplane.local_modules.yaml index 5d74582b1..e5510694d 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_modules.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_modules.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 3b2e34cfb50a4b2312502522937d45442c012cb313195cd2a556f69af01f3a96 + gameplane.local/crd-bundle-sha256: 9ffb2bdfc486b12cb927fed97d0ce3f185b9cb06d7875092aa4c28b16490d752 controller-gen.kubebuilder.io/version: v0.22.0 name: modules.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_modulesources.yaml b/charts/gameplane/crd-manifests/gameplane.local_modulesources.yaml index 8cf33a5ec..7e91cae57 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_modulesources.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_modulesources.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 3b2e34cfb50a4b2312502522937d45442c012cb313195cd2a556f69af01f3a96 + gameplane.local/crd-bundle-sha256: 9ffb2bdfc486b12cb927fed97d0ce3f185b9cb06d7875092aa4c28b16490d752 controller-gen.kubebuilder.io/version: v0.22.0 name: modulesources.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_networkcaptures.yaml b/charts/gameplane/crd-manifests/gameplane.local_networkcaptures.yaml index f2cd36c44..50f36554c 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_networkcaptures.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_networkcaptures.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 3b2e34cfb50a4b2312502522937d45442c012cb313195cd2a556f69af01f3a96 + gameplane.local/crd-bundle-sha256: 9ffb2bdfc486b12cb927fed97d0ce3f185b9cb06d7875092aa4c28b16490d752 controller-gen.kubebuilder.io/version: v0.22.0 name: networkcaptures.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_restores.yaml b/charts/gameplane/crd-manifests/gameplane.local_restores.yaml index d2f47f0ec..9a6a6aa88 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_restores.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_restores.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 3b2e34cfb50a4b2312502522937d45442c012cb313195cd2a556f69af01f3a96 + gameplane.local/crd-bundle-sha256: 9ffb2bdfc486b12cb927fed97d0ce3f185b9cb06d7875092aa4c28b16490d752 controller-gen.kubebuilder.io/version: v0.22.0 name: restores.gameplane.local spec: diff --git a/design.pen b/design.pen index 876a2bbb1..40d350102 100644 --- a/design.pen +++ b/design.pen @@ -1,5 +1,5 @@ { - "version": "2.21", + "version": "2.22", "children": [ { "type": "frame", @@ -1466,6 +1466,7 @@ "width": "fill_container", "gap": 0, "padding": 0, + "height": "fill_container", "children": [ { "type": "frame", @@ -1507,8 +1508,9 @@ { "type": "text", "id": "QLkWj", + "name": "Measured disk usage", "fill": "$foreground/foreground", - "content": "13%", + "content": "3.7 GB", "fontFamily": "$typography/font-mono", "fontSize": 24, "fontWeight": "700" @@ -1516,6 +1518,9 @@ { "type": "path", "id": "vdDJJ", + "x": 16, + "y": 80, + "enabled": false, "geometry": "M0 21l12-1 13 1 12-2 13 1 12-1 13 2 12-1 13 0", "viewBox": [ 0, @@ -1524,7 +1529,7 @@ 24 ], "fill": "#00000000", - "width": "fill_container", + "width": "fill_container(201.33333333333334)", "height": 24, "stroke": "$success/soft-foreground", "strokeWidth": 1.5 @@ -1532,8 +1537,11 @@ { "type": "frame", "id": "JpkaE", + "x": 16, + "y": 112, + "enabled": false, "clip": true, - "width": "fill_container", + "width": "fill_container(201.33333333333334)", "height": 6, "fill": "$default/default", "cornerRadius": 999, @@ -1551,8 +1559,9 @@ { "type": "text", "id": "gijqx", + "name": "Requested PVC capacity", "fill": "$muted", - "content": "3.7 GB / 29 GB", + "content": "30Gi requested", "fontFamily": "$typography/font-sans", "fontSize": 12, "fontWeight": "normal" @@ -37027,6 +37036,7 @@ "id": "Mcf1C", "name": "Metric CPU", "width": "fill_container", + "height": "fill_container", "fill": "$surface/surface", "stroke": "$border/border", "strokeWidth": 1, @@ -37154,6 +37164,7 @@ "id": "QGBuR", "name": "Metric Memory", "width": "fill_container", + "height": "fill_container", "fill": "$surface/surface", "stroke": "$border/border", "strokeWidth": 1, @@ -37390,11 +37401,9 @@ { "type": "text", "id": "bsk1A", - "x": 16, - "y": 126, - "enabled": false, + "name": "Requested PVC capacity", "fill": "$muted", - "content": "3.7 GB / 29 GB", + "content": "30Gi requested", "fontFamily": "$typography/font-sans", "fontSize": 12, "fontWeight": "normal" diff --git a/i18n/locales/en/app.json b/i18n/locales/en/app.json index af51c12f9..fedc7b86e 100644 --- a/i18n/locales/en/app.json +++ b/i18n/locales/en/app.json @@ -8843,6 +8843,10 @@ "defaultMessage": "Recent events", "description": "Overview tab: small uppercase heading of the recent events card" }, + "web.routes.tabs.overview.storage_requested": { + "defaultMessage": "{size} requested", + "description": "Requested size of the server data volume when measured capacity is unavailable" + }, "web.routes.tabs.overview.tailnet_only": { "defaultMessage": "Tailnet only", "description": "Overview tab: warning badge beside a tunnel address meaning it is reachable only inside the tailnet; keep it short" diff --git a/operator/api/v1alpha1/gameserver_types.go b/operator/api/v1alpha1/gameserver_types.go index a8683597b..1043dd4a5 100644 --- a/operator/api/v1alpha1/gameserver_types.go +++ b/operator/api/v1alpha1/gameserver_types.go @@ -759,12 +759,12 @@ type AgentStatus struct { // +optional MemoryLimitBytes *int64 `json:"memoryLimitBytes,omitempty"` - // DiskUsedBytes is the used space on the game data volume (statfs). + // DiskUsedBytes is observed usage scoped to the game data directory. // nil means "unknown". // +optional DiskUsedBytes *int64 `json:"diskUsedBytes,omitempty"` - // DiskTotalBytes is the total size of the game data volume (statfs). + // DiskTotalBytes is the observed capacity scoped to the game data volume. // nil means "unknown". // +optional DiskTotalBytes *int64 `json:"diskTotalBytes,omitempty"` diff --git a/operator/config/crd/gameplane.local_gameservers.yaml b/operator/config/crd/gameplane.local_gameservers.yaml index 2e97d8a55..c0d0d8b10 100644 --- a/operator/config/crd/gameplane.local_gameservers.yaml +++ b/operator/config/crd/gameplane.local_gameservers.yaml @@ -2433,13 +2433,13 @@ spec: type: integer diskTotalBytes: description: |- - DiskTotalBytes is the total size of the game data volume (statfs). + DiskTotalBytes is the observed capacity scoped to the game data volume. nil means "unknown". format: int64 type: integer diskUsedBytes: description: |- - DiskUsedBytes is the used space on the game data volume (statfs). + DiskUsedBytes is observed usage scoped to the game data directory. nil means "unknown". format: int64 type: integer diff --git a/web/e2e/screenshots/slice2a.spec.ts b/web/e2e/screenshots/slice2a.spec.ts index 9f85525af..a24f105bf 100644 --- a/web/e2e/screenshots/slice2a.spec.ts +++ b/web/e2e/screenshots/slice2a.spec.ts @@ -62,7 +62,10 @@ test.describe("Slice 2a: Servers + core tabs (Desktop — 1440x900) @screenshots await capture(page, "F9pUrx"); }); - test("EZFW0: Server Detail — Overview (running)", async ({ page }) => { + test("EZFW0: Server Detail — Overview (running)", async ({ context, page }) => { + await context.addCookies([ + { name: "e2e_server_variant", value: "shared-storage", domain: "localhost", path: "/" }, + ]); await page.goto("/servers/mc-survival"); await expect(page.getByRole("heading", { name: "mc-survival" })).toBeVisible({ timeout: 10_000, @@ -70,6 +73,8 @@ test.describe("Slice 2a: Servers + core tabs (Desktop — 1440x900) @screenshots await expect(page.getByText("Connection")).toBeVisible(); await expect(page.getByText("Recent events")).toBeVisible(); await page.waitForTimeout(200); + await expect(page.getByText("3.7 GB", { exact: true })).toBeVisible(); + await expect(page.getByText("30Gi requested", { exact: true })).toBeVisible(); await capture(page, "EZFW0"); }); @@ -99,6 +104,7 @@ test.describe("Slice 2a: Servers + core tabs (Desktop — 1440x900) @screenshots await expect(page.getByText("Asleep", { exact: true }).first()).toBeVisible({ timeout: 10_000 }); await expect(page.getByText("Asleep since")).toBeVisible(); await page.waitForTimeout(200); + await expect(page.getByText("30Gi requested", { exact: true })).toBeVisible(); await capture(page, "TE2jI"); }); diff --git a/web/src/routes/tabs/Overview.test.tsx b/web/src/routes/tabs/Overview.test.tsx index 6fcc31de0..f62cd48c1 100644 --- a/web/src/routes/tabs/Overview.test.tsx +++ b/web/src/routes/tabs/Overview.test.tsx @@ -56,6 +56,15 @@ describe("OverviewTab recent events", () => { }); describe("OverviewTab metric tiles", () => { + it("shows measured bytes and the requested claim size when capacity is unknown", async () => { + fetchMock.mockImplementation(() => Promise.resolve(jsonRes([]))); + const server = gs({ phase: "Starting", agent: { lastHeartbeat: "now", diskUsedBytes: 0, diskTotalBytes: null } }); + server.spec.storage = { size: "10Gi" }; + renderWithQuery(); + expect(await screen.findByText("10Gi requested")).toBeInTheDocument(); + expect(screen.getByText("0 B")).toBeInTheDocument(); + }); + it("renders real CPU, memory and disk usage from the heartbeat", async () => { fetchMock.mockImplementation(() => Promise.resolve(jsonRes({ online: 0, max: 20, players: [], asOf: "now" }))); @@ -105,7 +114,7 @@ describe("OverviewTab metric tiles", () => { lastHeartbeat: "now", cpuMillicores: 1500, // no limit → "1.50 cores", no bar memoryBytes: 536870912, // no limit → "512 MB" primary + "512 MB / —" - diskUsedBytes: 1073741824, diskTotalBytes: 0, // no total → "1.0 GB" primary + "1.0 GB / —" + diskUsedBytes: 1073741824, diskTotalBytes: null, // A directory scan observes usage only. playersOnline: 3, // exercises the players fallback branch }, })} />); @@ -113,7 +122,7 @@ describe("OverviewTab metric tiles", () => { expect(screen.getByText("512 MB")).toBeInTheDocument(); expect(screen.getByText("512 MB / —")).toBeInTheDocument(); expect(screen.getByText("1.0 GB")).toBeInTheDocument(); - expect(screen.getByText("1.0 GB / —")).toBeInTheDocument(); + expect(screen.queryByText("1.0 GB / —")).not.toBeInTheDocument(); // Players fallback (phase != Running) feeds the sidebar card. expect(screen.getByText(/names not yet available/)).toBeInTheDocument(); }); diff --git a/web/src/routes/tabs/Overview.tsx b/web/src/routes/tabs/Overview.tsx index ad6cd7922..52e46c659 100644 --- a/web/src/routes/tabs/Overview.tsx +++ b/web/src/routes/tabs/Overview.tsx @@ -84,10 +84,10 @@ export function OverviewTab({ typeof agent?.memoryLimitBytes === "number" ? (agent?.memoryLimitBytes as number) : 0; const memPct = memKnown && memLimit ? (memUsed / memLimit) * 100 : 0; - const diskKnown = - typeof agent?.diskUsedBytes === "number" && typeof agent?.diskTotalBytes === "number"; + const diskKnown = typeof agent?.diskUsedBytes === "number"; const diskUsed = diskKnown ? (agent?.diskUsedBytes as number) : 0; - const diskTotal = diskKnown ? (agent?.diskTotalBytes as number) : 0; + const diskTotal = agent?.diskTotalBytes ?? 0; + const requestedStorage = gs?.spec.storage?.size; const diskPct = diskKnown && diskTotal ? (diskUsed / diskTotal) * 100 : 0; // Player count is shown in the Players card (sidebar). null/undefined @@ -165,7 +165,9 @@ export function OverviewTab({ icon={} value={diskKnown ? (diskTotal ? `${diskPct.toFixed(0)}%` : formatBytes(diskUsed)) : intl.formatMessage(OVERVIEW_TEXT.empty_value)} sub={ - diskKnown ? intl.formatMessage({ id: "web.routes.tabs.overview.used_of_total", defaultMessage: "{used} / {total}", description: "Overview: memory or disk usage against its limit; {used} is the amount in use, {total} the limit" }, { used: formatBytes(diskUsed), total: diskTotal ? formatBytes(diskTotal) : intl.formatMessage(OVERVIEW_TEXT.empty_value) }) : undefined + diskKnown && diskTotal > 0 ? intl.formatMessage({ id: "web.routes.tabs.overview.used_of_total", defaultMessage: "{used} / {total}", description: "Overview: memory or disk usage against its limit; {used} is the amount in use, {total} the limit" }, { used: formatBytes(diskUsed), total: formatBytes(diskTotal) }) + : requestedStorage ? intl.formatMessage({ id: "web.routes.tabs.overview.storage_requested", defaultMessage: "{size} requested", description: "Requested size of the server data volume when measured capacity is unavailable" }, { size: requestedStorage }) + : undefined } progress={diskKnown && diskTotal ? diskPct : undefined} trend={diskKnown && diskTotal ? diskPct : undefined} diff --git a/web/src/test/handlers.ts b/web/src/test/handlers.ts index 1186a0f23..fdcf1484d 100644 --- a/web/src/test/handlers.ts +++ b/web/src/test/handlers.ts @@ -940,6 +940,16 @@ export function buildScreenshotHandlers() { } if (name === "mc-survival" && cookies.e2e_server_variant) { const variant = cookies.e2e_server_variant; + if (variant === "shared-storage") { + return HttpResponse.json({ + ...server, + spec: { ...server.spec, storage: { ...server.spec.storage, size: "30Gi" } }, + status: { + ...server.status, + agent: { ...server.status?.agent, diskUsedBytes: 3_972_844_749, diskTotalBytes: null }, + }, + }); + } if (variant === "idle") { return HttpResponse.json({ ...server, @@ -962,6 +972,7 @@ export function buildScreenshotHandlers() { spec: { ...server.spec, suspend: true, + storage: { ...server.spec.storage, size: "30Gi" }, }, status: { ...server.status, @@ -974,6 +985,8 @@ export function buildScreenshotHandlers() { ...server.status?.agent, playersOnline: null, playersMax: 0, + diskUsedBytes: null, + diskTotalBytes: null, }, startedAt: undefined, },