From 0682f043194e61ac2df3c8f2f2d1d4ed54748f89 Mon Sep 17 00:00:00 2001 From: shreddedbacon Date: Sun, 25 Oct 2026 08:56:59 +1100 Subject: [PATCH] feat: add snapshot summaries to metrics and webhooks --- restic/cli/stats.go | 52 +++++++++++++++++--------- restic/cli/stats_test.go | 44 ++++++++++++++++++++++ restic/dto/snapshot.go | 21 +++++++++++ restic/dto/snapshot_test.go | 73 +++++++++++++++++++++++++++++++++++++ restic/logging/logging.go | 4 +- 5 files changed, 175 insertions(+), 19 deletions(-) create mode 100644 restic/cli/stats_test.go create mode 100644 restic/dto/snapshot_test.go diff --git a/restic/cli/stats.go b/restic/cli/stats.go index bdb903dce..2454d5c9c 100644 --- a/restic/cli/stats.go +++ b/restic/cli/stats.go @@ -30,6 +30,7 @@ type RawMetrics struct { ChangedDirs float64 `json:"changed_dirs"` UnmodifiedDirs float64 `json:"unmodified_dirs"` DataTransferred float64 `json:"data_transferred"` + TotalBytesProcessed float64 `json:"total_bytes_processed"` MountedPVCs []string `json:"mounted_PVCs"` availableSnapshots float64 Folder string @@ -45,15 +46,16 @@ type BackupStats struct { } type PromMetrics struct { - Errors *prometheus.GaugeVec - AvailableSnapshots prometheus.Gauge - NewFiles *prometheus.GaugeVec - ChangedFiles *prometheus.GaugeVec - UnmodifiedFiles *prometheus.GaugeVec - NewDirs *prometheus.GaugeVec - ChangedDirs *prometheus.GaugeVec - UnmodifiedDirs *prometheus.GaugeVec - DataTransferred *prometheus.GaugeVec + Errors *prometheus.GaugeVec + AvailableSnapshots prometheus.Gauge + NewFiles *prometheus.GaugeVec + ChangedFiles *prometheus.GaugeVec + UnmodifiedFiles *prometheus.GaugeVec + NewDirs *prometheus.GaugeVec + ChangedDirs *prometheus.GaugeVec + UnmodifiedDirs *prometheus.GaugeVec + DataTransferred *prometheus.GaugeVec + TotalBytesProcessed *prometheus.GaugeVec } func newPromMetrics() *PromMetrics { @@ -64,15 +66,16 @@ func newPromMetrics() *PromMetrics { } return &PromMetrics{ - Errors: errorsGaugeVec(labels), - AvailableSnapshots: availableSnapshotsGauge(), - NewFiles: newFilesGaugeVec(labels), - ChangedFiles: changedFiledGaugeVec(labels), - UnmodifiedFiles: unmodifiedFilesGaugeVec(labels), - NewDirs: newDirsGaugeVec(labels), - ChangedDirs: changedDirsGaugeVec(labels), - UnmodifiedDirs: unmodifiedDirsGaugeVec(labels), - DataTransferred: dataTransferredGaugeVec(labels), + Errors: errorsGaugeVec(labels), + AvailableSnapshots: availableSnapshotsGauge(), + NewFiles: newFilesGaugeVec(labels), + ChangedFiles: changedFiledGaugeVec(labels), + UnmodifiedFiles: unmodifiedFilesGaugeVec(labels), + NewDirs: newDirsGaugeVec(labels), + ChangedDirs: changedDirsGaugeVec(labels), + UnmodifiedDirs: unmodifiedDirsGaugeVec(labels), + DataTransferred: dataTransferredGaugeVec(labels), + TotalBytesProcessed: totalBytesProcessedGaugeVec(labels), } } @@ -85,6 +88,15 @@ func dataTransferredGaugeVec(labels []string) *prometheus.GaugeVec { }, labels) } +func totalBytesProcessedGaugeVec(labels []string) *prometheus.GaugeVec { + return prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: prometheusNamespace, + Subsystem: prometheusSubsystem, + Name: "total_bytes_processed_during_backup", + Help: "Total size in bytes of the data read during the last backup", + }, labels) +} + func unmodifiedDirsGaugeVec(labels []string) *prometheus.GaugeVec { return prometheus.NewGaugeVec(prometheus.GaugeOpts{ Namespace: prometheusNamespace, @@ -175,6 +187,7 @@ func (r *Restic) parseSummary(summary logging.BackupSummary, errorCount int, fol BackupEndTimestamp: float64(endTimestamp), runningBackupDuration: summary.TotalDuration, DataTransferred: float64(summary.DataAdded), + TotalBytesProcessed: float64(summary.TotalBytesProcessed), ID: summary.SnapshotID, } } @@ -190,6 +203,8 @@ func (r *RawMetrics) prometheus() *PromMetrics { metrics.NewFiles.WithLabelValues(r.Folder, r.hostname).Set(r.NewFiles) metrics.UnmodifiedDirs.WithLabelValues(r.Folder, r.hostname).Set(r.UnmodifiedDirs) metrics.UnmodifiedFiles.WithLabelValues(r.Folder, r.hostname).Set(r.UnmodifiedFiles) + metrics.DataTransferred.WithLabelValues(r.Folder, r.hostname).Set(r.DataTransferred) + metrics.TotalBytesProcessed.WithLabelValues(r.Folder, r.hostname).Set(r.TotalBytesProcessed) return metrics } @@ -214,6 +229,7 @@ func (p *PromMetrics) ToProm() []prometheus.Collector { p.ChangedDirs, p.UnmodifiedDirs, p.DataTransferred, + p.TotalBytesProcessed, } } diff --git a/restic/cli/stats_test.go b/restic/cli/stats_test.go new file mode 100644 index 000000000..011417c16 --- /dev/null +++ b/restic/cli/stats_test.go @@ -0,0 +1,44 @@ +package cli + +import ( + "encoding/json" + "testing" + + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestRawMetrics_PrometheusSetsByteGauges(t *testing.T) { + raw := &RawMetrics{ + DataTransferred: 1024, + TotalBytesProcessed: 4096, + Folder: "data-pvc", + hostname: "my-namespace", + } + + metrics := raw.prometheus() + + assert.Equal(t, float64(1024), testutil.ToFloat64(metrics.DataTransferred.WithLabelValues("data-pvc", "my-namespace"))) + assert.Equal(t, float64(4096), testutil.ToFloat64(metrics.TotalBytesProcessed.WithLabelValues("data-pvc", "my-namespace"))) + assert.Equal(t, 1, testutil.CollectAndCount(metrics.DataTransferred)) + assert.Equal(t, 1, testutil.CollectAndCount(metrics.TotalBytesProcessed)) +} + +func TestBackupStats_ToJSONIncludesByteFields(t *testing.T) { + stats := &BackupStats{ + Name: "my-namespace", + BackupMetrics: &RawMetrics{ + DataTransferred: 1024, + TotalBytesProcessed: 4096, + }, + } + + var payload struct { + BackupMetrics map[string]any `json:"backup_metrics"` + } + require.NoError(t, json.Unmarshal(stats.ToJSON(), &payload)) + + assert.Equal(t, float64(1024), payload.BackupMetrics["data_transferred"]) + assert.Equal(t, float64(4096), payload.BackupMetrics["total_bytes_processed"]) +} diff --git a/restic/dto/snapshot.go b/restic/dto/snapshot.go index 9b144224c..1461ed569 100644 --- a/restic/dto/snapshot.go +++ b/restic/dto/snapshot.go @@ -14,4 +14,25 @@ type Snapshot struct { UID int `json:"uid"` Gid int `json:"gid"` Tags []string `json:"tags"` + // Summary is only present for snapshots created by restic >= 0.17. + Summary *SnapshotSummary `json:"summary,omitempty"` +} + +// SnapshotSummary holds the backup statistics restic stores in each +// snapshot (restic >= 0.17). +type SnapshotSummary struct { + BackupStart time.Time `json:"backup_start"` + BackupEnd time.Time `json:"backup_end"` + FilesNew uint `json:"files_new"` + FilesChanged uint `json:"files_changed"` + FilesUnmodified uint `json:"files_unmodified"` + DirsNew uint `json:"dirs_new"` + DirsChanged uint `json:"dirs_changed"` + DirsUnmodified uint `json:"dirs_unmodified"` + DataBlobs int `json:"data_blobs"` + TreeBlobs int `json:"tree_blobs"` + DataAdded uint64 `json:"data_added"` + DataAddedPacked uint64 `json:"data_added_packed"` + TotalFilesProcessed uint `json:"total_files_processed"` + TotalBytesProcessed uint64 `json:"total_bytes_processed"` } diff --git a/restic/dto/snapshot_test.go b/restic/dto/snapshot_test.go new file mode 100644 index 000000000..6b86f5d14 --- /dev/null +++ b/restic/dto/snapshot_test.go @@ -0,0 +1,73 @@ +package dto + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Shape of `restic snapshots --json` output from restic >= 0.17: the first +// snapshot was created by restic >= 0.17 and carries a summary, the second +// one predates it and doesn't. +const resticSnapshotsJSON = `[ + { + "time": "2026-10-24T10:00:00.123456789Z", + "tree": "aaaa", + "paths": ["/data/data-pvc"], + "hostname": "my-namespace", + "username": "root", + "id": "1111111111111111111111111111111111111111111111111111111111111111", + "short_id": "11111111", + "summary": { + "backup_start": "2026-10-24T09:59:00Z", + "backup_end": "2026-10-24T10:00:00Z", + "files_new": 3, + "files_changed": 1, + "files_unmodified": 100, + "dirs_new": 0, + "dirs_changed": 1, + "dirs_unmodified": 10, + "data_blobs": 4, + "tree_blobs": 2, + "data_added": 2048, + "data_added_packed": 1024, + "total_files_processed": 104, + "total_bytes_processed": 524288000 + } + }, + { + "time": "2023-01-01T00:00:00Z", + "tree": "bbbb", + "paths": ["/data/data-pvc"], + "hostname": "my-namespace", + "id": "2222222222222222222222222222222222222222222222222222222222222222", + "short_id": "22222222" + } +]` + +func TestSnapshot_UnmarshalSummary(t *testing.T) { + var snapshots []Snapshot + require.NoError(t, json.Unmarshal([]byte(resticSnapshotsJSON), &snapshots)) + require.Len(t, snapshots, 2) + + withSummary := snapshots[0] + require.NotNil(t, withSummary.Summary) + assert.Equal(t, uint64(524288000), withSummary.Summary.TotalBytesProcessed) + assert.Equal(t, uint64(2048), withSummary.Summary.DataAdded) + assert.Equal(t, uint64(1024), withSummary.Summary.DataAddedPacked) + assert.Equal(t, uint(104), withSummary.Summary.TotalFilesProcessed) + + assert.Nil(t, snapshots[1].Summary, "snapshots from restic < 0.17 have no summary") +} + +func TestSnapshot_MarshalOmitsMissingSummary(t *testing.T) { + out, err := json.Marshal(Snapshot{ID: "abc"}) + require.NoError(t, err) + assert.NotContains(t, string(out), "summary") + + out, err = json.Marshal(Snapshot{ID: "abc", Summary: &SnapshotSummary{TotalBytesProcessed: 42}}) + require.NoError(t, err) + assert.Contains(t, string(out), `"total_bytes_processed":42`) +} diff --git a/restic/logging/logging.go b/restic/logging/logging.go index b0c2b1463..0c75b14bf 100644 --- a/restic/logging/logging.go +++ b/restic/logging/logging.go @@ -65,6 +65,8 @@ type BackupError struct { Op string `json:"Op"` Path string `json:"Path"` Err int `json:"Err"` + // Message is used instead of Op/Path/Err by restic >= 0.17. + Message string `json:"message"` } `json:"error"` During string `json:"during"` Item string `json:"item"` @@ -153,7 +155,7 @@ func (b *BackupOutputParser) out(s string) { switch envelope.MessageType { case "error": b.errorCount++ - b.log.Error(fmt.Errorf("error occurred during backup"), envelope.Item+" during "+envelope.During+" "+envelope.Error.Op) + b.log.Error(fmt.Errorf("error occurred during backup"), envelope.Item+" during "+envelope.During+" "+envelope.Error.Op+envelope.Error.Message) case "status": b.percentageFunc(b.log, envelope.PercentDone) case "summary":