feat: expose snapshot sizes in metrics and webhooks (upstream #1283)

restic/cli/stats.go

@@ -30,6 +30,7 @@ type RawMetrics struct {
 	ChangedDirs           float64  `json:"changed_dirs"`
 	UnmodifiedDirs        float64  `json:"unmodified_dirs"`
 	DataTransferred       float64  `json:"data_transferred"`
+	TotalBytesProcessed   float64  `json:"total_bytes_processed"`
 	MountedPVCs           []string `json:"mounted_PVCs"`
 	availableSnapshots    float64
 	Folder                string
@@ -45,15 +46,16 @@ type BackupStats struct {
 }
 
 type PromMetrics struct {
-	Errors             *prometheus.GaugeVec
-	AvailableSnapshots prometheus.Gauge
-	NewFiles           *prometheus.GaugeVec
-	ChangedFiles       *prometheus.GaugeVec
-	UnmodifiedFiles    *prometheus.GaugeVec
-	NewDirs            *prometheus.GaugeVec
-	ChangedDirs        *prometheus.GaugeVec
-	UnmodifiedDirs     *prometheus.GaugeVec
-	DataTransferred    *prometheus.GaugeVec
+	Errors              *prometheus.GaugeVec
+	AvailableSnapshots  prometheus.Gauge
+	NewFiles            *prometheus.GaugeVec
+	ChangedFiles        *prometheus.GaugeVec
+	UnmodifiedFiles     *prometheus.GaugeVec
+	NewDirs             *prometheus.GaugeVec
+	ChangedDirs         *prometheus.GaugeVec
+	UnmodifiedDirs      *prometheus.GaugeVec
+	DataTransferred     *prometheus.GaugeVec
+	TotalBytesProcessed *prometheus.GaugeVec
 }
 
 func newPromMetrics() *PromMetrics {
@@ -64,15 +66,16 @@ func newPromMetrics() *PromMetrics {
 	}
 
 	return &PromMetrics{
-		Errors:             errorsGaugeVec(labels),
-		AvailableSnapshots: availableSnapshotsGauge(),
-		NewFiles:           newFilesGaugeVec(labels),
-		ChangedFiles:       changedFiledGaugeVec(labels),
-		UnmodifiedFiles:    unmodifiedFilesGaugeVec(labels),
-		NewDirs:            newDirsGaugeVec(labels),
-		ChangedDirs:        changedDirsGaugeVec(labels),
-		UnmodifiedDirs:     unmodifiedDirsGaugeVec(labels),
-		DataTransferred:    dataTransferredGaugeVec(labels),
+		Errors:              errorsGaugeVec(labels),
+		AvailableSnapshots:  availableSnapshotsGauge(),
+		NewFiles:            newFilesGaugeVec(labels),
+		ChangedFiles:        changedFiledGaugeVec(labels),
+		UnmodifiedFiles:     unmodifiedFilesGaugeVec(labels),
+		NewDirs:             newDirsGaugeVec(labels),
+		ChangedDirs:         changedDirsGaugeVec(labels),
+		UnmodifiedDirs:      unmodifiedDirsGaugeVec(labels),
+		DataTransferred:     dataTransferredGaugeVec(labels),
+		TotalBytesProcessed: totalBytesProcessedGaugeVec(labels),
 	}
 }
 
@@ -85,6 +88,15 @@ func dataTransferredGaugeVec(labels []string) *prometheus.GaugeVec {
 	}, labels)
 }
 
+func totalBytesProcessedGaugeVec(labels []string) *prometheus.GaugeVec {
+	return prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: prometheusNamespace,
+		Subsystem: prometheusSubsystem,
+		Name:      "total_bytes_processed_during_backup",
+		Help:      "Total size in bytes of the data read during the last backup",
+	}, labels)
+}
+
 func unmodifiedDirsGaugeVec(labels []string) *prometheus.GaugeVec {
 	return prometheus.NewGaugeVec(prometheus.GaugeOpts{
 		Namespace: prometheusNamespace,
@@ -175,6 +187,7 @@ func (r *Restic) parseSummary(summary logging.BackupSummary, errorCount int, fol
 		BackupEndTimestamp:    float64(endTimestamp),
 		runningBackupDuration: summary.TotalDuration,
 		DataTransferred:       float64(summary.DataAdded),
+		TotalBytesProcessed:   float64(summary.TotalBytesProcessed),
 		ID:                    summary.SnapshotID,
 	}
 }
@@ -190,6 +203,8 @@ func (r *RawMetrics) prometheus() *PromMetrics {
 	metrics.NewFiles.WithLabelValues(r.Folder, r.hostname).Set(r.NewFiles)
 	metrics.UnmodifiedDirs.WithLabelValues(r.Folder, r.hostname).Set(r.UnmodifiedDirs)
 	metrics.UnmodifiedFiles.WithLabelValues(r.Folder, r.hostname).Set(r.UnmodifiedFiles)
+	metrics.DataTransferred.WithLabelValues(r.Folder, r.hostname).Set(r.DataTransferred)
+	metrics.TotalBytesProcessed.WithLabelValues(r.Folder, r.hostname).Set(r.TotalBytesProcessed)
 
 	return metrics
 }
@@ -214,6 +229,7 @@ func (p *PromMetrics) ToProm() []prometheus.Collector {
 		p.ChangedDirs,
 		p.UnmodifiedDirs,
 		p.DataTransferred,
+		p.TotalBytesProcessed,
 	}
 }
 

restic/cli/stats_test.go

@@ -0,0 +1,44 @@
+package cli
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/prometheus/client_golang/prometheus/testutil"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRawMetrics_PrometheusSetsByteGauges(t *testing.T) {
+	raw := &RawMetrics{
+		DataTransferred:     1024,
+		TotalBytesProcessed: 4096,
+		Folder:              "data-pvc",
+		hostname:            "my-namespace",
+	}
+
+	metrics := raw.prometheus()
+
+	assert.Equal(t, float64(1024), testutil.ToFloat64(metrics.DataTransferred.WithLabelValues("data-pvc", "my-namespace")))
+	assert.Equal(t, float64(4096), testutil.ToFloat64(metrics.TotalBytesProcessed.WithLabelValues("data-pvc", "my-namespace")))
+	assert.Equal(t, 1, testutil.CollectAndCount(metrics.DataTransferred))
+	assert.Equal(t, 1, testutil.CollectAndCount(metrics.TotalBytesProcessed))
+}
+
+func TestBackupStats_ToJSONIncludesByteFields(t *testing.T) {
+	stats := &BackupStats{
+		Name: "my-namespace",
+		BackupMetrics: &RawMetrics{
+			DataTransferred:     1024,
+			TotalBytesProcessed: 4096,
+		},
+	}
+
+	var payload struct {
+		BackupMetrics map[string]any `json:"backup_metrics"`
+	}
+	require.NoError(t, json.Unmarshal(stats.ToJSON(), &payload))
+
+	assert.Equal(t, float64(1024), payload.BackupMetrics["data_transferred"])
+	assert.Equal(t, float64(4096), payload.BackupMetrics["total_bytes_processed"])
+}

restic/dto/snapshot.go

@@ -14,4 +14,25 @@ type Snapshot struct {
 	UID      int       `json:"uid"`
 	Gid      int       `json:"gid"`
 	Tags     []string  `json:"tags"`
+	// Summary is only present for snapshots created by restic >= 0.17.
+	Summary *SnapshotSummary `json:"summary,omitempty"`
+}
+
+// SnapshotSummary holds the backup statistics restic stores in each
+// snapshot (restic >= 0.17).
+type SnapshotSummary struct {
+	BackupStart         time.Time `json:"backup_start"`
+	BackupEnd           time.Time `json:"backup_end"`
+	FilesNew            uint      `json:"files_new"`
+	FilesChanged        uint      `json:"files_changed"`
+	FilesUnmodified     uint      `json:"files_unmodified"`
+	DirsNew             uint      `json:"dirs_new"`
+	DirsChanged         uint      `json:"dirs_changed"`
+	DirsUnmodified      uint      `json:"dirs_unmodified"`
+	DataBlobs           int       `json:"data_blobs"`
+	TreeBlobs           int       `json:"tree_blobs"`
+	DataAdded           uint64    `json:"data_added"`
+	DataAddedPacked     uint64    `json:"data_added_packed"`
+	TotalFilesProcessed uint      `json:"total_files_processed"`
+	TotalBytesProcessed uint64    `json:"total_bytes_processed"`
 }

restic/dto/snapshot_test.go

@@ -0,0 +1,73 @@
+package dto
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// Shape of `restic snapshots --json` output from restic >= 0.17: the first
+// snapshot was created by restic >= 0.17 and carries a summary, the second
+// one predates it and doesn't.
+const resticSnapshotsJSON = `[
+  {
+    "time": "2026-10-24T10:00:00.123456789Z",
+    "tree": "aaaa",
+    "paths": ["/data/data-pvc"],
+    "hostname": "my-namespace",
+    "username": "root",
+    "id": "1111111111111111111111111111111111111111111111111111111111111111",
+    "short_id": "11111111",
+    "summary": {
+      "backup_start": "2026-10-24T09:59:00Z",
+      "backup_end": "2026-10-24T10:00:00Z",
+      "files_new": 3,
+      "files_changed": 1,
+      "files_unmodified": 100,
+      "dirs_new": 0,
+      "dirs_changed": 1,
+      "dirs_unmodified": 10,
+      "data_blobs": 4,
+      "tree_blobs": 2,
+      "data_added": 2048,
+      "data_added_packed": 1024,
+      "total_files_processed": 104,
+      "total_bytes_processed": 524288000
+    }
+  },
+  {
+    "time": "2023-01-01T00:00:00Z",
+    "tree": "bbbb",
+    "paths": ["/data/data-pvc"],
+    "hostname": "my-namespace",
+    "id": "2222222222222222222222222222222222222222222222222222222222222222",
+    "short_id": "22222222"
+  }
+]`
+
+func TestSnapshot_UnmarshalSummary(t *testing.T) {
+	var snapshots []Snapshot
+	require.NoError(t, json.Unmarshal([]byte(resticSnapshotsJSON), &snapshots))
+	require.Len(t, snapshots, 2)
+
+	withSummary := snapshots[0]
+	require.NotNil(t, withSummary.Summary)
+	assert.Equal(t, uint64(524288000), withSummary.Summary.TotalBytesProcessed)
+	assert.Equal(t, uint64(2048), withSummary.Summary.DataAdded)
+	assert.Equal(t, uint64(1024), withSummary.Summary.DataAddedPacked)
+	assert.Equal(t, uint(104), withSummary.Summary.TotalFilesProcessed)
+
+	assert.Nil(t, snapshots[1].Summary, "snapshots from restic < 0.17 have no summary")
+}
+
+func TestSnapshot_MarshalOmitsMissingSummary(t *testing.T) {
+	out, err := json.Marshal(Snapshot{ID: "abc"})
+	require.NoError(t, err)
+	assert.NotContains(t, string(out), "summary")
+
+	out, err = json.Marshal(Snapshot{ID: "abc", Summary: &SnapshotSummary{TotalBytesProcessed: 42}})
+	require.NoError(t, err)
+	assert.Contains(t, string(out), `"total_bytes_processed":42`)
+}

restic/logging/logging.go

@@ -65,6 +65,8 @@ type BackupError struct {
 		Op   string `json:"Op"`
 		Path string `json:"Path"`
 		Err  int    `json:"Err"`
+		// Message is used instead of Op/Path/Err by restic >= 0.17.
+		Message string `json:"message"`
 	} `json:"error"`
 	During string `json:"during"`
 	Item   string `json:"item"`
@@ -153,7 +155,7 @@ func (b *BackupOutputParser) out(s string) {
 	switch envelope.MessageType {
 	case "error":
 		b.errorCount++
-		b.log.Error(fmt.Errorf("error occurred during backup"), envelope.Item+" during "+envelope.During+" "+envelope.Error.Op)
+		b.log.Error(fmt.Errorf("error occurred during backup"), envelope.Item+" during "+envelope.During+" "+envelope.Error.Op+envelope.Error.Message)
 	case "status":
 		b.percentageFunc(b.log, envelope.PercentDone)
 	case "summary":