feat(drivers): one place to see and choose a driver version

.changeset/driver-versions-one-place.md

@@ -0,0 +1,13 @@
+---
+"ftw": minor
+---
+
+Driver versions have one place: each device's Versions list under Settings →
+Devices. It shows the release's own copy, the signed stable versions and any beta
+newer than stable, each with a "What changed" link to the driver's history, and a
+button to check the signed channels for new versions. The device line says where
+the running driver comes from and whether a chosen version is kept across
+updates; `ftw status` says the same. Update Center and System no longer list or
+refresh drivers, and no new version lights up the header. Adding a device can
+fetch driver types the release does not carry, from the driver channel or, marked,
+from beta, in the same list.

docs/device-repository.md

@@ -35,6 +35,11 @@ or an update trial that falls back, therefore runs the selection again.
 `ftw status` lists the version each configured driver runs and, for an
 override, the release's own version.
 
+Settings → Devices is the one place a driver version is seen and changed.
+Each device's Versions list shows the release's copy, the signed stable
+versions and any beta newer than stable, with a link to what changed. Nothing
+there announces a new version; the owner checks and picks.
+
 Refreshing the signed manifest only updates discovery data. It never installs,
 activates or restarts a driver. FTW verifies the Ed25519 signature, driver ID,
 SemVer, host API range, URL, file size and SHA-256. Installation then compiles

go/internal/api/api.go

@@ -1907,6 +1907,9 @@ func (s *Server) handleDriversCatalog(w http.ResponseWriter, r *http.Request) {
 		if release, err := drivers.ParseCatalogFile(filepath.Join(dir, filepath.FromSlash(rel))); err == nil {
 			entries[i].ReleaseVersion = release.Version
 		}
+		if entries[i].Source == "managed" && s.deps.DriverRepository != nil {
+			entries[i].Chosen = s.deps.DriverRepository.Chosen(entries[i].Path, entries[i].Version)
+		}
 	}
 	writeJSON(w, 200, map[string]any{"path": dir, "entries": entries})
 }

go/internal/api/api_device_repository.go

@@ -2,6 +2,7 @@ package api
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"net/http"
@@ -64,11 +65,22 @@ func (s *Server) handleDeviceRepositoryRefresh(w http.ResponseWriter, r *http.Re
 			return
 		}
 	}
-	if err := s.deps.DriverRepository.Refresh(r.Context(), body.RepositoryID); err != nil {
+	// Without a repository this is the owner's "check for new versions",
+	// which also reads the beta channel; a beta outage is only a warning.
+	var warnings []string
+	var err error
+	if body.RepositoryID == "" {
+		warnings, err = s.deps.DriverRepository.RefreshAll(r.Context())
+	} else {
+		err = s.deps.DriverRepository.Refresh(r.Context(), body.RepositoryID)
+	}
+	if err != nil {
 		writeJSON(w, 502, map[string]any{"error": err.Error(), "status": s.deps.DriverRepository.Status()})
 		return
 	}
-	writeJSON(w, 200, s.deps.DriverRepository.Status())
+	status := s.deps.DriverRepository.Status()
+	status.Warnings = warnings
+	writeJSON(w, 200, status)
 }
 
 func (s *Server) handleDeviceRepositoryInstall(w http.ResponseWriter, r *http.Request) {
@@ -278,7 +290,33 @@ func (s *Server) handleDeviceRepositoryVersions(w http.ResponseWriter, r *http.R
 		writeJSON(w, 500, map[string]string{"error": availableErr.Error()})
 		return
 	}
-	writeJSON(w, 200, map[string]any{"driver_id": r.PathValue("id"), "installed": versions, "available": available})
+	// The release's own copy and the owner's choice belong in the same list:
+	// the picker is the one place a driver version is seen and changed.
+	logicalPath := ""
+	for _, installed := range versions {
+		if installed.Active {
+			logicalPath = installed.LogicalPath
+		}
+	}
+	for _, candidate := range available {
+		if logicalPath == "" {
+			logicalPath = candidate.Driver.Path
+		}
+	}
+	release, chosen := "", ""
+	if logicalPath != "" {
+		release = s.deps.DriverRepository.ReleaseVersion(logicalPath)
+		for _, installed := range versions {
+			if installed.Active && s.deps.DriverRepository.Chosen(logicalPath, installed.Version) {
+				chosen = installed.Version
+			}
+		}
+	}
+	writeJSON(w, 200, map[string]any{
+		"driver_id": r.PathValue("id"), "installed": versions, "available": available,
+		"logical_path": logicalPath, "release_version": release, "chosen_version": chosen,
+		"release_source": s.bundledSource(),
+	})
 }
 
 func (s *Server) handleDeviceRepositoryActivate(w http.ResponseWriter, r *http.Request) {
@@ -408,6 +446,23 @@ func (r managedDriverRestartState) response(status string, artifact any) map[str
 		"runtime_verified": len(names) > 0, "restarted_drivers": names, "logical_path": r.LogicalPath, "config_changed": r.ConfigChanged}
 }
 
+// bundledSource names where this release's drivers were taken from: the
+// repository and commit pinned in drivers/BUNDLED_SOURCE.json.
+func (s *Server) bundledSource() map[string]string {
+	raw, err := os.ReadFile(filepath.Join(s.deps.DriverDir, "BUNDLED_SOURCE.json"))
+	if err != nil {
+		return nil
+	}
+	var pin struct {
+		Repository string `json:"repository"`
+		Commit     string `json:"commit"`
+	}
+	if json.Unmarshal(raw, &pin) != nil || pin.Repository == "" || pin.Commit == "" {
+		return nil
+	}
+	return map[string]string{"repository": "https://github.com/" + pin.Repository, "commit": pin.Commit}
+}
+
 // sameDriverFile reports whether a driver file, declaring id, holds the
 // driver of a signed artifact. A driver has one id everywhere: its source,
 // the signed channel and the copy bundled with this release.

go/internal/api/api_device_repository_test.go

@@ -101,6 +101,7 @@ func newDriverUpdateFixture(t *testing.T, mode string) *driverUpdateFixture {
 	t.Cleanup(func() { _ = st.Close() })
 	rc := &config.DeviceRepository{Enabled: true, Repositories: []config.DriverRepositorySource{{ID: "test", ManifestURL: f.repo.URL + "/manifest.json", Enabled: true, AllowInsecure: true, TrustedKeys: map[string]string{"test": base64.StdEncoding.EncodeToString(public)}}}}
 	m := driverrepo.New(rc, dir, st)
+	m.SetBundledDir(bundledDir)
 	tel := telemetry.NewStore()
 	registry := drivers.NewRegistry(tel)
 	registry.RuntimePolicyResolver = m.RuntimePolicy
@@ -597,3 +598,38 @@ func TestDriverCatalogCreditsOnlyTheFileThatRuns(t *testing.T) {
 		}
 	}
 }
+
+// The picker shows the release's own version and marks the owner's choice,
+// and the device card reads the same mark from the catalog.
+func TestVersionsShowTheReleaseAndTheOwnersChoice(t *testing.T) {
+	f := newDriverUpdateFixture(t, "running")
+	f.publishAs("esphome_dsmr", "esphome_dsmr.lua", "1.0.1")
+	f.requestFor("esphome_dsmr", "install", `{"repository_id":"test"}`, 200)
+
+	w := httptest.NewRecorder()
+	f.s.Handler().ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/api/device_repository/drivers/esphome_dsmr/versions", nil))
+	var versions struct {
+		Release string `json:"release_version"`
+		Chosen  string `json:"chosen_version"`
+	}
+	if w.Code != 200 || json.Unmarshal(w.Body.Bytes(), &versions) != nil {
+		t.Fatalf("versions: HTTP %d %s", w.Code, w.Body.String())
+	}
+	if versions.Release != "1.0.2" || versions.Chosen != "1.0.1" {
+		t.Fatalf("versions = %+v; want release 1.0.2 and the chosen 1.0.1", versions)
+	}
+
+	w = httptest.NewRecorder()
+	f.s.Handler().ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/api/drivers/catalog", nil))
+	var catalog struct {
+		Entries []drivers.CatalogEntry `json:"entries"`
+	}
+	if err := json.Unmarshal(w.Body.Bytes(), &catalog); err != nil {
+		t.Fatal(err)
+	}
+	for _, e := range catalog.Entries {
+		if e.Filename == "esphome_dsmr.lua" && (!e.Chosen || e.ReleaseVersion != "1.0.2") {
+			t.Fatalf("catalog entry = %+v; want chosen over release 1.0.2", e)
+		}
+	}
+}

go/internal/driverrepo/manager.go

@@ -107,12 +107,21 @@ type CatalogCandidate struct {
 }
 
 type VersionCandidate struct {
-	RepositoryID string                   `json:"repository_id"`
-	Driver       ManifestDriver           `json:"driver"`
-	Installed    *state.DriverRepoInstall `json:"installed,omitempty"`
+	RepositoryID string `json:"repository_id"`
+	// Channel is "beta" for the built-in beta channel and "stable" for a
+	// configured repository.
+	Channel string `json:"channel"`
+	// Repository is the signed manifest's source repository URL; with the
+	// entry's source commit it names the history behind this version.
+	Repository string                   `json:"repository,omitempty"`
+	Driver     ManifestDriver           `json:"driver"`
+	Installed  *state.DriverRepoInstall `json:"installed,omitempty"`
 }
 
 type Status struct {
+	// Warnings are problems a refresh met that did not stop it, such as an
+	// unreachable beta channel.
+	Warnings     []string                  `json:"warnings,omitempty"`
 	Enabled      bool                      `json:"enabled"`
 	HostAPI      int                       `json:"driver_host_api"`
 	RootDir      string                    `json:"root_dir"`
@@ -312,6 +321,20 @@ func (m *Manager) Refresh(ctx context.Context, repositoryID string) error {
 	return errors.Join(errs...)
 }
 
+// RefreshAll is an owner's "check for new versions": every configured
+// source and the beta channel. A beta that cannot be reached is a warning,
+// not a failure, so the versions list still redraws from what did refresh.
+// The periodic refresh stays on Refresh, which reads only configured sources.
+func (m *Manager) RefreshAll(ctx context.Context) (warnings []string, err error) {
+	if err := m.Refresh(ctx, ""); err != nil {
+		return nil, err
+	}
+	if betaErr := m.refreshOne(ctx, m.betaRepo); betaErr != nil {
+		warnings = append(warnings, "beta channel: "+betaErr.Error())
+	}
+	return warnings, nil
+}
+
 func (m *Manager) refreshOne(ctx context.Context, repo config.DriverRepositorySource) error {
 	if repositoryFormat(repo) == config.DriverRepositoryFormatSourcefulIndexV1 {
 		return m.refreshSourceful(ctx, repo)
@@ -735,33 +758,63 @@ func (m *Manager) AvailableVersions(driverID string) ([]VersionCandidate, error)
 	if err != nil {
 		return nil, err
 	}
-	installedByKey := make(map[string]state.DriverRepoInstall, len(installed))
+	// An install is matched by content, not by the channel it came from: a
+	// beta file that stable later publishes byte for byte is the file that
+	// runs, whichever row lists it.
+	installedByContent := make(map[string]state.DriverRepoInstall, len(installed))
 	for _, artifact := range installed {
-		installedByKey[artifact.RepoID+"\x00"+artifact.Version+"\x00"+strings.ToLower(artifact.SHA256)] = artifact
+		key := artifact.Version + "\x00" + strings.ToLower(artifact.SHA256)
+		if prior, ok := installedByContent[key]; !ok || (artifact.Active && !prior.Active) {
+			installedByContent[key] = artifact
+		}
 	}
+	listed := make(map[int64]bool)
 	var out []VersionCandidate
 	seen := make(map[string]bool)
+	offered := make(map[string]bool) // by content, so a promoted beta file is listed once
+	sources := make([]config.DriverRepositorySource, 0, len(m.cfg.Repositories)+1)
 	for _, repo := range m.cfg.Repositories {
-		if !repo.Enabled {
-			continue
+		if repo.Enabled {
+			sources = append(sources, repo)
 		}
+	}
+	if m.cfg.Enabled {
+		// Cached only: listing versions never reaches the network.
+		sources = append(sources, m.betaRepo)
+	}
+	newestStable := ""
+	for _, repo := range sources {
 		manifest, err := m.manifestFor(repo)
 		if err != nil {
 			continue
 		}
+		channel := "stable"
+		if repo.ID == m.betaRepo.ID {
+			channel = "beta"
+		}
 		for _, driver := range append(append([]ManifestDriver{}, manifest.Drivers...), manifest.History...) {
 			if driver.ID != driverID {
 				continue
 			}
+			if channel == "stable" && (newestStable == "" || compareSemver(driver.Version, newestStable) > 0) {
+				newestStable = driver.Version
+			}
+			// Beta is where the next version waits. Its history repeats
+			// versions stable has passed, which would only be noise here.
+			if channel == "beta" && newestStable != "" && compareSemver(driver.Version, newestStable) <= 0 {
+				continue
+			}
 			key := repo.ID + "\x00" + driver.Version + "\x00" + strings.ToLower(driver.SHA256)
-			if seen[key] {
+			if seen[key] || (channel == "beta" && offered[strings.ToLower(driver.SHA256)]) {
 				continue
 			}
 			seen[key] = true
-			candidate := VersionCandidate{RepositoryID: repo.ID, Driver: driver}
-			if artifact, ok := installedByKey[key]; ok {
+			offered[strings.ToLower(driver.SHA256)] = true
+			candidate := VersionCandidate{RepositoryID: repo.ID, Channel: channel, Repository: manifest.Repository, Driver: driver}
+			if artifact, ok := installedByContent[driver.Version+"\x00"+strings.ToLower(driver.SHA256)]; ok {
 				copy := artifact
 				candidate.Installed = &copy
+				listed[artifact.ID] = true
 			}
 			out = append(out, candidate)
 		}
@@ -772,13 +825,18 @@ func (m *Manager) AvailableVersions(driverID string) ([]VersionCandidate, error)
 	// return to a known local version.
 	for _, artifact := range installed {
 		key := artifact.RepoID + "\x00" + artifact.Version + "\x00" + strings.ToLower(artifact.SHA256)
-		if seen[key] {
+		if seen[key] || listed[artifact.ID] {
 			continue
 		}
 		seen[key] = true
 		copy := artifact
+		channel := "stable"
+		if artifact.RepoID == m.betaRepo.ID {
+			channel = "beta"
+		}
 		out = append(out, VersionCandidate{
 			RepositoryID: artifact.RepoID,
+			Channel:      channel,
 			Driver: ManifestDriver{
 				ID: artifact.DriverID, Path: artifact.LogicalPath,
 				Filename: filepath.Base(artifact.InstalledPath), Version: artifact.Version,
@@ -928,6 +986,21 @@ func (m *Manager) bundledVersion(logicalPath string) string {
 	return entry.Version
 }
 
+// ReleaseVersion is the version this Core release bundles at logicalPath, or
+// "" when it bundles none there.
+func (m *Manager) ReleaseVersion(logicalPath string) string { return m.bundledVersion(logicalPath) }
+
+// Chosen reports whether the managed selection at logicalPath runs version
+// because the owner went back to it from a newer one; such a choice stays
+// across Core updates.
+func (m *Manager) Chosen(logicalPath, version string) bool {
+	if m.store == nil || version == "" {
+		return false
+	}
+	pinned, _ := m.store.LoadConfig(pinKey(logicalPath))
+	return pinned == version
+}
+
 // pinKey holds the version of a managed selection the owner chose over a
 // newer one that was running. previousPinKey keeps the value it replaced, so
 // undoing an activation also undoes its effect on the choice.