feat(nova): probe os_type for KVM servers during server sync

internal/knowledge/datasources/plugins/openstack/nova/nova_api.go

@@ -20,15 +20,18 @@ import (
 	"github.com/gophercloud/gophercloud/v2/openstack"
 	"github.com/gophercloud/gophercloud/v2/openstack/compute/v2/aggregates"
 	"github.com/gophercloud/gophercloud/v2/openstack/compute/v2/flavors"
+	"github.com/gophercloud/gophercloud/v2/openstack/compute/v2/servers"
 	glanceimages "github.com/gophercloud/gophercloud/v2/openstack/image/v2/images"
 	"github.com/gophercloud/gophercloud/v2/pagination"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/sapcc/go-bits/liquidapi"
 )
 
 type NovaAPI interface {
 	// Init the nova API.
 	Init(ctx context.Context) error
 	// Get all nova servers that are NOT deleted. (Includes ERROR, SHUTOFF etc)
+	// For KVM flavors, os_type is probed concurrently using the OSTypeProber.
 	GetAllServers(ctx context.Context) ([]Server, error)
 	// Get all deleted nova servers since the timestamp.
 	GetDeletedServers(ctx context.Context, since time.Time) ([]DeletedServer, error)
@@ -56,6 +59,8 @@ type novaAPI struct {
 	sc *gophercloud.ServiceClient
 	// Authenticated Glance image service client (only used for NovaDatasourceTypeImages).
 	glance *gophercloud.ServiceClient
+	// OS type prober for determining VM operating system type (only for NovaDatasourceTypeServers).
+	osTypeProber *liquidapi.OSTypeProber
 }
 
 func NewNovaAPI(mon datasources.Monitor, k keystone.KeystoneClient, conf v1alpha1.NovaDatasource) NovaAPI {
@@ -95,6 +100,11 @@ func (api *novaAPI) Init(ctx context.Context) error {
 		}
 		api.glance = glanceClient
 	}
+	// Initialize the OS type prober only for the servers datasource.
+	if api.conf.Type == v1alpha1.NovaDatasourceTypeServers {
+		eo := gophercloud.EndpointOpts{Availability: gophercloud.Availability(sameAsKeystone)}
+		api.osTypeProber = initOSTypeProber(provider, eo)
+	}
 	return nil
 }
 
@@ -157,10 +167,56 @@ func (api *novaAPI) GetAllServers(ctx context.Context) ([]Server, error) {
 		}
 	}
 
+	// Probe OS type concurrently for KVM servers.
+	api.probeOSTypes(ctx, allServers)
+
 	slog.Info("fetched", "label", label, "count", len(allServers))
 	return allServers, nil
 }
 
+// probeOSTypes determines the OS type for all KVM servers sequentially.
+// The prober caches by image ID internally, so repeated images are instant.
+func (api *novaAPI) probeOSTypes(ctx context.Context, allServers []Server) {
+	if api.osTypeProber == nil {
+		slog.Info("os_type prober not initialized, skipping")
+		return
+	}
+	var probed, resolved, unknown, rootdiskMissing int
+	for i := range allServers {
+		if isKVMFlavor(allServers[i].FlavorName) {
+			probed++
+			osType := api.probeOSType(ctx, allServers[i])
+			switch osType {
+			case "unknown":
+				unknown++
+			case "rootdisk-missing":
+				rootdiskMissing++
+			default:
+				if osType != "" {
+					resolved++
+				}
+			}
+			allServers[i].OSType = osType
+		}
+	}
+	slog.Info("probed os_type for KVM servers",
+		"total", len(allServers),
+		"kvm", probed,
+		"resolved", resolved,
+		"unknown", unknown,
+		"rootdiskMissing", rootdiskMissing,
+	)
+}
+
+// probeOSType determines the OS type for a single server.
+func (api *novaAPI) probeOSType(ctx context.Context, s Server) string {
+	var imageMap map[string]any
+	if s.ImageRef != "" {
+		imageMap = map[string]any{"id": s.ImageRef}
+	}
+	return api.osTypeProber.Get(ctx, servers.Server{ID: s.ID, Image: imageMap})
+}
+
 // Get all deleted Nova servers.
 // Note on Nova terminology: Nova uses "instance" internally in its database and code,
 // but exposes these as "server" objects through the public API.
@@ -522,3 +578,19 @@ func deriveOSType(properties map[string]any, tags []string) string {
 	}
 	return "unknown"
 }
+
+// initOSTypeProber safely creates an OSTypeProber, returning nil on any error or panic.
+func initOSTypeProber(provider *gophercloud.ProviderClient, eo gophercloud.EndpointOpts) (prober *liquidapi.OSTypeProber) {
+	defer func() {
+		if r := recover(); r != nil {
+			slog.Warn("panic during OS type prober initialization - os_type will be empty", "panic", r)
+			prober = nil
+		}
+	}()
+	p, err := liquidapi.NewOSTypeProber(provider, eo)
+	if err != nil {
+		slog.Warn("failed to initialize OS type prober - os_type will be empty", "error", err)
+		return nil
+	}
+	return p
+}

internal/knowledge/datasources/plugins/openstack/nova/nova_sync.go

@@ -5,6 +5,7 @@ package nova
 
 import (
 	"context"
+	"strings"
 	"time"
 
 	"github.com/cobaltcore-dev/cortex/api/v1alpha1"
@@ -75,6 +76,11 @@ func (s *NovaSyncer) Sync(ctx context.Context) (int64, error) {
 	return nResults, err
 }
 
+// isKVMFlavor returns true if the flavor name indicates a KVM-based VM.
+func isKVMFlavor(flavorName string) bool {
+	return strings.Contains(flavorName, "_k_")
+}
+
 // Sync all the active OpenStack servers into the database. (Includes ERROR, SHUTOFF, etc. state)
 func (s *NovaSyncer) SyncAllServers(ctx context.Context) (int64, error) {
 	allServers, err := s.API.GetAllServers(ctx)

internal/knowledge/datasources/plugins/openstack/nova/nova_types.go

@@ -116,6 +116,10 @@ type Server struct {
 	// Empty string for volume-booted servers.
 	ImageRef string `json:"-" db:"image_ref"`
 
+	// OSType is the operating system type determined by the OSTypeProber at sync time.
+	// Only populated for KVM servers (flavor name contains "_k_").
+	OSType string `json:"-" db:"os_type"`
+
 	// From nested server.fault JSON
 
 	// The error response code.
@@ -234,7 +238,7 @@ func (s *Server) MarshalJSON() ([]byte, error) {
 }
 
 // Table in which the openstack model is stored.
-func (Server) TableName() string { return "openstack_servers_v3" }
+func (Server) TableName() string { return "openstack_servers_v4" }
 
 // Index for the openstack model.
 func (Server) Indexes() map[string][]string { return nil }

internal/knowledge/extractor/plugins/compute/libvirt_domain_cpu_steal_pct.sql

@@ -3,6 +3,6 @@ SELECT
     os.os_ext_srv_attr_host AS host,
     MAX(value) AS max_steal_time_pct
 FROM kvm_libvirt_domain_metrics kvm
-JOIN openstack_servers_v3 os ON os.os_ext_srv_attr_instance_name = kvm.domain
+JOIN openstack_servers_v4 os ON os.os_ext_srv_attr_instance_name = kvm.domain
 WHERE kvm.name = 'kvm_libvirt_domain_steal_pct' AND os.id IS NOT NULL
 GROUP BY os.os_ext_srv_attr_host, os.id;

internal/knowledge/extractor/plugins/compute/vm_host_residency.sql

@@ -21,7 +21,7 @@ WITH durations AS (
             )) AS BIGINT)
         ) AS duration
     FROM openstack_migrations AS migrations
-    LEFT JOIN openstack_servers_v3 AS servers ON servers.id = migrations.instance_uuid
+    LEFT JOIN openstack_servers_v4 AS servers ON servers.id = migrations.instance_uuid
     LEFT JOIN openstack_flavors_v2 AS flavors ON flavors.name = servers.flavor_name
 )
 SELECT

internal/knowledge/extractor/plugins/compute/vm_life_span.sql

@@ -13,7 +13,7 @@ running_servers AS (
         EXTRACT(EPOCH FROM (NOW()::timestamp - servers.created::timestamp))::BIGINT AS duration,
         COALESCE(flavors.name, 'unknown')::TEXT AS flavor_name,
         false::BOOLEAN AS deleted
-    FROM openstack_servers_v3 servers
+    FROM openstack_servers_v4 servers
     LEFT JOIN openstack_flavors_v2 flavors ON flavors.name = servers.flavor_name
     WHERE servers.created IS NOT NULL
 )

internal/knowledge/extractor/plugins/compute/vrops_hostsystem_resolver.sql

@@ -3,5 +3,5 @@ SELECT DISTINCT
     m.hostsystem AS vrops_hostsystem,
     s.os_ext_srv_attr_host AS nova_compute_host
 FROM vrops_vm_metrics m
-LEFT JOIN openstack_servers_v3 s ON m.instance_uuid = s.id
+LEFT JOIN openstack_servers_v4 s ON m.instance_uuid = s.id
 WHERE s.os_ext_srv_attr_host IS NOT NULL;

internal/knowledge/extractor/plugins/compute/vrops_project_noisiness.sql

@@ -19,7 +19,7 @@ host_cpu_usage AS (
         s.tenant_id,
         h.service_host,
         AVG(p.avg_cpu) AS avg_cpu_of_project
-    FROM openstack_servers_v3 s
+    FROM openstack_servers_v4 s
     JOIN vrops_vm_metrics m ON s.id = m.instance_uuid
     JOIN projects_avg_cpu p ON s.tenant_id = p.tenant_id
     JOIN openstack_hypervisors h ON s.os_ext_srv_attr_hypervisor_hostname = h.hostname

internal/scheduling/reservations/commitments/usage.go

@@ -707,10 +707,9 @@ func (c *dbUsageClient) ListProjectVMs(ctx context.Context, projectID string) ([
 			COALESCE(f.vcpus, 0)        AS flavor_vcpus,
 			COALESCE(f.disk, 0)         AS flavor_disk,
 			COALESCE(f.extra_specs, '') AS flavor_extras,
-			COALESCE(NULLIF(i.os_type, ''), 'unknown') AS os_type
+			COALESCE(NULLIF(s.os_type, ''), 'unknown') AS os_type
 		FROM ` + nova.Server{}.TableName() + ` s
 		LEFT JOIN ` + nova.Flavor{}.TableName() + ` f ON f.name = s.flavor_name
-		LEFT JOIN ` + nova.Image{}.TableName() + ` i ON i.id = s.image_ref
 		WHERE s.tenant_id = $1`
 
 	var rows []vmQueryRow