From 7c353edbdb9ac2bd209da255e711eb69cbe573ba Mon Sep 17 00:00:00 2001 From: Malte Viering Date: Tue, 26 May 2026 13:17:09 +0000 Subject: [PATCH 1/2] feat(nova): probe os_type for KVM servers during server sync Integrate liquidapi.OSTypeProber into the nova datasource to automatically determine os_type for KVM-flavored servers at sync time. The prober resolves os_type from Glance image metadata (vmware_ostype property or ostype: tags). - Add osTypeProber to novaAPI, initialized during Init() for servers datasource - Add probeOSTypes/probeOSType methods that iterate KVM servers sequentially (prober is not goroutine-safe due to internal map cache) - Add OSType field to Server and Image types - Add GetAllImages API method with deriveOSType logic for the images datasource - Remove ProbeOSType from NovaAPI interface (now internal to GetAllServers) - Add initOSTypeProber with panic recovery for environments without Glance --- .../plugins/openstack/nova/nova_api.go | 59 +++++++++++++++++++ .../plugins/openstack/nova/nova_sync.go | 6 ++ .../plugins/openstack/nova/nova_types.go | 6 +- 3 files changed, 70 insertions(+), 1 deletion(-) diff --git a/internal/knowledge/datasources/plugins/openstack/nova/nova_api.go b/internal/knowledge/datasources/plugins/openstack/nova/nova_api.go index 03298e9db..781f7f1ba 100644 --- a/internal/knowledge/datasources/plugins/openstack/nova/nova_api.go +++ b/internal/knowledge/datasources/plugins/openstack/nova/nova_api.go @@ -20,15 +20,18 @@ import ( "github.com/gophercloud/gophercloud/v2/openstack" "github.com/gophercloud/gophercloud/v2/openstack/compute/v2/aggregates" "github.com/gophercloud/gophercloud/v2/openstack/compute/v2/flavors" + "github.com/gophercloud/gophercloud/v2/openstack/compute/v2/servers" glanceimages "github.com/gophercloud/gophercloud/v2/openstack/image/v2/images" "github.com/gophercloud/gophercloud/v2/pagination" "github.com/prometheus/client_golang/prometheus" + "github.com/sapcc/go-bits/liquidapi" ) type NovaAPI interface { // Init the nova API. Init(ctx context.Context) error // Get all nova servers that are NOT deleted. (Includes ERROR, SHUTOFF etc) + // For KVM flavors, os_type is probed concurrently using the OSTypeProber. GetAllServers(ctx context.Context) ([]Server, error) // Get all deleted nova servers since the timestamp. GetDeletedServers(ctx context.Context, since time.Time) ([]DeletedServer, error) @@ -56,6 +59,8 @@ type novaAPI struct { sc *gophercloud.ServiceClient // Authenticated Glance image service client (only used for NovaDatasourceTypeImages). glance *gophercloud.ServiceClient + // OS type prober for determining VM operating system type (only for NovaDatasourceTypeServers). + osTypeProber *liquidapi.OSTypeProber } func NewNovaAPI(mon datasources.Monitor, k keystone.KeystoneClient, conf v1alpha1.NovaDatasource) NovaAPI { @@ -95,6 +100,11 @@ func (api *novaAPI) Init(ctx context.Context) error { } api.glance = glanceClient } + // Initialize the OS type prober only for the servers datasource. + if api.conf.Type == v1alpha1.NovaDatasourceTypeServers { + eo := gophercloud.EndpointOpts{Availability: gophercloud.Availability(sameAsKeystone)} + api.osTypeProber = initOSTypeProber(provider, eo) + } return nil } @@ -157,10 +167,43 @@ func (api *novaAPI) GetAllServers(ctx context.Context) ([]Server, error) { } } + // Probe OS type concurrently for KVM servers. + api.probeOSTypes(ctx, allServers) + slog.Info("fetched", "label", label, "count", len(allServers)) return allServers, nil } +// probeOSTypes determines the OS type for all KVM servers sequentially. +// The prober caches by image ID internally, so repeated images are instant. +func (api *novaAPI) probeOSTypes(ctx context.Context, allServers []Server) { + if api.osTypeProber == nil { + slog.Info("os_type prober not initialized, skipping") + return + } + var probed, resolved int + for i := range allServers { + if isKVMFlavor(allServers[i].FlavorName) { + probed++ + osType := api.probeOSType(ctx, allServers[i]) + if osType != "" { + resolved++ + } + allServers[i].OSType = osType + } + } + slog.Info("probed os_type for KVM servers", "total", len(allServers), "kvm", probed, "resolved", resolved) +} + +// probeOSType determines the OS type for a single server. +func (api *novaAPI) probeOSType(ctx context.Context, s Server) string { + var imageMap map[string]any + if s.ImageRef != "" { + imageMap = map[string]any{"id": s.ImageRef} + } + return api.osTypeProber.Get(ctx, servers.Server{ID: s.ID, Image: imageMap}) +} + // Get all deleted Nova servers. // Note on Nova terminology: Nova uses "instance" internally in its database and code, // but exposes these as "server" objects through the public API. @@ -522,3 +565,19 @@ func deriveOSType(properties map[string]any, tags []string) string { } return "unknown" } + +// initOSTypeProber safely creates an OSTypeProber, returning nil on any error or panic. +func initOSTypeProber(provider *gophercloud.ProviderClient, eo gophercloud.EndpointOpts) (prober *liquidapi.OSTypeProber) { + defer func() { + if r := recover(); r != nil { + slog.Warn("panic during OS type prober initialization - os_type will be empty", "panic", r) + prober = nil + } + }() + p, err := liquidapi.NewOSTypeProber(provider, eo) + if err != nil { + slog.Warn("failed to initialize OS type prober - os_type will be empty", "error", err) + return nil + } + return p +} diff --git a/internal/knowledge/datasources/plugins/openstack/nova/nova_sync.go b/internal/knowledge/datasources/plugins/openstack/nova/nova_sync.go index a2c466c42..ec0533f84 100644 --- a/internal/knowledge/datasources/plugins/openstack/nova/nova_sync.go +++ b/internal/knowledge/datasources/plugins/openstack/nova/nova_sync.go @@ -5,6 +5,7 @@ package nova import ( "context" + "strings" "time" "github.com/cobaltcore-dev/cortex/api/v1alpha1" @@ -75,6 +76,11 @@ func (s *NovaSyncer) Sync(ctx context.Context) (int64, error) { return nResults, err } +// isKVMFlavor returns true if the flavor name indicates a KVM-based VM. +func isKVMFlavor(flavorName string) bool { + return strings.Contains(flavorName, "_k_") +} + // Sync all the active OpenStack servers into the database. (Includes ERROR, SHUTOFF, etc. state) func (s *NovaSyncer) SyncAllServers(ctx context.Context) (int64, error) { allServers, err := s.API.GetAllServers(ctx) diff --git a/internal/knowledge/datasources/plugins/openstack/nova/nova_types.go b/internal/knowledge/datasources/plugins/openstack/nova/nova_types.go index 5fef71d6e..17c422194 100644 --- a/internal/knowledge/datasources/plugins/openstack/nova/nova_types.go +++ b/internal/knowledge/datasources/plugins/openstack/nova/nova_types.go @@ -116,6 +116,10 @@ type Server struct { // Empty string for volume-booted servers. ImageRef string `json:"-" db:"image_ref"` + // OSType is the operating system type determined by the OSTypeProber at sync time. + // Only populated for KVM servers (flavor name contains "_k_"). + OSType string `json:"-" db:"os_type"` + // From nested server.fault JSON // The error response code. @@ -234,7 +238,7 @@ func (s *Server) MarshalJSON() ([]byte, error) { } // Table in which the openstack model is stored. -func (Server) TableName() string { return "openstack_servers_v3" } +func (Server) TableName() string { return "openstack_servers_v4" } // Index for the openstack model. func (Server) Indexes() map[string][]string { return nil } From 9024b1d54b8d136c0de374177a839bf17886633b Mon Sep 17 00:00:00 2001 From: Malte Viering Date: Tue, 26 May 2026 15:20:46 +0200 Subject: [PATCH 2/2] bump to v4 --- .../plugins/openstack/nova/nova_api.go | 21 +++++++++++++++---- .../compute/libvirt_domain_cpu_steal_pct.sql | 2 +- .../plugins/compute/vm_host_residency.sql | 2 +- .../plugins/compute/vm_life_span.sql | 2 +- .../compute/vrops_hostsystem_resolver.sql | 2 +- .../compute/vrops_project_noisiness.sql | 2 +- .../reservations/commitments/usage.go | 3 +-- 7 files changed, 23 insertions(+), 11 deletions(-) diff --git a/internal/knowledge/datasources/plugins/openstack/nova/nova_api.go b/internal/knowledge/datasources/plugins/openstack/nova/nova_api.go index 781f7f1ba..906a46a1e 100644 --- a/internal/knowledge/datasources/plugins/openstack/nova/nova_api.go +++ b/internal/knowledge/datasources/plugins/openstack/nova/nova_api.go @@ -181,18 +181,31 @@ func (api *novaAPI) probeOSTypes(ctx context.Context, allServers []Server) { slog.Info("os_type prober not initialized, skipping") return } - var probed, resolved int + var probed, resolved, unknown, rootdiskMissing int for i := range allServers { if isKVMFlavor(allServers[i].FlavorName) { probed++ osType := api.probeOSType(ctx, allServers[i]) - if osType != "" { - resolved++ + switch osType { + case "unknown": + unknown++ + case "rootdisk-missing": + rootdiskMissing++ + default: + if osType != "" { + resolved++ + } } allServers[i].OSType = osType } } - slog.Info("probed os_type for KVM servers", "total", len(allServers), "kvm", probed, "resolved", resolved) + slog.Info("probed os_type for KVM servers", + "total", len(allServers), + "kvm", probed, + "resolved", resolved, + "unknown", unknown, + "rootdiskMissing", rootdiskMissing, + ) } // probeOSType determines the OS type for a single server. diff --git a/internal/knowledge/extractor/plugins/compute/libvirt_domain_cpu_steal_pct.sql b/internal/knowledge/extractor/plugins/compute/libvirt_domain_cpu_steal_pct.sql index 56b20a980..cf2f3ca50 100644 --- a/internal/knowledge/extractor/plugins/compute/libvirt_domain_cpu_steal_pct.sql +++ b/internal/knowledge/extractor/plugins/compute/libvirt_domain_cpu_steal_pct.sql @@ -3,6 +3,6 @@ SELECT os.os_ext_srv_attr_host AS host, MAX(value) AS max_steal_time_pct FROM kvm_libvirt_domain_metrics kvm -JOIN openstack_servers_v3 os ON os.os_ext_srv_attr_instance_name = kvm.domain +JOIN openstack_servers_v4 os ON os.os_ext_srv_attr_instance_name = kvm.domain WHERE kvm.name = 'kvm_libvirt_domain_steal_pct' AND os.id IS NOT NULL GROUP BY os.os_ext_srv_attr_host, os.id; \ No newline at end of file diff --git a/internal/knowledge/extractor/plugins/compute/vm_host_residency.sql b/internal/knowledge/extractor/plugins/compute/vm_host_residency.sql index 190f2da19..69987328e 100644 --- a/internal/knowledge/extractor/plugins/compute/vm_host_residency.sql +++ b/internal/knowledge/extractor/plugins/compute/vm_host_residency.sql @@ -21,7 +21,7 @@ WITH durations AS ( )) AS BIGINT) ) AS duration FROM openstack_migrations AS migrations - LEFT JOIN openstack_servers_v3 AS servers ON servers.id = migrations.instance_uuid + LEFT JOIN openstack_servers_v4 AS servers ON servers.id = migrations.instance_uuid LEFT JOIN openstack_flavors_v2 AS flavors ON flavors.name = servers.flavor_name ) SELECT diff --git a/internal/knowledge/extractor/plugins/compute/vm_life_span.sql b/internal/knowledge/extractor/plugins/compute/vm_life_span.sql index 38b8762ba..ba2fc4d27 100644 --- a/internal/knowledge/extractor/plugins/compute/vm_life_span.sql +++ b/internal/knowledge/extractor/plugins/compute/vm_life_span.sql @@ -13,7 +13,7 @@ running_servers AS ( EXTRACT(EPOCH FROM (NOW()::timestamp - servers.created::timestamp))::BIGINT AS duration, COALESCE(flavors.name, 'unknown')::TEXT AS flavor_name, false::BOOLEAN AS deleted - FROM openstack_servers_v3 servers + FROM openstack_servers_v4 servers LEFT JOIN openstack_flavors_v2 flavors ON flavors.name = servers.flavor_name WHERE servers.created IS NOT NULL ) diff --git a/internal/knowledge/extractor/plugins/compute/vrops_hostsystem_resolver.sql b/internal/knowledge/extractor/plugins/compute/vrops_hostsystem_resolver.sql index 21f3104fd..39705d585 100644 --- a/internal/knowledge/extractor/plugins/compute/vrops_hostsystem_resolver.sql +++ b/internal/knowledge/extractor/plugins/compute/vrops_hostsystem_resolver.sql @@ -3,5 +3,5 @@ SELECT DISTINCT m.hostsystem AS vrops_hostsystem, s.os_ext_srv_attr_host AS nova_compute_host FROM vrops_vm_metrics m -LEFT JOIN openstack_servers_v3 s ON m.instance_uuid = s.id +LEFT JOIN openstack_servers_v4 s ON m.instance_uuid = s.id WHERE s.os_ext_srv_attr_host IS NOT NULL; diff --git a/internal/knowledge/extractor/plugins/compute/vrops_project_noisiness.sql b/internal/knowledge/extractor/plugins/compute/vrops_project_noisiness.sql index 850cbbca1..e539263e4 100644 --- a/internal/knowledge/extractor/plugins/compute/vrops_project_noisiness.sql +++ b/internal/knowledge/extractor/plugins/compute/vrops_project_noisiness.sql @@ -19,7 +19,7 @@ host_cpu_usage AS ( s.tenant_id, h.service_host, AVG(p.avg_cpu) AS avg_cpu_of_project - FROM openstack_servers_v3 s + FROM openstack_servers_v4 s JOIN vrops_vm_metrics m ON s.id = m.instance_uuid JOIN projects_avg_cpu p ON s.tenant_id = p.tenant_id JOIN openstack_hypervisors h ON s.os_ext_srv_attr_hypervisor_hostname = h.hostname diff --git a/internal/scheduling/reservations/commitments/usage.go b/internal/scheduling/reservations/commitments/usage.go index a6d360bd1..fb9984a75 100644 --- a/internal/scheduling/reservations/commitments/usage.go +++ b/internal/scheduling/reservations/commitments/usage.go @@ -707,10 +707,9 @@ func (c *dbUsageClient) ListProjectVMs(ctx context.Context, projectID string) ([ COALESCE(f.vcpus, 0) AS flavor_vcpus, COALESCE(f.disk, 0) AS flavor_disk, COALESCE(f.extra_specs, '') AS flavor_extras, - COALESCE(NULLIF(i.os_type, ''), 'unknown') AS os_type + COALESCE(NULLIF(s.os_type, ''), 'unknown') AS os_type FROM ` + nova.Server{}.TableName() + ` s LEFT JOIN ` + nova.Flavor{}.TableName() + ` f ON f.name = s.flavor_name - LEFT JOIN ` + nova.Image{}.TableName() + ` i ON i.id = s.image_ref WHERE s.tenant_id = $1` var rows []vmQueryRow