diff --git a/api/v1alpha1/tenant_compatibility_test.go b/api/v1alpha1/tenant_compatibility_test.go new file mode 100644 index 0000000..1bf741d --- /dev/null +++ b/api/v1alpha1/tenant_compatibility_test.go @@ -0,0 +1,51 @@ +package v1alpha1 + +import ( + "encoding/json" + "os" + "testing" + + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "sigs.k8s.io/yaml" +) + +func TestTenantCompatibility_NetworkPolicyEnforcementReasonIsAdditive(t *testing.T) { + legacy := []byte(`{"apiVersion":"agentic.clawdlinux.org/v1alpha1","kind":"Tenant","metadata":{"name":"legacy"},"spec":{"displayName":"Legacy","namespace":"tenant-a","providers":["openai"]},"status":{"networkPolicyActive":true}}`) + var tenant Tenant + if err := json.Unmarshal(legacy, &tenant); err != nil { + t.Fatalf("unmarshal legacy Tenant: %v", err) + } + if tenant.Status.NetworkPolicyEnforcementReason != "" { + t.Fatalf("legacy enforcement reason = %q, want empty", tenant.Status.NetworkPolicyEnforcementReason) + } + + current := []byte(`{"status":{"networkPolicyEnforcementReason":"Enforcing"}}`) + if err := json.Unmarshal(current, &tenant); err != nil { + t.Fatalf("unmarshal current Tenant status: %v", err) + } + if tenant.Status.NetworkPolicyEnforcementReason != "Enforcing" { + t.Fatalf("enforcement reason = %q, want Enforcing", tenant.Status.NetworkPolicyEnforcementReason) + } +} + +func TestTenantCompatibility_CRDNetworkPolicyEnforcementReason(t *testing.T) { + data, err := os.ReadFile("../../config/crd/bases/agentic.clawdlinux.org_tenants.yaml") + if err != nil { + t.Fatalf("read Tenant CRD: %v", err) + } + var crd apiextensionsv1.CustomResourceDefinition + if err := yaml.Unmarshal(data, &crd); err != nil { + t.Fatalf("decode Tenant CRD: %v", err) + } + if len(crd.Spec.Versions) != 1 || crd.Spec.Versions[0].Schema == nil || crd.Spec.Versions[0].Schema.OpenAPIV3Schema == nil { + t.Fatal("Tenant CRD has no v1alpha1 OpenAPI schema") + } + statusSchema, ok := crd.Spec.Versions[0].Schema.OpenAPIV3Schema.Properties["status"] + if !ok { + t.Fatal("Tenant CRD schema has no status property") + } + reasonSchema, ok := statusSchema.Properties["networkPolicyEnforcementReason"] + if !ok || reasonSchema.Type != "string" { + t.Fatalf("Tenant CRD status.networkPolicyEnforcementReason = %#v, want string", reasonSchema) + } +} diff --git a/api/v1alpha1/tenant_types.go b/api/v1alpha1/tenant_types.go index d0767a5..b1a01e9 100644 --- a/api/v1alpha1/tenant_types.go +++ b/api/v1alpha1/tenant_types.go @@ -85,6 +85,10 @@ type TenantStatus struct { // NetworkPolicyActive indicates if network policies are active NetworkPolicyActive bool `json:"networkPolicyActive,omitempty"` + // NetworkPolicyEnforcementReason describes passive CNI evidence. It does not + // represent an active packet-flow verification. + NetworkPolicyEnforcementReason string `json:"networkPolicyEnforcementReason,omitempty"` + // WorkloadCount is the number of active workloads for this tenant WorkloadCount int `json:"workloadCount,omitempty"` diff --git a/charts/templates/rbac.yaml b/charts/templates/rbac.yaml index eb433f5..a6757e9 100644 --- a/charts/templates/rbac.yaml +++ b/charts/templates/rbac.yaml @@ -31,6 +31,7 @@ rules: - configmaps - secrets - namespaces + - resourcequotas - serviceaccounts verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] # Sandbox node readiness checks @@ -64,16 +65,19 @@ rules: resources: - agentworkloads - agentcards + - tenants verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] - apiGroups: ["agentic.clawdlinux.org"] resources: - agentworkloads/status - agentcards/status + - tenants/status verbs: ["get", "update", "patch"] - apiGroups: ["agentic.clawdlinux.org"] resources: - agentworkloads/finalizers - agentcards/finalizers + - tenants/finalizers verbs: ["update"] # RBAC (for sub-agent service accounts — namespace-scoped only) - apiGroups: ["rbac.authorization.k8s.io"] diff --git a/charts/tests/operator_rbac_test.yaml b/charts/tests/operator_rbac_test.yaml index b1fac65..e7a8014 100644 --- a/charts/tests/operator_rbac_test.yaml +++ b/charts/tests/operator_rbac_test.yaml @@ -2,7 +2,7 @@ suite: operator controller RBAC templates: - templates/rbac.yaml tests: - - it: grants the running controllers access to AgentWorkload and AgentCard resources + - it: grants the running controllers access to AgentWorkload, AgentCard, and Tenant resources set: license.key: dev-license asserts: @@ -14,6 +14,32 @@ tests: resources: - agentworkloads - agentcards + - tenants + verbs: + - get + - list + - watch + - create + - update + - patch + - delete + documentIndex: 1 + - contains: + path: rules[0].resources + content: resourcequotas + documentIndex: 1 + - contains: + path: rules[0].verbs + content: watch + documentIndex: 1 + - contains: + path: rules + content: + apiGroups: + - rbac.authorization.k8s.io + resources: + - roles + - rolebindings verbs: - get - list @@ -31,6 +57,7 @@ tests: resources: - agentworkloads/status - agentcards/status + - tenants/status verbs: - get - update @@ -44,6 +71,7 @@ tests: resources: - agentworkloads/finalizers - agentcards/finalizers + - tenants/finalizers verbs: - update documentIndex: 1 diff --git a/cmd/main.go b/cmd/main.go index b120835..233626f 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -31,6 +31,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. @@ -38,6 +39,8 @@ import ( "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/client-go/discovery" + "k8s.io/client-go/kubernetes" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/cache" @@ -52,6 +55,7 @@ import ( agenticv1alpha1 "github.com/Clawdlinux/agentic-operator-core/api/v1alpha1" runtimeadmission "github.com/Clawdlinux/agentic-operator-core/internal/admission" "github.com/Clawdlinux/agentic-operator-core/internal/controller" + "github.com/Clawdlinux/agentic-operator-core/internal/netpolicy" "github.com/Clawdlinux/agentic-operator-core/pkg/evaluation" "github.com/Clawdlinux/agentic-operator-core/pkg/finops" "github.com/Clawdlinux/agentic-operator-core/pkg/governance" @@ -289,6 +293,26 @@ func main() { os.Exit(1) } + discoveryClient, err := discovery.NewDiscoveryClientForConfig(ctrl.GetConfigOrDie()) + if err != nil { + setupLog.Error(err, "Failed to create discovery client") + os.Exit(1) + } + kubeClient, err := kubernetes.NewForConfig(ctrl.GetConfigOrDie()) + if err != nil { + setupLog.Error(err, "Failed to create Kubernetes client") + os.Exit(1) + } + if err := (&controller.TenantReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + DiscoveryClient: netpolicy.DiscoveryClient(discoveryClient), + DaemonSets: kubeClient.AppsV1().DaemonSets(metav1.NamespaceSystem), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "Failed to create controller", "controller", "Tenant") + os.Exit(1) + } + if os.Getenv("ENABLE_WEBHOOKS") != "false" { if err := (&agenticv1alpha1.AgentWorkload{}).SetupWebhookWithManager(mgr); err != nil { setupLog.Error(err, "Failed to setup webhook", "webhook", "AgentWorkload") diff --git a/config/crd/bases/agentic.clawdlinux.org_tenants.yaml b/config/crd/bases/agentic.clawdlinux.org_tenants.yaml index 85016ef..c0db7ee 100644 --- a/config/crd/bases/agentic.clawdlinux.org_tenants.yaml +++ b/config/crd/bases/agentic.clawdlinux.org_tenants.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.20.1 + controller-gen.kubebuilder.io/version: v0.22.0 name: tenants.agentic.clawdlinux.org spec: group: agentic.clawdlinux.org @@ -175,6 +175,11 @@ spec: description: NetworkPolicyActive indicates if network policies are active type: boolean + networkPolicyEnforcementReason: + description: |- + NetworkPolicyEnforcementReason describes passive CNI evidence. It does not + represent an active packet-flow verification. + type: string phase: description: Phase is the current provisioning phase enum: diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 3464e20..0048f2a 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -27,12 +27,12 @@ rules: - "" resources: - resourcequotas - - serviceaccounts verbs: - create - get - list - patch + - watch - apiGroups: - "" resources: @@ -42,6 +42,15 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - patch - apiGroups: - agentic.clawdlinux.org resources: @@ -75,14 +84,11 @@ rules: - patch - update - apiGroups: - - networking.k8s.io + - apps resources: - - networkpolicies + - daemonsets verbs: - - create - - get - list - - patch - apiGroups: - rbac.authorization.k8s.io resources: @@ -93,3 +99,4 @@ rules: - get - list - patch + - watch diff --git a/internal/controller/tenant_controller.go b/internal/controller/tenant_controller.go index 422707e..0d1b3f1 100644 --- a/internal/controller/tenant_controller.go +++ b/internal/controller/tenant_controller.go @@ -21,6 +21,7 @@ import ( "fmt" "time" + "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -34,12 +35,15 @@ import ( logf "sigs.k8s.io/controller-runtime/pkg/log" agenticv1alpha1 "github.com/Clawdlinux/agentic-operator-core/api/v1alpha1" + "github.com/Clawdlinux/agentic-operator-core/internal/netpolicy" ) // TenantReconciler reconciles a Tenant object type TenantReconciler struct { client.Client - Scheme *runtime.Scheme + Scheme *runtime.Scheme + DiscoveryClient netpolicy.DiscoveryClient + DaemonSets netpolicy.DaemonSetClient } // +kubebuilder:rbac:groups=agentic.clawdlinux.org,resources=tenants,verbs=get;list;watch;create;update;patch;delete @@ -48,10 +52,10 @@ type TenantReconciler struct { // +kubebuilder:rbac:groups="",resources=namespaces,verbs=get;list;watch;create;patch // +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;create // +kubebuilder:rbac:groups="",resources=serviceaccounts,verbs=create;get;list;patch -// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles,verbs=create;get;list;patch -// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=rolebindings,verbs=create;get;list;patch -// +kubebuilder:rbac:groups="",resources=resourcequotas,verbs=create;get;list;patch -// +kubebuilder:rbac:groups=networking.k8s.io,resources=networkpolicies,verbs=create;get;list;patch +// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles,verbs=create;get;list;watch;patch +// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=rolebindings,verbs=create;get;list;watch;patch +// +kubebuilder:rbac:groups="",resources=resourcequotas,verbs=create;get;list;watch;patch +// +kubebuilder:rbac:groups=apps,resources=daemonsets,verbs=list // Reconcile implements the reconciliation loop for Tenant provisioning func (r *TenantReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { @@ -79,11 +83,15 @@ func (r *TenantReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr Message: "Tenant provisioning in progress", }) tenant.Status.Phase = "Provisioning" + r.updateNetworkPolicyStatus(ctx, &tenant) // Step 1: Create Namespace if !tenant.Status.NamespaceCreated { if err := r.createNamespace(ctx, &tenant); err != nil { log.Error(err, "failed to create namespace") + if statusErr := r.persistTenantStatus(ctx, &tenant, log); statusErr != nil { + return ctrl.Result{}, statusErr + } return ctrl.Result{RequeueAfter: 30 * time.Second}, nil } tenant.Status.NamespaceCreated = true @@ -94,6 +102,9 @@ func (r *TenantReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr if !tenant.Status.SecretsProvisioned { if err := r.provisionSecrets(ctx, &tenant); err != nil { log.Error(err, "failed to provision secrets") + if statusErr := r.persistTenantStatus(ctx, &tenant, log); statusErr != nil { + return ctrl.Result{}, statusErr + } return ctrl.Result{RequeueAfter: 30 * time.Second}, nil } tenant.Status.SecretsProvisioned = true @@ -104,6 +115,9 @@ func (r *TenantReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr if !tenant.Status.RBACConfigured { if err := r.configureRBAC(ctx, &tenant); err != nil { log.Error(err, "failed to configure RBAC") + if statusErr := r.persistTenantStatus(ctx, &tenant, log); statusErr != nil { + return ctrl.Result{}, statusErr + } return ctrl.Result{RequeueAfter: 30 * time.Second}, nil } tenant.Status.RBACConfigured = true @@ -114,6 +128,9 @@ func (r *TenantReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr if !tenant.Status.QuotasEnforced { if err := r.enforceQuotas(ctx, &tenant); err != nil { log.Error(err, "failed to enforce quotas") + if statusErr := r.persistTenantStatus(ctx, &tenant, log); statusErr != nil { + return ctrl.Result{}, statusErr + } return ctrl.Result{RequeueAfter: 30 * time.Second}, nil } tenant.Status.QuotasEnforced = true @@ -130,8 +147,7 @@ func (r *TenantReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr tenant.Status.Phase = "Active" tenant.Status.LastReconciliation = &metav1.Time{Time: time.Now()} - if err := r.Status().Update(ctx, &tenant); err != nil { - log.Error(err, "failed to update status") + if err := r.persistTenantStatus(ctx, &tenant, log); err != nil { return ctrl.Result{}, err } @@ -139,6 +155,31 @@ func (r *TenantReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr return ctrl.Result{RequeueAfter: 5 * time.Minute}, nil } +func (r *TenantReconciler) persistTenantStatus(ctx context.Context, tenant *agenticv1alpha1.Tenant, log logr.Logger) error { + if err := r.Status().Update(ctx, tenant); err != nil { + log.Error(err, "failed to update status") + return err + } + return nil +} + +func (r *TenantReconciler) updateNetworkPolicyStatus(ctx context.Context, tenant *agenticv1alpha1.Tenant) { + if r.DiscoveryClient == nil || r.DaemonSets == nil { + tenant.Status.NetworkPolicyActive = false + tenant.Status.NetworkPolicyEnforcementReason = "DetectionFailed" + return + } + detection, err := netpolicy.DetectCNIEnforcement(ctx, r.DiscoveryClient, r.DaemonSets) + if err != nil { + tenant.Status.NetworkPolicyActive = false + tenant.Status.NetworkPolicyEnforcementReason = "DetectionFailed" + logf.FromContext(ctx).Error(err, "failed to detect NetworkPolicy CNI enforcement", "tenant", tenant.Name) + return + } + tenant.Status.NetworkPolicyActive = detection.Enforcement == netpolicy.EnforcementEnforcing + tenant.Status.NetworkPolicyEnforcementReason = string(detection.Enforcement) +} + func (r *TenantReconciler) createNamespace(ctx context.Context, tenant *agenticv1alpha1.Tenant) error { ns := &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ diff --git a/internal/controller/tenant_networkpolicy_test.go b/internal/controller/tenant_networkpolicy_test.go new file mode 100644 index 0000000..1698bde --- /dev/null +++ b/internal/controller/tenant_networkpolicy_test.go @@ -0,0 +1,100 @@ +package controller + +import ( + "context" + "errors" + "testing" + + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + agenticv1alpha1 "github.com/Clawdlinux/agentic-operator-core/api/v1alpha1" + "github.com/Clawdlinux/agentic-operator-core/internal/netpolicy" +) + +func TestTenantReconcilerNetworkPolicyStatus(t *testing.T) { + tests := []struct { + name string + groups *metav1.APIGroupList + daemonSets *appsv1.DaemonSetList + err error + wantActive bool + wantReason string + }{ + {name: "enforcing", groups: &metav1.APIGroupList{Groups: []metav1.APIGroup{{Name: "cilium.io"}}}, daemonSets: &appsv1.DaemonSetList{}, wantActive: true, wantReason: string(netpolicy.EnforcementEnforcing)}, + {name: "known non enforcing", groups: &metav1.APIGroupList{}, daemonSets: &appsv1.DaemonSetList{Items: []appsv1.DaemonSet{{ObjectMeta: metav1.ObjectMeta{Name: "kindnet", Namespace: "kube-system"}}}}, wantReason: string(netpolicy.EnforcementKnownNonEnforcing)}, + {name: "unknown", groups: &metav1.APIGroupList{}, daemonSets: &appsv1.DaemonSetList{}, wantReason: string(netpolicy.EnforcementUnknown)}, + {name: "detection failure", err: errors.New("API unavailable"), wantReason: "DetectionFailed"}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + reconciler := &TenantReconciler{DiscoveryClient: tenantDiscoveryClient{groups: test.groups, err: test.err}, DaemonSets: tenantDaemonSetClient{daemonSets: test.daemonSets, err: test.err}} + tenant := &agenticv1alpha1.Tenant{ObjectMeta: metav1.ObjectMeta{Name: "tenant-a"}} + reconciler.updateNetworkPolicyStatus(context.Background(), tenant) + if tenant.Status.NetworkPolicyActive != test.wantActive || tenant.Status.NetworkPolicyEnforcementReason != test.wantReason { + t.Fatalf("status = active=%t reason=%q, want active=%t reason=%q", tenant.Status.NetworkPolicyActive, tenant.Status.NetworkPolicyEnforcementReason, test.wantActive, test.wantReason) + } + }) + } +} + +func TestTenantReconcilerNetworkPolicyStatusFailsClosedWithoutClients(t *testing.T) { + tenant := &agenticv1alpha1.Tenant{} + (&TenantReconciler{}).updateNetworkPolicyStatus(context.Background(), tenant) + if tenant.Status.NetworkPolicyActive || tenant.Status.NetworkPolicyEnforcementReason != "DetectionFailed" { + t.Fatalf("status = %#v, want inactive DetectionFailed", tenant.Status) + } +} + +func TestTenantReconcilePersistsNetworkPolicyStatusOnProvisioningFailure(t *testing.T) { + tenant := &agenticv1alpha1.Tenant{ + ObjectMeta: metav1.ObjectMeta{Name: "tenant-a"}, + Spec: agenticv1alpha1.TenantSpec{Namespace: "tenant-a", Providers: []string{"openai"}}, + Status: agenticv1alpha1.TenantStatus{NamespaceCreated: true}, + } + scheme := newControllerTestScheme(t) + client := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&agenticv1alpha1.Tenant{}).WithObjects(tenant).Build() + reconciler := &TenantReconciler{ + Client: client, + Scheme: scheme, + DiscoveryClient: tenantDiscoveryClient{groups: &metav1.APIGroupList{Groups: []metav1.APIGroup{{Name: "cilium.io"}}}}, + DaemonSets: tenantDaemonSetClient{daemonSets: &appsv1.DaemonSetList{}}, + } + + result, err := reconciler.Reconcile(context.Background(), ctrl.Request{NamespacedName: types.NamespacedName{Name: tenant.Name}}) + if err != nil { + t.Fatalf("Reconcile() error = %v", err) + } + if result.RequeueAfter == 0 { + t.Fatal("Reconcile() did not requeue after provisioning failure") + } + updated := &agenticv1alpha1.Tenant{} + if err := client.Get(context.Background(), types.NamespacedName{Name: tenant.Name}, updated); err != nil { + t.Fatalf("get updated Tenant: %v", err) + } + if !updated.Status.NetworkPolicyActive || updated.Status.NetworkPolicyEnforcementReason != string(netpolicy.EnforcementEnforcing) { + t.Fatalf("status = active=%t reason=%q, want active=true reason=%q", updated.Status.NetworkPolicyActive, updated.Status.NetworkPolicyEnforcementReason, netpolicy.EnforcementEnforcing) + } +} + +type tenantDiscoveryClient struct { + groups *metav1.APIGroupList + err error +} + +func (c tenantDiscoveryClient) ServerGroups() (*metav1.APIGroupList, error) { + return c.groups, c.err +} + +type tenantDaemonSetClient struct { + daemonSets *appsv1.DaemonSetList + err error +} + +func (c tenantDaemonSetClient) List(context.Context, metav1.ListOptions) (*appsv1.DaemonSetList, error) { + return c.daemonSets, c.err +} diff --git a/scripts/check_api_crd_compatibility.sh b/scripts/check_api_crd_compatibility.sh index 4b1f0fc..aa87a69 100755 --- a/scripts/check_api_crd_compatibility.sh +++ b/scripts/check_api_crd_compatibility.sh @@ -6,6 +6,6 @@ cd "${REPO_ROOT}" echo "Running API/CRD compatibility checks..." -go test ./api/v1alpha1 -run '^TestAgentWorkloadCompatibility_' -count=1 -v +go test ./api/v1alpha1 -run '^(TestAgentWorkloadCompatibility_|TestTenantCompatibility_)' -count=1 -v echo "API/CRD compatibility checks passed." \ No newline at end of file