From 4cb117df73ca2b44ca7ebe87bd7eda0a4afdf093 Mon Sep 17 00:00:00 2001 From: Satyam Trivedi Date: Tue, 20 Oct 2026 00:42:42 +0530 Subject: [PATCH 1/2] fix(aws-kafka): async lifecycle, scheduled AWS/Kafka metrics, real broker subnets, managed KMS key Close the open real-cloud parity gaps in #1517: - ListNodes reports each broker's clientSubnet from the cluster's own BrokerNodeGroupInfo.ClientSubnets, spreading brokers evenly across them (round-robin by broker ID), instead of a fabricated subnet-. The field is omitted when the cluster records no subnets. - Under AsyncSettle a new cluster reports CREATING (create response, Describe, List) and settles to ACTIVE; a deleted cluster reports DELETING and is gone (NotFoundException) once its window ends, keeping its name until then. Update operations on a non-ACTIVE cluster are rejected with BadRequestException. A new VPC connection reports CREATING, then AVAILABLE. The default (AsyncSettle off) stays synchronous. The windows are per-record deadlines persisted in snapshots. - Publish DEFAULT-level AWS/Kafka metrics from the shared serve scheduler (Kafka is now a config.Tickable, ticked before CloudWatch), once a minute per ACTIVE provisioned cluster: ActiveControllerCount, GlobalTopicCount, GlobalPartitionCount and OfflinePartitionsCount on "Cluster Name", and UnderReplicatedPartitions and UnderMinIsrPartitionCount on "Cluster Name" + "Broker ID". Host and traffic metrics cloudemu does not model are not published. Kafka.SetMonitoring is wired to CloudWatch. - The default encryption-at-rest key is the account+region's AWS managed aws/kafka key ARN (one stable UUID key id per account and region), completed by the provider at create time and reported by both v1 and v2 describes, instead of a hard-coded key/msk-default in us-east-1. Part of #1517 --- providers/aws/aws.go | 10 +- providers/aws/kafka/clusters.go | 101 +++-- providers/aws/kafka/clusters_v2.go | 29 +- providers/aws/kafka/encryption.go | 97 +++++ providers/aws/kafka/kafka.go | 33 +- providers/aws/kafka/lifecycle.go | 144 +++++++ providers/aws/kafka/metrics.go | 142 +++++++ providers/aws/kafka/nodes.go | 37 +- providers/aws/kafka/operations.go | 4 + providers/aws/kafka/parity_test.go | 517 +++++++++++++++++++++++++ providers/aws/kafka/snapshot.go | 34 +- providers/aws/kafka/vpc_connections.go | 19 +- providers/aws/tickables_test.go | 22 +- server/aws/kafka/parity_sdk_test.go | 291 ++++++++++++++ server/aws/kafka/types.go | 28 +- 15 files changed, 1401 insertions(+), 107 deletions(-) create mode 100644 providers/aws/kafka/encryption.go create mode 100644 providers/aws/kafka/lifecycle.go create mode 100644 providers/aws/kafka/metrics.go create mode 100644 providers/aws/kafka/parity_test.go create mode 100644 server/aws/kafka/parity_sdk_test.go diff --git a/providers/aws/aws.go b/providers/aws/aws.go index 5aafa2483..06427cd47 100644 --- a/providers/aws/aws.go +++ b/providers/aws/aws.go @@ -500,6 +500,9 @@ func newProvider(o *config.Options, shared *GlobalServices) *Provider { // matching real EKS (which auto-creates the cluster SG and infers the VPC). p.EKS.SetSubnetResolver(p.VPC) p.SageMaker.SetMonitoring(p.CloudWatch) + // MSK publishes DEFAULT-level AWS/Kafka metrics for each ACTIVE provisioned + // cluster once a minute, from the serve scheduler (see Tickables). + p.Kafka.SetMonitoring(p.CloudWatch) // CloudWatch alarm -> SNS: an alarm state transition fires its configured // SNS-topic actions, fanning a notification out to the topic's subscribers. p.CloudWatch.SetSNSPublisher(p.SNS) @@ -638,9 +641,10 @@ func awsDrivers(p *Provider) *resourcediscovery.Drivers { // background ticker calls each one on its interval. New time-driven services // register here. func (p *Provider) Tickables() []config.Tickable { - // SQS and EFS run before CloudWatch so the alarms evaluated in the same - // tick see the queue gauges and AWS/EFS metrics they just published. - return []config.Tickable{p.SQS, p.EFS, p.CloudWatch, p.SSM, p.CloudFormation, p.Batch} + // SQS, EFS and Kafka run before CloudWatch so the alarms evaluated in the + // same tick see the queue gauges and AWS/EFS and AWS/Kafka metrics they just + // published. + return []config.Tickable{p.SQS, p.EFS, p.Kafka, p.CloudWatch, p.SSM, p.CloudFormation, p.Batch} } // Close tears down any real engines wired into the provider via diff --git a/providers/aws/kafka/clusters.go b/providers/aws/kafka/clusters.go index 31ca7b830..e580c43fd 100644 --- a/providers/aws/kafka/clusters.go +++ b/providers/aws/kafka/clusters.go @@ -5,8 +5,8 @@ import ( "encoding/json" "sort" "strings" + "time" - "github.com/stackshy/cloudemu/v2/internal/idgen" "github.com/stackshy/cloudemu/v2/services/kafka/driver" ) @@ -150,16 +150,17 @@ func (m *Mock) getClusterBR(arn string) (*clusterData, error) { // with mkErr so each op returns the exception the SDK actually models for it. func (m *Mock) getClusterErr(arn string, mkErr func(string, ...any) error) (*clusterData, error) { cd, ok := m.clusters.Get(arn) - if !ok { + if !ok || m.reapIfGone(arn, cd) { return nil, mkErr("cluster not found: %s", arn) } return cd, nil } -// CreateCluster provisions a provisioned (v1) cluster that is immediately -// Active. The cluster name is claimed atomically under createMu so a duplicate -// name is rejected with ConflictException. +// CreateCluster provisions a provisioned (v1) cluster. It reports ACTIVE, or +// CREATING for a settle window under AsyncSettle. The cluster name is claimed +// atomically under createMu so a duplicate name is rejected with +// ConflictException. // //nolint:gocritic // hugeParam: signature fixed by driver.Kafka (by-value input). func (m *Mock) CreateCluster(_ context.Context, in driver.CreateClusterInput) (*driver.Cluster, error) { @@ -192,37 +193,17 @@ func (m *Mock) CreateCluster(_ context.Context, in driver.CreateClusterInput) (* kafkaVer = defaultKafkaVer } - m.createMu.Lock() - defer m.createMu.Unlock() - - if _, dup := m.clusterNames.Get(in.ClusterName); dup { - return nil, conflict("cluster already exists: %s", in.ClusterName) - } - - arn := m.clusterARN(in.ClusterName) - - cluster := driver.Cluster{ - ClusterARN: arn, + return m.insertCluster(driver.Cluster{ ClusterName: in.ClusterName, ClusterType: driver.ClusterTypeProvisioned, - State: driver.ClusterStateActive, - CurrentVersion: "K" + idgen.GenerateID(""), KafkaVersion: kafkaVer, NumberOfBrokerNodes: in.NumberOfBrokerNodes, BrokerNodeGroupInfo: copyBrokerNodeGroupInfo(in.BrokerNodeGroupInfo), StorageMode: in.StorageMode, EnhancedMonitoring: in.EnhancedMonitoring, Tags: copyTags(in.Tags), - CreationTime: m.now(), RawOptions: copyRaw(in.RawOptions), - } - - m.clusters.Set(arn, &clusterData{cluster: cluster}) - m.clusterNames.Set(in.ClusterName, arn) - - out := snapshotCluster(cluster) - - return &out, nil + }) } // DescribeCluster returns a deep copy of the stored cluster. @@ -233,34 +214,55 @@ func (m *Mock) DescribeCluster(_ context.Context, arn string) (*driver.Cluster, } cd.mu.RLock() - defer cd.mu.RUnlock() - - out := snapshotCluster(cd.cluster) + out := m.observedClusterLocked(cd) + cd.mu.RUnlock() return &out, nil } -// ListClusters lists clusters (optionally filtered by name prefix), sorted by -// name for deterministic paging. -func (m *Mock) ListClusters( - _ context.Context, namePrefix string, page driver.Page, -) (clusters []driver.Cluster, next string, err error) { +// visibleClusters returns observed deep copies of every cluster real MSK would +// still list, lazily removing any whose DELETING window has elapsed. +func (m *Mock) visibleClusters() []driver.Cluster { vals := m.clusters.SortedValues() all := make([]driver.Cluster, 0, len(vals)) for _, cd := range vals { cd.mu.RLock() - snap := snapshotCluster(cd.cluster) + arn := cd.cluster.ClusterARN cd.mu.RUnlock() - if namePrefix != "" && !strings.HasPrefix(snap.ClusterName, namePrefix) { + if m.reapIfGone(arn, cd) { continue } + cd.mu.RLock() + snap := m.observedClusterLocked(cd) + cd.mu.RUnlock() + all = append(all, snap) } + return all +} + +// ListClusters lists clusters (optionally filtered by name prefix), sorted by +// name for deterministic paging. +func (m *Mock) ListClusters( + _ context.Context, namePrefix string, page driver.Page, +) (clusters []driver.Cluster, next string, err error) { + visible := m.visibleClusters() + + all := make([]driver.Cluster, 0, len(visible)) + + for i := range visible { + if namePrefix != "" && !strings.HasPrefix(visible[i].ClusterName, namePrefix) { + continue + } + + all = append(all, visible[i]) + } + sort.Slice(all, func(i, j int) bool { return all[i].ClusterName < all[j].ClusterName }) start, end, nextTok, err := m.paginate(len(all), page) @@ -271,7 +273,11 @@ func (m *Mock) ListClusters( return all[start:end], nextTok, nil } -// DeleteCluster removes a cluster and frees its name for reuse. +// DeleteCluster deletes a cluster. With AsyncSettle off (the default) it is +// removed at once and its name freed. Under AsyncSettle it reports DELETING via +// Describe/List for a settle window and is then removed (lazily, on the next +// read), as real MSK does. Deleting a cluster that is already DELETING returns +// DELETING again. func (m *Mock) DeleteCluster(_ context.Context, arn, currentVersion string) (arnOut, state string, err error) { cd, err := m.getCluster(arn) if err != nil { @@ -280,6 +286,12 @@ func (m *Mock) DeleteCluster(_ context.Context, arn, currentVersion string) (arn cd.mu.Lock() + if cd.cluster.State == driver.ClusterStateDeleting { + cd.mu.Unlock() + + return arn, driver.ClusterStateDeleting, nil + } + // Honor the optimistic-concurrency version: a stale currentVersion is a 400. if currentVersion != "" && cd.cluster.CurrentVersion != currentVersion { cd.mu.Unlock() @@ -290,10 +302,19 @@ func (m *Mock) DeleteCluster(_ context.Context, arn, currentVersion string) (arn name := cd.cluster.ClusterName cd.cluster.State = driver.ClusterStateDeleting + // A delete supersedes any still-open CREATING window. + cd.readyAt = time.Time{} + cd.deleteAt = m.settleUntil(clusterDeleteSettle) + async := !cd.deleteAt.IsZero() cd.mu.Unlock() - m.clusters.Delete(arn) - m.clusterNames.Delete(name) + if async { + return arn, driver.ClusterStateDeleting, nil + } + + m.reapMu.Lock() + m.removeCluster(arn, name) + m.reapMu.Unlock() return arn, driver.ClusterStateDeleting, nil } diff --git a/providers/aws/kafka/clusters_v2.go b/providers/aws/kafka/clusters_v2.go index afaf1acf2..b94d0a3b5 100644 --- a/providers/aws/kafka/clusters_v2.go +++ b/providers/aws/kafka/clusters_v2.go @@ -95,13 +95,18 @@ func (m *Mock) createClusterV2Serverless(in driver.CreateClusterV2Input) (*drive } // insertCluster finishes minting a cluster (ARN, version, state, creation time) -// and stores it atomically under createMu, rejecting a duplicate name. +// and stores it atomically under createMu, rejecting a duplicate name. The +// stored state is ACTIVE; under AsyncSettle a CREATING window is overlaid on +// reads, and the returned cluster reports it. // //nolint:gocritic // hugeParam: takes the assembled cluster by value to finalize. func (m *Mock) insertCluster(cluster driver.Cluster) (*driver.Cluster, error) { m.createMu.Lock() defer m.createMu.Unlock() + // A name held by a cluster that has finished deleting is free again. + m.reapName(cluster.ClusterName) + if _, dup := m.clusterNames.Get(cluster.ClusterName); dup { return nil, conflict("cluster already exists: %s", cluster.ClusterName) } @@ -111,10 +116,18 @@ func (m *Mock) insertCluster(cluster driver.Cluster) (*driver.Cluster, error) { cluster.CurrentVersion = "K" + idgen.GenerateID("") cluster.CreationTime = m.now() - m.clusters.Set(cluster.ClusterARN, &clusterData{cluster: cluster}) + if cluster.ClusterType == driver.ClusterTypeProvisioned { + cluster.RawOptions = m.withDefaultEncryption(cluster.RawOptions) + } + + cd := &clusterData{cluster: cluster, readyAt: m.settleUntil(clusterCreateSettle)} + + m.clusters.Set(cluster.ClusterARN, cd) m.clusterNames.Set(cluster.ClusterName, cluster.ClusterARN) - out := snapshotCluster(cluster) + cd.mu.RLock() + out := m.observedClusterLocked(cd) + cd.mu.RUnlock() return &out, nil } @@ -134,14 +147,12 @@ func (m *Mock) ListClustersV2( return nil, "", vErr } - vals := m.clusters.SortedValues() + visible := m.visibleClusters() - all := make([]driver.Cluster, 0, len(vals)) + all := make([]driver.Cluster, 0, len(visible)) - for _, cd := range vals { - cd.mu.RLock() - snap := snapshotCluster(cd.cluster) - cd.mu.RUnlock() + for i := range visible { + snap := visible[i] if namePrefix != "" && !strings.HasPrefix(snap.ClusterName, namePrefix) { continue diff --git a/providers/aws/kafka/encryption.go b/providers/aws/kafka/encryption.go new file mode 100644 index 000000000..7778bd282 --- /dev/null +++ b/providers/aws/kafka/encryption.go @@ -0,0 +1,97 @@ +package kafka + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + + "github.com/stackshy/cloudemu/v2/internal/idgen" +) + +// encryptionInfoKey is the RawOptions key holding a provisioned cluster's +// EncryptionInfo block (the wire layer stores the request block under it). +const encryptionInfoKey = "encryptionInfo" + +// managedKafkaKeyARN returns the ARN of this account+region's AWS managed key +// for MSK (alias/aws/kafka). Real MSK encrypts data at rest under that key when +// a create names none, and DescribeCluster reports its key ARN in +// encryptionAtRest.dataVolumeKMSKeyId +// (https://docs.aws.amazon.com/msk/latest/developerguide/msk-working-with-encryption.html). +// There is one such key per account and region, so the key id is a UUID derived +// deterministically from both: every cluster in the account+region reports the +// same key, as in AWS. +func (m *Mock) managedKafkaKeyARN() string { + sum := sha256.Sum256([]byte("alias/aws/kafka|" + m.opts.AccountID + "|" + m.opts.Region)) + // Shape the digest as a version-4 UUID, like a real KMS key id. + sum[6] = sum[6]&0x0f | 0x40 + sum[8] = sum[8]&0x3f | 0x80 + h := hex.EncodeToString(sum[:16]) + keyID := h[0:8] + "-" + h[8:12] + "-" + h[12:16] + "-" + h[16:20] + "-" + h[20:32] + + return idgen.AWSARN("kms", m.opts.Region, m.opts.AccountID, "key/"+keyID) +} + +// withDefaultEncryption returns raw with the cluster's EncryptionInfo completed +// the way real MSK completes it: with no encryptionInfo, at-rest encryption +// under the AWS managed key plus the documented in-transit defaults (ClientBroker +// TLS, InCluster true); with an encryptionInfo that omits encryptionAtRest (or +// its key), the managed key is filled in and the rest is kept verbatim. A block +// that already names a key is left untouched. raw is not mutated. +func (m *Mock) withDefaultEncryption(raw map[string]json.RawMessage) map[string]json.RawMessage { + out := copyRaw(raw) + if out == nil { + out = map[string]json.RawMessage{} + } + + keyARN := m.managedKafkaKeyARN() + + blk, ok := out[encryptionInfoKey] + if !ok || len(blk) == 0 { + out[encryptionInfoKey] = mustJSON(map[string]any{ + "encryptionAtRest": map[string]any{"dataVolumeKMSKeyId": keyARN}, + "encryptionInTransit": map[string]any{"clientBroker": "TLS", "inCluster": true}, + }) + + return out + } + + var info map[string]json.RawMessage + if err := json.Unmarshal(blk, &info); err != nil || info == nil { + return out + } + + var atRest map[string]json.RawMessage + if r, has := info["encryptionAtRest"]; has { + _ = json.Unmarshal(r, &atRest) + } + + if atRest == nil { + atRest = map[string]json.RawMessage{} + } + + var existing string + if k, has := atRest["dataVolumeKMSKeyId"]; has { + _ = json.Unmarshal(k, &existing) + } + + if existing != "" { + return out + } + + atRest["dataVolumeKMSKeyId"] = mustJSON(keyARN) + info["encryptionAtRest"] = mustJSON(atRest) + out[encryptionInfoKey] = mustJSON(info) + + return out +} + +// mustJSON marshals a value built from JSON-safe types (maps, strings, bools, +// raw messages), which cannot fail. +func mustJSON(v any) json.RawMessage { + b, err := json.Marshal(v) + if err != nil { + return json.RawMessage("null") + } + + return b +} diff --git a/providers/aws/kafka/kafka.go b/providers/aws/kafka/kafka.go index b4316fbb3..85371dc0e 100644 --- a/providers/aws/kafka/kafka.go +++ b/providers/aws/kafka/kafka.go @@ -10,9 +10,11 @@ // secrets, policy, operations) live on clusterData under its lock so a mutation // is atomic with its reads. // -// Clusters are provisioned immediately Active (deterministic, no wall-clock -// wait), with a server-minted ARN. Cluster names are unique per account -// (ConflictException on a duplicate). All reads deep-copy maps, slices, and +// Clusters report ACTIVE immediately by default (deterministic, no wall-clock +// wait), with a server-minted ARN. Under config.Options.AsyncSettle a new +// cluster reports CREATING and a deleted one DELETING for a settle window (see +// lifecycle.go), and a new VPC connection reports CREATING. Cluster names are +// unique per account (ConflictException on a duplicate). All reads deep-copy maps, slices, and // json.RawMessage so a caller can never mutate stored state through a result. package kafka @@ -26,6 +28,7 @@ import ( "github.com/stackshy/cloudemu/v2/internal/idgen" "github.com/stackshy/cloudemu/v2/internal/memstore" "github.com/stackshy/cloudemu/v2/services/kafka/driver" + mondriver "github.com/stackshy/cloudemu/v2/services/monitoring/driver" ) // Compile-time check that Mock implements driver.Kafka. @@ -52,14 +55,25 @@ type clusterData struct { scramSecrets []string policy string policyVersion string - mu sync.RWMutex + + // readyAt closes the CREATING window of a new cluster and deleteAt the + // DELETING window of a deleted one (both zero unless AsyncSettle is on). + // metricsSampleAt is when Tick last published its AWS/Kafka metrics. + readyAt time.Time + deleteAt time.Time + metricsSampleAt time.Time + + mu sync.RWMutex } // vpcConnData is a stored VPC connection plus its lock. State transitions (e.g. // a client-vpc-connection reject) mutate under mu. type vpcConnData struct { vpc driver.VpcConnection - mu sync.RWMutex + // readyAt closes the CREATING window of a new connection (zero unless + // AsyncSettle is on). + readyAt time.Time + mu sync.RWMutex } // replicatorData is a stored replicator plus its lock. UpdateReplicationInfo @@ -103,6 +117,15 @@ type Mock struct { // serializes replicator creation so the name-scan and ARN insert are atomic. createMu sync.Mutex + // reapMu serializes the lazy removal of a cluster whose DELETING window has + // elapsed, so two readers never both free its name. Lock order: + // createMu -> reapMu -> clusterData.mu. + reapMu sync.Mutex + + // monitoring receives the AWS/Kafka metrics Tick publishes; nil (the + // default) disables publishing. + monitoring mondriver.Monitoring + opts *config.Options } diff --git a/providers/aws/kafka/lifecycle.go b/providers/aws/kafka/lifecycle.go new file mode 100644 index 000000000..383357579 --- /dev/null +++ b/providers/aws/kafka/lifecycle.go @@ -0,0 +1,144 @@ +package kafka + +import ( + "time" + + "github.com/stackshy/cloudemu/v2/internal/settle" + "github.com/stackshy/cloudemu/v2/services/kafka/driver" +) + +// Settle windows for the MSK lifecycle under config.Options.AsyncSettle. Real +// MSK takes many minutes for each; the emulator keeps the transitions short so +// a polling client observes them without slowing tests down. With AsyncSettle +// off, opts.SettleDuration returns 0 and every transition is immediate. +const ( + clusterCreateSettle = settle.DefaultClusterSettle // CREATING -> ACTIVE + clusterDeleteSettle = settle.DefaultClusterSettle // DELETING -> removed + vpcConnCreateSettle = settle.DefaultClusterSettle // CREATING -> AVAILABLE +) + +// vpcConnStateCreating is the transient state a new VPC connection reports +// under AsyncSettle (VpcConnectionState: CREATING | AVAILABLE | INACTIVE | +// UPDATING | DEACTIVATING | DELETING | FAILED | REJECTED | REJECTING). +const vpcConnStateCreating = "CREATING" + +// settleUntil returns the instant a window of the given default length opened +// now closes, or the zero time when AsyncSettle is off (no window). +func (m *Mock) settleUntil(d time.Duration) time.Time { + dur := m.opts.SettleDuration(d) + if dur <= 0 { + return time.Time{} + } + + return m.now().Add(dur) +} + +// clusterStateLocked returns the cluster's externally visible state: CREATING +// while its create window is open, otherwise the stored state. The caller holds +// cd.mu (read or write). +func (m *Mock) clusterStateLocked(cd *clusterData) string { + if cd.cluster.State == driver.ClusterStateActive && m.now().Before(cd.readyAt) { + return driver.ClusterStateCreating + } + + return cd.cluster.State +} + +// observedClusterLocked returns a deep copy of the stored cluster carrying its +// visible state. The caller holds cd.mu (read or write). +func (m *Mock) observedClusterLocked(cd *clusterData) driver.Cluster { + out := snapshotCluster(cd.cluster) + out.State = m.clusterStateLocked(cd) + + return out +} + +// requireActive rejects an update against a cluster that is not ACTIVE. Real +// MSK accepts no update operations while a cluster is CREATING or DELETING +// (https://docs.aws.amazon.com/msk/latest/developerguide/msk-cluster-states.html). +// Without AsyncSettle a cluster is always ACTIVE, so this never fires. The +// caller holds cd.mu. +func (m *Mock) requireActive(cd *clusterData) error { + if st := m.clusterStateLocked(cd); st != driver.ClusterStateActive { + return badRequest("cluster %s is in %s state; update operations require ACTIVE", + cd.cluster.ClusterARN, st) + } + + return nil +} + +// gone reports whether a stored cluster is DELETING and its delete window has +// elapsed, i.e. real MSK would no longer return it. A DELETING cluster with no +// window (restored from a snapshot that predates windows) counts as elapsed. +func (m *Mock) gone(cd *clusterData) bool { + cd.mu.RLock() + defer cd.mu.RUnlock() + + return cd.cluster.State == driver.ClusterStateDeleting && !m.now().Before(cd.deleteAt) +} + +// reapIfGone removes a cluster whose DELETING window has elapsed and frees its +// name, reporting whether it was removed. Removal is lazy (on the next read), +// mirroring how ELBv2 retires a drained target. +func (m *Mock) reapIfGone(arn string, cd *clusterData) bool { + if !m.gone(cd) { + return false + } + + m.reapMu.Lock() + defer m.reapMu.Unlock() + + // Re-check under reapMu: a concurrent reader may have removed it already. + cur, ok := m.clusters.Get(arn) + if !ok || cur != cd { + return true + } + + cd.mu.RLock() + name := cd.cluster.ClusterName + cd.mu.RUnlock() + + m.removeCluster(arn, name) + + return true +} + +// removeCluster drops a cluster from the store and frees its name, but only +// when the name still points at this ARN (a later cluster may hold it). +func (m *Mock) removeCluster(arn, name string) { + m.clusters.Delete(arn) + + if owner, ok := m.clusterNames.Get(name); ok && owner == arn { + m.clusterNames.Delete(name) + } +} + +// reapName frees name when it is claimed by a cluster that has finished +// deleting, so a create can reuse it. The caller holds createMu. +func (m *Mock) reapName(name string) { + arn, ok := m.clusterNames.Get(name) + if !ok { + return + } + + cd, ok := m.clusters.Get(arn) + if !ok { + m.clusterNames.Delete(name) + + return + } + + m.reapIfGone(arn, cd) +} + +// observedVpcConnectionLocked returns a copy of the stored VPC connection +// carrying its visible state: CREATING while its create window is open. The +// caller holds vd.mu (read or write). +func (m *Mock) observedVpcConnectionLocked(vd *vpcConnData) driver.VpcConnection { + out := snapshotVpcConnection(vd.vpc) + if out.State == vpcConnStateAvailable && m.now().Before(vd.readyAt) { + out.State = vpcConnStateCreating + } + + return out +} diff --git a/providers/aws/kafka/metrics.go b/providers/aws/kafka/metrics.go new file mode 100644 index 000000000..5bb746b40 --- /dev/null +++ b/providers/aws/kafka/metrics.go @@ -0,0 +1,142 @@ +package kafka + +import ( + "context" + "sort" + "strconv" + "time" + + "github.com/stackshy/cloudemu/v2/services/kafka/driver" + mondriver "github.com/stackshy/cloudemu/v2/services/monitoring/driver" +) + +// AWS/Kafka CloudWatch metrics for provisioned clusters +// (https://docs.aws.amazon.com/msk/latest/developerguide/metrics-details.html). +// +// Real MSK pushes its metrics on a schedule, not per request: "The metrics that +// you configure for your MSK Provisioned clusters are automatically collected +// and pushed to CloudWatch at 1 minute intervals." Tick does the same on the +// shared serve scheduler, and only for clusters that are ACTIVE (every DEFAULT +// metric is documented as visible "After the cluster gets to the ACTIVE +// state"). The dimension names contain spaces ("Cluster Name", "Broker ID"). +// +// Only DEFAULT-level metrics whose values cloudemu actually knows are +// published (DEFAULT metrics are published at every monitoring level): +// +// - Cluster Name: ActiveControllerCount (1: exactly one active controller), +// GlobalTopicCount and GlobalPartitionCount (from the cluster's stored +// topics; partitions exclude replicas, as documented) and +// OfflinePartitionsCount (0: no broker is ever unavailable). +// - Cluster Name, Broker ID (brokers 1..NumberOfBrokerNodes, the IDs +// ListNodes reports): UnderReplicatedPartitions and +// UnderMinIsrPartitionCount (0: replicas never fall behind, since no +// broker runs). +// +// The CPU, memory, disk, network, connection and throughput metrics describe +// broker hosts and client traffic that cloudemu does not run, and per-broker +// PartitionCount/LeaderCount need a partition assignment it does not model, +// so none of them is published. Serverless clusters publish a different +// metric set and are skipped. +const ( + metricsNamespace = "AWS/Kafka" + + dimClusterName = "Cluster Name" + dimBrokerID = "Broker ID" + + unitCount = "Count" + + // metricsInterval is how often the AWS/Kafka metrics are published. + metricsInterval = time.Minute +) + +// SetMonitoring wires the CloudWatch backend that receives the AWS/Kafka +// metrics. With no backend wired (the default) nothing is published. +func (m *Mock) SetMonitoring(mon mondriver.Monitoring) { + m.monitoring = mon +} + +// Tick publishes the AWS/Kafka metrics that are due at now for every ACTIVE +// provisioned cluster, and reports whether it published anything. It has the +// config.Tickable signature of the shared serve scheduler; library mode never +// calls it, so tests drive it with a FakeClock. Each cluster is sampled under +// its own lock and everything is published after all locks are released. +func (m *Mock) Tick(now time.Time) bool { + if m.monitoring == nil { + return false + } + + arns := m.clusters.Keys() + sort.Strings(arns) + + var data []mondriver.MetricDatum + + for _, arn := range arns { + cd, ok := m.clusters.Get(arn) + if !ok { + continue + } + + data = append(data, m.dueMetrics(cd, now)...) + } + + if len(data) == 0 { + return false + } + + _ = m.monitoring.PutMetricData(context.Background(), data) + + return true +} + +// dueMetrics samples cd's metrics if they are due at now and records the sample +// time. A cluster that is not ACTIVE, or is serverless, reports nothing. +// Callers must not hold cd.mu. +func (m *Mock) dueMetrics(cd *clusterData, now time.Time) []mondriver.MetricDatum { + cd.mu.Lock() + defer cd.mu.Unlock() + + if cd.cluster.ClusterType != driver.ClusterTypeProvisioned || + m.clusterStateLocked(cd) != driver.ClusterStateActive { + return nil + } + + if !cd.metricsSampleAt.IsZero() && now.Sub(cd.metricsSampleAt) < metricsInterval { + return nil + } + + cd.metricsSampleAt = now + + name := cd.cluster.ClusterName + + var partitions int64 + for _, t := range cd.topics { + partitions += int64(t.NumberOfPartitions) + } + + datum := func(metric string, value float64, dims map[string]string) mondriver.MetricDatum { + return mondriver.MetricDatum{ + Namespace: metricsNamespace, MetricName: metric, Value: value, + Unit: unitCount, Dimensions: dims, Timestamp: now, + } + } + + clusterDims := map[string]string{dimClusterName: name} + + data := []mondriver.MetricDatum{ + datum("ActiveControllerCount", 1, clusterDims), + datum("GlobalTopicCount", float64(len(cd.topics)), clusterDims), + datum("GlobalPartitionCount", float64(partitions), clusterDims), + datum("OfflinePartitionsCount", 0, clusterDims), + } + + for i := 1; i <= int(cd.cluster.NumberOfBrokerNodes); i++ { + dims := map[string]string{dimClusterName: name, dimBrokerID: strconv.Itoa(i)} + + data = append(data, + datum("UnderReplicatedPartitions", 0, dims), + datum("UnderMinIsrPartitionCount", 0, dims), + ) + } + + return data +} diff --git a/providers/aws/kafka/nodes.go b/providers/aws/kafka/nodes.go index 312089942..d6e4baae1 100644 --- a/providers/aws/kafka/nodes.go +++ b/providers/aws/kafka/nodes.go @@ -9,8 +9,9 @@ import ( ) // ListNodes returns one synthetic broker node per broker in the cluster, sized -// to the cluster's NumberOfBrokerNodes. Broker IDs are 1..N; endpoints and the -// client subnet are derived deterministically from the cluster name. +// to the cluster's NumberOfBrokerNodes. Broker IDs are 1..N; endpoints are +// derived deterministically from the cluster name, and each broker's client +// subnet is one of the cluster's own ClientSubnets (see brokerSubnet). func (m *Mock) ListNodes( _ context.Context, arn string, page driver.Page, ) (nodes []driver.Node, next string, err error) { @@ -45,6 +46,11 @@ func (m *Mock) synthNodes(c *driver.Cluster) []driver.Node { instance = c.BrokerNodeGroupInfo.InstanceType } + var subnets []string + if c.BrokerNodeGroupInfo != nil { + subnets = c.BrokerNodeGroupInfo.ClientSubnets + } + nodes := make([]driver.Node, 0, count) for i := 1; i <= count; i++ { @@ -56,7 +62,7 @@ func (m *Mock) synthNodes(c *driver.Cluster) []driver.Node { NodeType: "BROKER", InstanceType: instance, RawOptions: map[string]json.RawMessage{ - "brokerNodeInfo": brokerNodeInfoJSON(i, endpoint, c.KafkaVersion), + "brokerNodeInfo": brokerNodeInfoJSON(i, endpoint, brokerSubnet(subnets, i), c.KafkaVersion), }, }) } @@ -64,13 +70,28 @@ func (m *Mock) synthNodes(c *driver.Cluster) []driver.Node { return nodes } +// brokerSubnet returns the client subnet broker brokerID (1-based) lives in. +// Real MSK distributes brokers evenly across the cluster's client subnets (one +// per AZ), so brokers are assigned round-robin; "" when the cluster has none. +func brokerSubnet(subnets []string, brokerID int) string { + if len(subnets) == 0 { + return "" + } + + return subnets[(brokerID-1)%len(subnets)] +} + // brokerNodeInfoJSON builds the raw brokerNodeInfo block for a synthetic node. -// brokerId is a JSON number (the SDK models it as *float64). -func brokerNodeInfoJSON(brokerID int, endpoint, kafkaVersion string) json.RawMessage { +// brokerId is a JSON number (the SDK models it as *float64). clientSubnet is +// omitted when the cluster records no client subnets. +func brokerNodeInfoJSON(brokerID int, endpoint, subnet, kafkaVersion string) json.RawMessage { info := map[string]any{ - "brokerId": brokerID, - "clientSubnet": "subnet-" + strconv.Itoa(brokerID), - "endpoints": []string{endpoint}, + "brokerId": brokerID, + "endpoints": []string{endpoint}, + } + + if subnet != "" { + info["clientSubnet"] = subnet } if kafkaVersion != "" { diff --git a/providers/aws/kafka/operations.go b/providers/aws/kafka/operations.go index 5ba752736..34e2b9470 100644 --- a/providers/aws/kafka/operations.go +++ b/providers/aws/kafka/operations.go @@ -117,6 +117,10 @@ func (m *Mock) mutateClusterErr( cd.mu.Lock() defer cd.mu.Unlock() + if err := m.requireActive(cd); err != nil { + return nil, err + } + if currentVersion != "" && cd.cluster.CurrentVersion != currentVersion { return nil, badRequest( "currentVersion %q does not match cluster version %q", diff --git a/providers/aws/kafka/parity_test.go b/providers/aws/kafka/parity_test.go new file mode 100644 index 000000000..9032236a5 --- /dev/null +++ b/providers/aws/kafka/parity_test.go @@ -0,0 +1,517 @@ +package kafka_test + +import ( + "context" + "encoding/json" + "errors" + "strconv" + "strings" + "testing" + "time" + + "github.com/stackshy/cloudemu/v2/config" + "github.com/stackshy/cloudemu/v2/providers/aws/cloudwatch" + "github.com/stackshy/cloudemu/v2/providers/aws/kafka" + "github.com/stackshy/cloudemu/v2/services/kafka/driver" + mondriver "github.com/stackshy/cloudemu/v2/services/monitoring/driver" +) + +// settleWindow comfortably covers the provider's settle windows. +const settleWindow = 10 * time.Second + +func newSettleMock(t *testing.T) (*kafka.Mock, *config.FakeClock) { + t.Helper() + + fc := config.NewFakeClock(time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)) + + return kafka.New(config.NewOptions(config.WithClock(fc), config.WithAsyncSettle())), fc +} + +func createThreeAZCluster(t *testing.T, m *kafka.Mock, name string) *driver.Cluster { + t.Helper() + + c, err := m.CreateCluster(context.Background(), driver.CreateClusterInput{ + ClusterName: name, + NumberOfBrokerNodes: 6, + BrokerNodeGroupInfo: &driver.BrokerNodeGroupInfo{ + ClientSubnets: []string{"subnet-aaa", "subnet-bbb", "subnet-ccc"}, + InstanceType: "kafka.m5.large", + }, + }) + if err != nil { + t.Fatalf("CreateCluster: %v", err) + } + + return c +} + +func wantException(t *testing.T, err error, ex string) { + t.Helper() + + var apiErr *driver.APIError + if !errors.As(err, &apiErr) || apiErr.Exception != ex { + t.Fatalf("want %s, got %v", ex, err) + } +} + +// TestListNodesClientSubnetFromCluster asserts each broker's clientSubnet is +// one of the cluster's ClientSubnets, distributed evenly (round-robin) the way +// MSK spreads brokers across AZs, not a fabricated subnet-. +func TestListNodesClientSubnetFromCluster(t *testing.T) { + m := newMock(t) + c := createThreeAZCluster(t, m, "subnets") + + nodes, _, err := m.ListNodes(context.Background(), c.ClusterARN, driver.Page{}) + if err != nil || len(nodes) != 6 { + t.Fatalf("ListNodes: %v len=%d", err, len(nodes)) + } + + want := []string{"subnet-aaa", "subnet-bbb", "subnet-ccc", "subnet-aaa", "subnet-bbb", "subnet-ccc"} + + for i, n := range nodes { + var info struct { + BrokerID int `json:"brokerId"` + ClientSubnet string `json:"clientSubnet"` + } + + if err := json.Unmarshal(n.RawOptions["brokerNodeInfo"], &info); err != nil { + t.Fatalf("decode brokerNodeInfo: %v", err) + } + + if info.BrokerID != i+1 || info.ClientSubnet != want[i] { + t.Fatalf("node %d: brokerId=%d clientSubnet=%q, want %d/%q", i, info.BrokerID, info.ClientSubnet, i+1, want[i]) + } + } +} + +// TestListNodesNoSubnetsOmitsClientSubnet asserts a cluster with no recorded +// client subnets does not get a fabricated one. +func TestListNodesNoSubnetsOmitsClientSubnet(t *testing.T) { + m := newMock(t) + ctx := context.Background() + + c, _ := m.CreateCluster(ctx, driver.CreateClusterInput{ClusterName: "nosubnet", NumberOfBrokerNodes: 2, BrokerNodeGroupInfo: bng()}) + + nodes, _, err := m.ListNodes(ctx, c.ClusterARN, driver.Page{}) + if err != nil { + t.Fatalf("ListNodes: %v", err) + } + + if strings.Contains(string(nodes[0].RawOptions["brokerNodeInfo"]), "clientSubnet") { + t.Fatalf("clientSubnet fabricated: %s", nodes[0].RawOptions["brokerNodeInfo"]) + } +} + +// TestClusterCreatingThenActive asserts that under AsyncSettle a new cluster is +// CREATING (create response, Describe, List) and rejects updates, then settles +// to ACTIVE. +func TestClusterCreatingThenActive(t *testing.T) { + m, fc := newSettleMock(t) + ctx := context.Background() + + c := createThreeAZCluster(t, m, "settling") + if c.State != driver.ClusterStateCreating { + t.Fatalf("create state = %s, want CREATING", c.State) + } + + d, err := m.DescribeCluster(ctx, c.ClusterARN) + if err != nil || d.State != driver.ClusterStateCreating { + t.Fatalf("describe = %v/%v, want CREATING", d, err) + } + + list, _, _ := m.ListClustersV2(ctx, "", "", driver.Page{}) + if len(list) != 1 || list[0].State != driver.ClusterStateCreating { + t.Fatalf("list = %+v, want one CREATING", list) + } + + _, err = m.UpdateBrokerCount(ctx, c.ClusterARN, c.CurrentVersion, 9) + wantException(t, err, driver.ExBadRequest) + + fc.Advance(settleWindow) + + d, _ = m.DescribeCluster(ctx, c.ClusterARN) + if d.State != driver.ClusterStateActive { + t.Fatalf("after settle state = %s, want ACTIVE", d.State) + } + + if _, err := m.UpdateBrokerCount(ctx, c.ClusterARN, d.CurrentVersion, 9); err != nil { + t.Fatalf("UpdateBrokerCount after ACTIVE: %v", err) + } +} + +// TestClusterActiveImmediatelyWithoutSettle pins the default behavior. +func TestClusterActiveImmediatelyWithoutSettle(t *testing.T) { + m := newMock(t) + + c := createThreeAZCluster(t, m, "instant") + if c.State != driver.ClusterStateActive { + t.Fatalf("state = %s, want ACTIVE", c.State) + } +} + +// TestDeleteClusterDeletingThenGone asserts that under AsyncSettle a deleted +// cluster stays visible as DELETING (Describe and List), keeps its name, and +// is gone once the window elapses. +func TestDeleteClusterDeletingThenGone(t *testing.T) { + m, fc := newSettleMock(t) + ctx := context.Background() + + c := createThreeAZCluster(t, m, "doomed") + fc.Advance(settleWindow) + + _, state, err := m.DeleteCluster(ctx, c.ClusterARN, "") + if err != nil || state != driver.ClusterStateDeleting { + t.Fatalf("DeleteCluster = %s/%v", state, err) + } + + d, err := m.DescribeCluster(ctx, c.ClusterARN) + if err != nil || d.State != driver.ClusterStateDeleting { + t.Fatalf("describe while deleting = %+v/%v, want DELETING", d, err) + } + + list, _, _ := m.ListClusters(ctx, "", driver.Page{}) + if len(list) != 1 || list[0].State != driver.ClusterStateDeleting { + t.Fatalf("list while deleting = %+v", list) + } + + // A second delete is idempotent; updates are rejected. + if _, state, err = m.DeleteCluster(ctx, c.ClusterARN, ""); err != nil || state != driver.ClusterStateDeleting { + t.Fatalf("second DeleteCluster = %s/%v", state, err) + } + + _, err = m.UpdateMonitoring(ctx, c.ClusterARN, "", json.RawMessage(`{"enhancedMonitoring":"PER_BROKER"}`)) + wantException(t, err, driver.ExBadRequest) + + // The name is still held while the cluster is DELETING. + _, err = m.CreateCluster(ctx, driver.CreateClusterInput{ClusterName: "doomed", NumberOfBrokerNodes: 3, BrokerNodeGroupInfo: bng()}) + wantException(t, err, driver.ExConflict) + + fc.Advance(settleWindow) + + _, err = m.DescribeCluster(ctx, c.ClusterARN) + wantException(t, err, driver.ExNotFound) + + if list, _, _ = m.ListClusters(ctx, "", driver.Page{}); len(list) != 0 { + t.Fatalf("list after delete = %+v", list) + } + + if _, err := m.CreateCluster(ctx, driver.CreateClusterInput{ClusterName: "doomed", NumberOfBrokerNodes: 3, BrokerNodeGroupInfo: bng()}); err != nil { + t.Fatalf("name not freed after delete: %v", err) + } +} + +// TestDeleteFreesNameForCreateWithoutRead asserts a create reclaims the name of +// a cluster whose delete window has elapsed even if nothing read it since. +func TestDeleteFreesNameForCreateWithoutRead(t *testing.T) { + m, fc := newSettleMock(t) + ctx := context.Background() + + c := createThreeAZCluster(t, m, "reuse") + fc.Advance(settleWindow) + + if _, _, err := m.DeleteCluster(ctx, c.ClusterARN, ""); err != nil { + t.Fatalf("DeleteCluster: %v", err) + } + + fc.Advance(settleWindow) + + if _, err := m.CreateCluster(ctx, driver.CreateClusterInput{ClusterName: "reuse", NumberOfBrokerNodes: 3, BrokerNodeGroupInfo: bng()}); err != nil { + t.Fatalf("CreateCluster reusing name: %v", err) + } +} + +// TestVpcConnectionCreatingThenAvailable asserts CREATING (the real transient +// VpcConnectionState; there is no PENDING) under AsyncSettle, then AVAILABLE. +func TestVpcConnectionCreatingThenAvailable(t *testing.T) { + m, fc := newSettleMock(t) + ctx := context.Background() + + c := createThreeAZCluster(t, m, "vpc-target") + fc.Advance(settleWindow) + + body := json.RawMessage(`{"targetClusterArn":"` + c.ClusterARN + + `","authentication":"SASL_IAM","vpcId":"vpc-1","clientSubnets":["subnet-x"],"securityGroups":["sg-1"]}`) + + v, err := m.CreateVpcConnection(ctx, body) + if err != nil || v.State != "CREATING" { + t.Fatalf("CreateVpcConnection = %+v/%v, want CREATING", v, err) + } + + d, _ := m.DescribeVpcConnection(ctx, v.VpcConnectionARN) + if d.State != "CREATING" { + t.Fatalf("describe state = %s, want CREATING", d.State) + } + + fc.Advance(settleWindow) + + d, _ = m.DescribeVpcConnection(ctx, v.VpcConnectionARN) + if d.State != "AVAILABLE" { + t.Fatalf("settled state = %s, want AVAILABLE", d.State) + } +} + +// kafkaSamples returns the per-minute SampleCount datapoints of an AWS/Kafka +// metric with exactly dims between from and to. +func kafkaSamples( + t *testing.T, cw *cloudwatch.Mock, metric string, dims map[string]string, from, to time.Time, +) []float64 { + t.Helper() + + res, err := cw.GetMetricData(context.Background(), mondriver.GetMetricInput{ + Namespace: "AWS/Kafka", MetricName: metric, Dimensions: dims, + StartTime: from, EndTime: to, Period: 60, Stat: "SampleCount", + }) + if err != nil { + t.Fatalf("GetMetricData %s: %v", metric, err) + } + + return res.Values +} + +// TestClusterMetricsPublishedByTick asserts the AWS/Kafka metrics come from the +// scheduler, not from requests: nothing is published on create, nothing while +// the cluster is CREATING, then one sample per minute once it is ACTIVE, under +// the documented dimension names ("Cluster Name", "Broker ID"), with the topic +// counts read from the cluster's stored topics. +func TestClusterMetricsPublishedByTick(t *testing.T) { + fc := config.NewFakeClock(time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)) + opts := config.NewOptions(config.WithClock(fc), config.WithAsyncSettle()) + m := kafka.New(opts) + cw := cloudwatch.New(opts) + m.SetMonitoring(cw) + + ctx := context.Background() + start := fc.Now() + c := createThreeAZCluster(t, m, "metered") + + ids, err := cw.ListMetricsDetailed(ctx) + if err != nil || len(ids) != 0 { + t.Fatalf("metrics published on create: %+v/%v", ids, err) + } + + if m.Tick(fc.Now()) { + t.Fatal("Tick published metrics for a CREATING cluster") + } + + fc.Advance(settleWindow) + + if _, err := m.CreateTopic(ctx, c.ClusterARN, json.RawMessage( + `{"topicName":"orders","partitionCount":6,"replicationFactor":3}`)); err != nil { + t.Fatalf("CreateTopic: %v", err) + } + + if !m.Tick(fc.Now()) { + t.Fatal("Tick published nothing for an ACTIVE cluster") + } + + // A second tick inside the same minute is not due. + fc.Advance(30 * time.Second) + + if m.Tick(fc.Now()) { + t.Fatal("Tick republished within the 1-minute interval") + } + + fc.Advance(30 * time.Second) + + if !m.Tick(fc.Now()) { + t.Fatal("Tick did not publish once the minute elapsed") + } + + end := fc.Now().Add(time.Second) + clusterDims := map[string]string{"Cluster Name": c.ClusterName} + + if got := kafkaSamples(t, cw, "ActiveControllerCount", clusterDims, start, end); len(got) != 2 { + t.Fatalf("ActiveControllerCount samples = %v, want 2 one-minute samples", got) + } + + topics, err := cw.GetMetricData(ctx, mondriver.GetMetricInput{ + Namespace: "AWS/Kafka", MetricName: "GlobalPartitionCount", Dimensions: clusterDims, + StartTime: start, EndTime: end, Period: 3600, Stat: "Maximum", + }) + if err != nil || len(topics.Values) != 1 || topics.Values[0] != 6 || topics.Unit != "Count" { + t.Fatalf("GlobalPartitionCount = %+v/%v, want 6 Count", topics, err) + } + + for broker := 1; broker <= 6; broker++ { + dims := map[string]string{"Cluster Name": c.ClusterName, "Broker ID": strconv.Itoa(broker)} + if got := kafkaSamples(t, cw, "UnderReplicatedPartitions", dims, start, end); len(got) != 2 { + t.Fatalf("broker %d UnderReplicatedPartitions samples = %v", broker, got) + } + } + + // Host and traffic metrics cloudemu cannot know are not fabricated. + ids, _ = cw.ListMetricsDetailed(ctx) + for _, id := range ids { + switch id.MetricName { + case "CpuUser", "CpuIdle", "MemoryUsed", "KafkaDataLogsDiskUsed", "BytesInPerSec": + t.Fatalf("fabricated metric published: %+v", id) + } + } +} + +// TestTickSkipsDeletingAndServerless asserts a DELETING cluster stops +// publishing and serverless clusters (a different metric set) never publish. +func TestTickSkipsDeletingAndServerless(t *testing.T) { + fc := config.NewFakeClock(time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)) + opts := config.NewOptions(config.WithClock(fc), config.WithAsyncSettle()) + m := kafka.New(opts) + m.SetMonitoring(cloudwatch.New(opts)) + + ctx := context.Background() + c := createThreeAZCluster(t, m, "short-lived") + fc.Advance(settleWindow) + + if _, _, err := m.DeleteCluster(ctx, c.ClusterARN, ""); err != nil { + t.Fatalf("DeleteCluster: %v", err) + } + + if m.Tick(fc.Now()) { + t.Fatal("Tick published for a DELETING cluster") + } + + if _, err := m.CreateClusterV2(ctx, driver.CreateClusterV2Input{ + ClusterName: "sls", + Serverless: json.RawMessage(`{"vpcConfigs":[{"subnetIds":["subnet-1","subnet-2"]}]}`), + }); err != nil { + t.Fatalf("CreateClusterV2: %v", err) + } + + fc.Advance(settleWindow) + + if m.Tick(fc.Now()) { + t.Fatal("Tick published for a serverless cluster") + } +} + +func encryptionInfo(t *testing.T, c *driver.Cluster) map[string]map[string]any { + t.Helper() + + var out map[string]map[string]any + if err := json.Unmarshal(c.RawOptions["encryptionInfo"], &out); err != nil { + t.Fatalf("decode encryptionInfo %s: %v", c.RawOptions["encryptionInfo"], err) + } + + return out +} + +// TestDefaultEncryptionAtRestKey asserts the default at-rest key is a KMS key +// ARN in the cluster's own account and region (one AWS managed aws/kafka key +// per account+region), not a hard-coded key/msk-default. +func TestDefaultEncryptionAtRestKey(t *testing.T) { + m := kafka.New(config.NewOptions(config.WithRegion("eu-west-1"), config.WithAccountID("123456789012"))) + ctx := context.Background() + + a := createThreeAZCluster(t, m, "enc-a") + b := createThreeAZCluster(t, m, "enc-b") + + keyA, _ := encryptionInfo(t, a)["encryptionAtRest"]["dataVolumeKMSKeyId"].(string) + keyB, _ := encryptionInfo(t, b)["encryptionAtRest"]["dataVolumeKMSKeyId"].(string) + + if !strings.HasPrefix(keyA, "arn:aws:kms:eu-west-1:123456789012:key/") || strings.Contains(keyA, "msk-default") { + t.Fatalf("default key = %q", keyA) + } + + if keyA != keyB { + t.Fatalf("managed key differs across clusters: %q vs %q", keyA, keyB) + } + + if len(strings.TrimPrefix(keyA, "arn:aws:kms:eu-west-1:123456789012:key/")) != 36 { + t.Fatalf("key id is not a UUID: %q", keyA) + } + + // An explicit key is kept; an in-transit-only block gets the managed key. + explicit := "arn:aws:kms:eu-west-1:123456789012:key/11111111-2222-3333-4444-555555555555" + + c, err := m.CreateCluster(ctx, driver.CreateClusterInput{ + ClusterName: "enc-explicit", NumberOfBrokerNodes: 3, BrokerNodeGroupInfo: bng(), + RawOptions: map[string]json.RawMessage{ + "encryptionInfo": json.RawMessage(`{"encryptionAtRest":{"dataVolumeKMSKeyId":"` + explicit + `"}}`), + }, + }) + if err != nil { + t.Fatalf("CreateCluster explicit: %v", err) + } + + if got := encryptionInfo(t, c)["encryptionAtRest"]["dataVolumeKMSKeyId"]; got != explicit { + t.Fatalf("explicit key = %v, want %s", got, explicit) + } + + d, err := m.CreateCluster(ctx, driver.CreateClusterInput{ + ClusterName: "enc-transit", NumberOfBrokerNodes: 3, BrokerNodeGroupInfo: bng(), + RawOptions: map[string]json.RawMessage{ + "encryptionInfo": json.RawMessage(`{"encryptionInTransit":{"clientBroker":"TLS_PLAINTEXT","inCluster":true}}`), + }, + }) + if err != nil { + t.Fatalf("CreateCluster transit-only: %v", err) + } + + info := encryptionInfo(t, d) + if info["encryptionAtRest"]["dataVolumeKMSKeyId"] != keyA || info["encryptionInTransit"]["clientBroker"] != "TLS_PLAINTEXT" { + t.Fatalf("transit-only block = %+v", info) + } +} + +// TestSnapshotSettleStates asserts the lifecycle windows survive a snapshot: a +// cluster and a VPC connection snapshotted mid-create restore still CREATING +// and settle on schedule, and a cluster snapshotted mid-delete restores +// DELETING and is gone once its window ends. +func TestSnapshotSettleStates(t *testing.T) { + src, fc := newSettleMock(t) + ctx := context.Background() + + target := createThreeAZCluster(t, src, "vpc-target") + deleting := createThreeAZCluster(t, src, "mid-delete") + + fc.Advance(settleWindow) + + creating := createThreeAZCluster(t, src, "mid-create") + + if _, _, err := src.DeleteCluster(ctx, deleting.ClusterARN, ""); err != nil { + t.Fatalf("DeleteCluster: %v", err) + } + + vpc, err := src.CreateVpcConnection(ctx, json.RawMessage( + `{"targetClusterArn":"`+target.ClusterARN+`","authentication":"SASL_IAM","vpcId":"vpc-1"}`)) + if err != nil { + t.Fatalf("CreateVpcConnection: %v", err) + } + + raw, err := src.Snapshot(ctx, true) + if err != nil { + t.Fatalf("Snapshot: %v", err) + } + + dst := kafka.New(config.NewOptions(config.WithClock(fc), config.WithAsyncSettle())) + if err := dst.Restore(ctx, raw); err != nil { + t.Fatalf("Restore: %v", err) + } + + got, err := dst.DescribeCluster(ctx, creating.ClusterARN) + if err != nil || got.State != driver.ClusterStateCreating { + t.Fatalf("restored mid-create = %+v/%v, want CREATING", got, err) + } + + got, err = dst.DescribeCluster(ctx, deleting.ClusterARN) + if err != nil || got.State != driver.ClusterStateDeleting { + t.Fatalf("restored mid-delete = %+v/%v, want DELETING", got, err) + } + + v, err := dst.DescribeVpcConnection(ctx, vpc.VpcConnectionARN) + if err != nil || v.State != "CREATING" { + t.Fatalf("restored vpc connection = %+v/%v, want CREATING", v, err) + } + + fc.Advance(settleWindow) + + if got, _ = dst.DescribeCluster(ctx, creating.ClusterARN); got.State != driver.ClusterStateActive { + t.Fatalf("restored cluster after window = %s, want ACTIVE", got.State) + } + + _, err = dst.DescribeCluster(ctx, deleting.ClusterARN) + wantException(t, err, driver.ExNotFound) + + if v, _ = dst.DescribeVpcConnection(ctx, vpc.VpcConnectionARN); v.State != "AVAILABLE" { + t.Fatalf("restored vpc connection after window = %s, want AVAILABLE", v.State) + } +} diff --git a/providers/aws/kafka/snapshot.go b/providers/aws/kafka/snapshot.go index cf989dcc0..f20ae0380 100644 --- a/providers/aws/kafka/snapshot.go +++ b/providers/aws/kafka/snapshot.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "time" "github.com/stackshy/cloudemu/v2/internal/snapshot" "github.com/stackshy/cloudemu/v2/services/kafka/driver" @@ -19,13 +20,20 @@ var _ snapshot.Snapshottable = (*Mock)(nil) // index of *clusterData pointers shared with the clusters store, rebuilt on // restore from each cluster's operations slice so the shared identity (a // DescribeClusterOperation resolves to the same cluster object) is preserved. -// The per-record mutexes, createMu, and the wired opts are not serialized. +// The AsyncSettle lifecycle windows (a cluster's CREATING/DELETING deadline, a +// VPC connection's CREATING deadline) and the metric sample time are +// serialized, so a restored resource keeps its transient state until its window +// ends and Tick keeps its publishing cadence. The per-record mutexes, +// createMu/reapMu, the wired opts and the monitoring backend are not. type kafkaSnapshot struct { Clusters map[string]*clusterSnapshot `json:"clusters,omitempty"` ClusterNames json.RawMessage `json:"clusterNames,omitempty"` Configs map[string]driver.Configuration `json:"configs,omitempty"` VPCConns map[string]driver.VpcConnection `json:"vpcConns,omitempty"` - Replicators map[string]*replicatorSnapshot `json:"replicators,omitempty"` + // VPCConnReadyAt holds the CREATING deadline of each VPC connection still + // inside its window (kept beside VPCConns so older snapshots still load). + VPCConnReadyAt map[string]time.Time `json:"vpcConnReadyAt,omitempty"` + Replicators map[string]*replicatorSnapshot `json:"replicators,omitempty"` } // clusterSnapshot mirrors clusterData (all fields unexported). @@ -36,6 +44,10 @@ type clusterSnapshot struct { ScramSecrets []string `json:"scramSecrets,omitempty"` Policy string `json:"policy,omitempty"` PolicyVersion string `json:"policyVersion,omitempty"` + // Lifecycle windows (AsyncSettle) and the last metric sample time. + ReadyAt time.Time `json:"readyAt,omitzero"` + DeleteAt time.Time `json:"deleteAt,omitzero"` + MetricsSampleAt time.Time `json:"metricsSampleAt,omitzero"` } // replicatorSnapshot mirrors replicatorData. @@ -57,6 +69,7 @@ func (m *Mock) Snapshot(_ context.Context, _ bool) (json.RawMessage, error) { snap.Clusters[arn] = &clusterSnapshot{ Cluster: cd.cluster, Operations: cd.operations, Topics: cd.topics, ScramSecrets: cd.scramSecrets, Policy: cd.policy, PolicyVersion: cd.policyVersion, + ReadyAt: cd.readyAt, DeleteAt: cd.deleteAt, MetricsSampleAt: cd.metricsSampleAt, } cd.mu.RUnlock() } @@ -85,6 +98,14 @@ func (m *Mock) Snapshot(_ context.Context, _ bool) (json.RawMessage, error) { for arn, vd := range m.vpcConns.All() { vd.mu.RLock() snap.VPCConns[arn] = vd.vpc + + if !vd.readyAt.IsZero() { + if snap.VPCConnReadyAt == nil { + snap.VPCConnReadyAt = map[string]time.Time{} + } + + snap.VPCConnReadyAt[arn] = vd.readyAt + } vd.mu.RUnlock() } } @@ -112,9 +133,16 @@ func (m *Mock) Restore(_ context.Context, data json.RawMessage) error { } for arn, cs := range snap.Clusters { + // A snapshot from before the provider completed encryptionInfo at create + // lacks it; backfill MSK's default so the restored cluster reports it. + if cs.Cluster.ClusterType == driver.ClusterTypeProvisioned { + cs.Cluster.RawOptions = m.withDefaultEncryption(cs.Cluster.RawOptions) + } + cd := &clusterData{ cluster: cs.Cluster, operations: cs.Operations, topics: cs.Topics, scramSecrets: cs.ScramSecrets, policy: cs.Policy, policyVersion: cs.PolicyVersion, + readyAt: cs.ReadyAt, deleteAt: cs.DeleteAt, metricsSampleAt: cs.MetricsSampleAt, } m.clusters.Set(arn, cd) @@ -137,7 +165,7 @@ func (m *Mock) Restore(_ context.Context, data json.RawMessage) error { } for arn, vpc := range snap.VPCConns { - m.vpcConns.Set(arn, &vpcConnData{vpc: vpc}) + m.vpcConns.Set(arn, &vpcConnData{vpc: vpc, readyAt: snap.VPCConnReadyAt[arn]}) } for arn, rs := range snap.Replicators { diff --git a/providers/aws/kafka/vpc_connections.go b/providers/aws/kafka/vpc_connections.go index ba3af342a..cbb1a0e36 100644 --- a/providers/aws/kafka/vpc_connections.go +++ b/providers/aws/kafka/vpc_connections.go @@ -38,7 +38,8 @@ type createVpcConnectionRequest struct { // CreateVpcConnection mints a VPC connection in the AVAILABLE state, carrying // the client subnets and security groups verbatim in raw options so a later -// Describe reflects them. +// Describe reflects them. Under AsyncSettle it reports CREATING for a settle +// window first (real VpcConnectionState has CREATING and no PENDING value). func (m *Mock) CreateVpcConnection(_ context.Context, body json.RawMessage) (*driver.VpcConnection, error) { var req createVpcConnectionRequest if len(body) > 0 { @@ -79,11 +80,14 @@ func (m *Mock) CreateVpcConnection(_ context.Context, body json.RawMessage) (*dr // The ARN embeds a fresh UUID; SetIfAbsent keeps the create atomic and guards // the (impossible-but-cheap) collision. - if !m.vpcConns.SetIfAbsent(vpc.VpcConnectionARN, &vpcConnData{vpc: vpc}) { + vd := &vpcConnData{vpc: vpc, readyAt: m.settleUntil(vpcConnCreateSettle)} + if !m.vpcConns.SetIfAbsent(vpc.VpcConnectionARN, vd) { return nil, badRequest("vpc connection already exists: %s", vpc.VpcConnectionARN) } - out := snapshotVpcConnection(vpc) + vd.mu.RLock() + out := m.observedVpcConnectionLocked(vd) + vd.mu.RUnlock() return &out, nil } @@ -139,9 +143,8 @@ func (m *Mock) DescribeVpcConnection(_ context.Context, arn string) (*driver.Vpc } vd.mu.RLock() - defer vd.mu.RUnlock() - - out := snapshotVpcConnection(vd.vpc) + out := m.observedVpcConnectionLocked(vd) + vd.mu.RUnlock() return &out, nil } @@ -154,7 +157,7 @@ func (m *Mock) DeleteVpcConnection(_ context.Context, arn string) (*driver.VpcCo } vd.mu.RLock() - out := snapshotVpcConnection(vd.vpc) + out := m.observedVpcConnectionLocked(vd) vd.mu.RUnlock() m.vpcConns.Delete(arn) @@ -204,7 +207,7 @@ func (m *Mock) snapshotVpcConnections(targetCluster string) []driver.VpcConnecti for _, vd := range vals { vd.mu.RLock() - snap := snapshotVpcConnection(vd.vpc) + snap := m.observedVpcConnectionLocked(vd) vd.mu.RUnlock() if targetCluster != "" && snap.TargetClusterARN != targetCluster { diff --git a/providers/aws/tickables_test.go b/providers/aws/tickables_test.go index ab76f1d5e..6f0dffaec 100644 --- a/providers/aws/tickables_test.go +++ b/providers/aws/tickables_test.go @@ -8,17 +8,23 @@ import ( ) // TestTickablesRegistersTimeDrivenServices checks the serve ticker reaches the -// region's own SQS, EFS, CloudWatch, SSM, CloudFormation and Batch, so queue -// gauges, EFS metrics, due alarms, parameter policies, settling stack operations -// and Batch job state-change events run without a read. SQS and EFS come before -// CloudWatch so alarms see the metrics published in the same tick. +// region's own SQS, EFS, Kafka, CloudWatch, SSM, CloudFormation and Batch, so +// queue gauges, EFS and Kafka metrics, due alarms, parameter policies, settling +// stack operations and Batch job state-change events run without a read. SQS, +// EFS and Kafka come before CloudWatch so alarms see the metrics published in +// the same tick. func TestTickablesRegistersTimeDrivenServices(t *testing.T) { p := aws.New() got := p.Tickables() - if len(got) != 6 || got[0] != config.Tickable(p.SQS) || got[1] != config.Tickable(p.EFS) || - got[2] != config.Tickable(p.CloudWatch) || got[3] != config.Tickable(p.SSM) || - got[4] != config.Tickable(p.CloudFormation) || got[5] != config.Tickable(p.Batch) { - t.Fatalf("Tickables() = %v, want this provider's SQS, EFS, CloudWatch, SSM, CloudFormation and Batch", got) + want := []config.Tickable{p.SQS, p.EFS, p.Kafka, p.CloudWatch, p.SSM, p.CloudFormation, p.Batch} + if len(got) != len(want) { + t.Fatalf("Tickables() = %v, want this provider's SQS, EFS, Kafka, CloudWatch, SSM, CloudFormation and Batch", got) + } + + for i := range want { + if got[i] != want[i] { + t.Fatalf("Tickables()[%d] = %v, want %v", i, got[i], want[i]) + } } } diff --git a/server/aws/kafka/parity_sdk_test.go b/server/aws/kafka/parity_sdk_test.go new file mode 100644 index 000000000..f34fc0932 --- /dev/null +++ b/server/aws/kafka/parity_sdk_test.go @@ -0,0 +1,291 @@ +package kafka_test + +import ( + "context" + "errors" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + awsconfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + awscw "github.com/aws/aws-sdk-go-v2/service/cloudwatch" + cwtypes "github.com/aws/aws-sdk-go-v2/service/cloudwatch/types" + awskafka "github.com/aws/aws-sdk-go-v2/service/kafka" + kafkatypes "github.com/aws/aws-sdk-go-v2/service/kafka/types" + + "github.com/stackshy/cloudemu/v2" + cloudconfig "github.com/stackshy/cloudemu/v2/config" + awsprovider "github.com/stackshy/cloudemu/v2/providers/aws" + awsserver "github.com/stackshy/cloudemu/v2/server/aws" +) + +// parityEnv is an in-process AWS server (Kafka + CloudWatch from one provider, +// so the AWS/Kafka metrics land in the CloudWatch the client reads) with real +// SDK clients, the FakeClock driving settle windows, and the provider whose +// Tickables the serve scheduler would run. +type parityEnv struct { + kafka *awskafka.Client + cw *awscw.Client + clock *cloudconfig.FakeClock + cloud *awsprovider.Provider +} + +// tick runs one round of the serve scheduler at the current FakeClock time. +func (e *parityEnv) tick() { + for _, tk := range e.cloud.Tickables() { + tk.Tick(e.clock.Now()) + } +} + +func newParityEnv(t *testing.T, opts ...cloudconfig.Option) *parityEnv { + t.Helper() + + c, cw, fc, cloud := newParityClientsFull(t, opts...) + + return &parityEnv{kafka: c, cw: cw, clock: fc, cloud: cloud} +} + +func newParityClients(t *testing.T, opts ...cloudconfig.Option) (*awskafka.Client, *awscw.Client, *cloudconfig.FakeClock) { + t.Helper() + + c, cw, fc, _ := newParityClientsFull(t, opts...) + + return c, cw, fc +} + +func newParityClientsFull( + t *testing.T, opts ...cloudconfig.Option, +) (*awskafka.Client, *awscw.Client, *cloudconfig.FakeClock, *awsprovider.Provider) { + t.Helper() + + fc := cloudconfig.NewFakeClock(time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)) + cloud := cloudemu.NewAWS(append([]cloudconfig.Option{cloudconfig.WithClock(fc)}, opts...)...) + srv := awsserver.New(awsserver.Drivers{Kafka: cloud.Kafka, CloudWatch: cloud.CloudWatch}) + + ts := httptest.NewServer(srv) + t.Cleanup(ts.Close) + + cfg, err := awsconfig.LoadDefaultConfig(context.Background(), + awsconfig.WithRegion("us-east-1"), + awsconfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider("test", "test", "")), + ) + if err != nil { + t.Fatalf("aws config: %v", err) + } + + cfg.BaseEndpoint = aws.String(ts.URL) + + return awskafka.NewFromConfig(cfg), awscw.NewFromConfig(cfg), fc, cloud +} + +func createSDKCluster(ctx context.Context, t *testing.T, c *awskafka.Client, name string) string { + t.Helper() + + out, err := c.CreateCluster(ctx, &awskafka.CreateClusterInput{ + ClusterName: aws.String(name), + KafkaVersion: aws.String("3.6.0"), + NumberOfBrokerNodes: aws.Int32(3), + BrokerNodeGroupInfo: &kafkatypes.BrokerNodeGroupInfo{ + ClientSubnets: []string{"subnet-0a1", "subnet-0b2", "subnet-0c3"}, + InstanceType: aws.String("kafka.m5.large"), + }, + }) + if err != nil { + t.Fatalf("CreateCluster: %v", err) + } + + return aws.ToString(out.ClusterArn) +} + +// TestSDKListNodesClientSubnet asserts ListNodes reports the cluster's own +// client subnets through the wire. +func TestSDKListNodesClientSubnet(t *testing.T) { + ctx := context.Background() + c, _, _ := newParityClients(t) + arn := createSDKCluster(ctx, t, c, "sdk-subnets") + + out, err := c.ListNodes(ctx, &awskafka.ListNodesInput{ClusterArn: aws.String(arn)}) + if err != nil { + t.Fatalf("ListNodes: %v", err) + } + + want := []string{"subnet-0a1", "subnet-0b2", "subnet-0c3"} + for i, n := range out.NodeInfoList { + if got := aws.ToString(n.BrokerNodeInfo.ClientSubnet); got != want[i] { + t.Fatalf("node %d clientSubnet = %q, want %q", i, got, want[i]) + } + } +} + +// TestSDKAsyncClusterLifecycle drives CREATING -> ACTIVE -> DELETING -> gone +// through the real SDK under AsyncSettle. +func TestSDKAsyncClusterLifecycle(t *testing.T) { + ctx := context.Background() + c, _, fc := newParityClients(t, cloudconfig.WithAsyncSettle()) + arn := createSDKCluster(ctx, t, c, "sdk-async") + + desc, err := c.DescribeCluster(ctx, &awskafka.DescribeClusterInput{ClusterArn: aws.String(arn)}) + if err != nil || desc.ClusterInfo.State != kafkatypes.ClusterStateCreating { + t.Fatalf("describe while creating = %v/%v", desc, err) + } + + _, err = c.UpdateMonitoring(ctx, &awskafka.UpdateMonitoringInput{ + ClusterArn: aws.String(arn), CurrentVersion: desc.ClusterInfo.CurrentVersion, + EnhancedMonitoring: kafkatypes.EnhancedMonitoringPerBroker, + }) + + var br *kafkatypes.BadRequestException + if !errors.As(err, &br) { + t.Fatalf("update while CREATING = %v, want BadRequestException", err) + } + + fc.Advance(10 * time.Second) + + v2, err := c.DescribeClusterV2(ctx, &awskafka.DescribeClusterV2Input{ClusterArn: aws.String(arn)}) + if err != nil || v2.ClusterInfo.State != kafkatypes.ClusterStateActive { + t.Fatalf("describe v2 after settle = %v/%v", v2, err) + } + + del, err := c.DeleteCluster(ctx, &awskafka.DeleteClusterInput{ClusterArn: aws.String(arn)}) + if err != nil || del.State != kafkatypes.ClusterStateDeleting { + t.Fatalf("DeleteCluster = %v/%v", del, err) + } + + desc, err = c.DescribeCluster(ctx, &awskafka.DescribeClusterInput{ClusterArn: aws.String(arn)}) + if err != nil || desc.ClusterInfo.State != kafkatypes.ClusterStateDeleting { + t.Fatalf("describe while deleting = %v/%v", desc, err) + } + + fc.Advance(10 * time.Second) + + _, err = c.DescribeCluster(ctx, &awskafka.DescribeClusterInput{ClusterArn: aws.String(arn)}) + + var nf *kafkatypes.NotFoundException + if !errors.As(err, &nf) { + t.Fatalf("describe after delete = %v, want NotFoundException", err) + } +} + +// TestSDKAsyncVpcConnectionCreating asserts a VPC connection reports CREATING +// then AVAILABLE through the wire under AsyncSettle. +func TestSDKAsyncVpcConnectionCreating(t *testing.T) { + ctx := context.Background() + c, _, fc := newParityClients(t, cloudconfig.WithAsyncSettle()) + arn := createSDKCluster(ctx, t, c, "sdk-vpc") + + fc.Advance(10 * time.Second) + + v, err := c.CreateVpcConnection(ctx, &awskafka.CreateVpcConnectionInput{ + TargetClusterArn: aws.String(arn), Authentication: aws.String("SASL_IAM"), + VpcId: aws.String("vpc-123"), ClientSubnets: []string{"subnet-x"}, SecurityGroups: []string{"sg-1"}, + }) + if err != nil || v.State != kafkatypes.VpcConnectionStateCreating { + t.Fatalf("CreateVpcConnection = %v/%v, want CREATING", v, err) + } + + fc.Advance(10 * time.Second) + + d, err := c.DescribeVpcConnection(ctx, &awskafka.DescribeVpcConnectionInput{Arn: v.VpcConnectionArn}) + if err != nil || d.State != kafkatypes.VpcConnectionStateAvailable { + t.Fatalf("DescribeVpcConnection = %v/%v, want AVAILABLE", d, err) + } +} + +// TestSDKKafkaMetricsAndDefaultKey asserts the AWS/Kafka metrics are published +// by the serve scheduler (none on create, one sample per minute after), +// readable through the CloudWatch API under the documented dimensions, and the +// default at-rest key is a real-shaped KMS key ARN in the v1 and v2 shapes. +func TestSDKKafkaMetricsAndDefaultKey(t *testing.T) { + ctx := context.Background() + env := newParityEnv(t) + c, cw := env.kafka, env.cw + start := env.clock.Now() + arn := createSDKCluster(ctx, t, c, "sdk-metrics") + + listed := func() []cwtypes.Metric { + out, err := cw.ListMetrics(ctx, &awscw.ListMetricsInput{Namespace: aws.String("AWS/Kafka")}) + if err != nil { + t.Fatalf("ListMetrics: %v", err) + } + + return out.Metrics + } + + if got := listed(); len(got) != 0 { + t.Fatalf("metrics published by a request: %+v", got) + } + + env.tick() + env.clock.Advance(time.Minute) + env.tick() + + var sawController, sawBroker bool + + for _, m := range listed() { + dims := map[string]string{} + for _, d := range m.Dimensions { + dims[aws.ToString(d.Name)] = aws.ToString(d.Value) + } + + if dims["Cluster Name"] != "sdk-metrics" { + continue + } + + switch aws.ToString(m.MetricName) { + case "ActiveControllerCount": + sawController = len(dims) == 1 + case "UnderReplicatedPartitions": + sawBroker = sawBroker || dims["Broker ID"] != "" + } + } + + if !sawController || !sawBroker { + t.Fatalf("AWS/Kafka metrics missing (controller=%v broker=%v)", sawController, sawBroker) + } + + stats, err := cw.GetMetricStatistics(ctx, &awscw.GetMetricStatisticsInput{ + Namespace: aws.String("AWS/Kafka"), + MetricName: aws.String("ActiveControllerCount"), + Dimensions: []cwtypes.Dimension{{Name: aws.String("Cluster Name"), Value: aws.String("sdk-metrics")}}, + StartTime: aws.Time(start), + EndTime: aws.Time(env.clock.Now().Add(time.Minute)), + Period: aws.Int32(60), + Statistics: []cwtypes.Statistic{cwtypes.StatisticSampleCount, cwtypes.StatisticMaximum}, + }) + if err != nil { + t.Fatalf("GetMetricStatistics: %v", err) + } + + if len(stats.Datapoints) != 2 { + t.Fatalf("ActiveControllerCount datapoints = %+v, want 2 one-minute samples", stats.Datapoints) + } + + for _, dp := range stats.Datapoints { + if aws.ToFloat64(dp.SampleCount) != 1 || aws.ToFloat64(dp.Maximum) != 1 || dp.Unit != cwtypes.StandardUnitCount { + t.Fatalf("datapoint = %+v, want one sample of 1 Count", dp) + } + } + + desc, err := c.DescribeCluster(ctx, &awskafka.DescribeClusterInput{ClusterArn: aws.String(arn)}) + if err != nil { + t.Fatalf("DescribeCluster: %v", err) + } + + key := aws.ToString(desc.ClusterInfo.EncryptionInfo.EncryptionAtRest.DataVolumeKMSKeyId) + if !strings.HasPrefix(key, "arn:aws:kms:us-east-1:") || strings.Contains(key, "msk-default") { + t.Fatalf("default key = %q", key) + } + + v2, err := c.DescribeClusterV2(ctx, &awskafka.DescribeClusterV2Input{ClusterArn: aws.String(arn)}) + if err != nil { + t.Fatalf("DescribeClusterV2: %v", err) + } + + enc := v2.ClusterInfo.Provisioned.EncryptionInfo + if enc == nil || aws.ToString(enc.EncryptionAtRest.DataVolumeKMSKeyId) != key { + t.Fatalf("v2 provisioned encryptionInfo = %+v, want key %s", enc, key) + } +} diff --git a/server/aws/kafka/types.go b/server/aws/kafka/types.go index ce35abdcc..651477db4 100644 --- a/server/aws/kafka/types.go +++ b/server/aws/kafka/types.go @@ -136,10 +136,8 @@ func clusterToWire(c *driver.Cluster) map[string]json.RawMessage { base["brokerNodeGroupInfo"] = bngToWire(c.BrokerNodeGroupInfo) } - if _, ok := c.RawOptions["encryptionInfo"]; !ok { - base["encryptionInfo"] = defaultEncryptionInfo() - } - + // encryptionInfo (including MSK's default at-rest key) is completed by the + // provider at create time and carried in RawOptions; mergeRaw surfaces it. addZookeeperConnect(base, c) return mergeRaw(base, c.RawOptions) @@ -191,22 +189,6 @@ func regionFromARN(arn string) string { return parts[regionIndex] } -// defaultEncryptionInfo returns the MSK default encryption configuration that -// real DescribeCluster surfaces when none is specified at create time: -// in-transit TLS (client-broker + in-cluster) and at-rest with an -// AWS-managed data-volume KMS key. -func defaultEncryptionInfo() map[string]any { - return map[string]any{ - "encryptionAtRest": map[string]any{ - "dataVolumeKMSKeyId": "arn:aws:kms:us-east-1:000000000000:key/msk-default", - }, - "encryptionInTransit": map[string]any{ - "clientBroker": "TLS", - "inCluster": true, - }, - } -} - // enhancedMonitoringOrDefault returns the cluster's EnhancedMonitoring level, // falling back to MSK's DEFAULT. Real DescribeCluster always surfaces this enum // (never omits it), so a cluster created without one still reports "DEFAULT". @@ -305,9 +287,9 @@ func provisionedBlock(c *driver.Cluster) map[string]any { block["brokerNodeGroupInfo"] = bngToWire(c.BrokerNodeGroupInfo) } - if _, ok := c.RawOptions["encryptionInfo"]; !ok { - block["encryptionInfo"] = defaultEncryptionInfo() - } + // The provider stores the cluster's (defaulted) encryptionInfo in RawOptions; + // v2 reports it inside the provisioned block. + overlayRaw(block, c.RawOptions, "encryptionInfo", "encryptionInfo") addZookeeperConnect(block, c) From d902f072c75353e68c825a5360c8d389dee8b506 Mon Sep 17 00:00:00 2001 From: Satyam Trivedi Date: Thu, 22 Oct 2026 13:57:22 +0530 Subject: [PATCH 2/2] fix(aws-kafka): address review on async lifecycle, metrics tick and encryption defaults - Snapshot deep-copies each cluster, configuration, VPC connection and replicator under its own lock (topics, operations, SCRAM secrets, tags, raw options), so encoding no longer races a concurrent CreateTopic. The per-store loops move into helpers, which also brings Snapshot back under the gocyclo limit. - withDefaultEncryption fills every EncryptionInfo field the request leaves out with its documented default (managed at-rest key, ClientBroker TLS, InCluster true) and keeps explicit values, including InCluster false. - Tick judges cluster state at its own now argument instead of the options clock, and removes clusters whose DELETING window has ended, reporting the change so persist saves it even when nothing reads the cluster. - Topic create/update/delete, SCRAM secret associate/disassociate and cluster policy put/delete are rejected with BadRequestException while a cluster is CREATING or DELETING (only reachable under --async-settle). - GetCompatibleKafkaVersions resolves the cluster through getCluster, so a cluster past its DELETING window is NotFound. Part of #1517 --- providers/aws/kafka/cluster_policy.go | 8 + providers/aws/kafka/encryption.go | 68 +++--- providers/aws/kafka/lifecycle.go | 43 ++-- providers/aws/kafka/metrics.go | 41 ++-- providers/aws/kafka/parity_test.go | 8 + providers/aws/kafka/review_test.go | 316 ++++++++++++++++++++++++++ providers/aws/kafka/scram_secrets.go | 8 + providers/aws/kafka/snapshot.go | 140 ++++++++---- providers/aws/kafka/topics.go | 12 + providers/aws/kafka/versions.go | 4 +- 10 files changed, 541 insertions(+), 107 deletions(-) create mode 100644 providers/aws/kafka/review_test.go diff --git a/providers/aws/kafka/cluster_policy.go b/providers/aws/kafka/cluster_policy.go index 4abf5b3bd..94c47ecf9 100644 --- a/providers/aws/kafka/cluster_policy.go +++ b/providers/aws/kafka/cluster_policy.go @@ -37,6 +37,10 @@ func (m *Mock) PutClusterPolicy(_ context.Context, clusterARN string, body json. cd.mu.Lock() defer cd.mu.Unlock() + if err := m.requireActive(cd); err != nil { + return "", err + } + if req.CurrentVersion != "" && req.CurrentVersion != cd.policyVersion { return "", badRequest( "currentVersion %q does not match policy version %q", req.CurrentVersion, cd.policyVersion) @@ -75,6 +79,10 @@ func (m *Mock) DeleteClusterPolicy(_ context.Context, clusterARN string) error { cd.mu.Lock() defer cd.mu.Unlock() + if err := m.requireActive(cd); err != nil { + return err + } + cd.policy = "" cd.policyVersion = "" diff --git a/providers/aws/kafka/encryption.go b/providers/aws/kafka/encryption.go index 7778bd282..81bfc927e 100644 --- a/providers/aws/kafka/encryption.go +++ b/providers/aws/kafka/encryption.go @@ -32,57 +32,61 @@ func (m *Mock) managedKafkaKeyARN() string { } // withDefaultEncryption returns raw with the cluster's EncryptionInfo completed -// the way real MSK completes it: with no encryptionInfo, at-rest encryption -// under the AWS managed key plus the documented in-transit defaults (ClientBroker -// TLS, InCluster true); with an encryptionInfo that omits encryptionAtRest (or -// its key), the managed key is filled in and the rest is kept verbatim. A block -// that already names a key is left untouched. raw is not mutated. +// the way real MSK completes it. Every field the request leaves out gets its +// documented default: encryptionAtRest.dataVolumeKMSKeyId the AWS managed key, +// encryptionInTransit.clientBroker TLS and encryptionInTransit.inCluster true +// (https://docs.aws.amazon.com/msk/1.0/apireference/clusters.html#clusters-model-encryptionintransit). +// Fields the request sets, including inCluster false, are kept, as is any other +// content of the block. A block that is not a JSON object is left as sent. raw +// is not mutated. func (m *Mock) withDefaultEncryption(raw map[string]json.RawMessage) map[string]json.RawMessage { out := copyRaw(raw) if out == nil { out = map[string]json.RawMessage{} } - keyARN := m.managedKafkaKeyARN() - - blk, ok := out[encryptionInfoKey] - if !ok || len(blk) == 0 { - out[encryptionInfoKey] = mustJSON(map[string]any{ - "encryptionAtRest": map[string]any{"dataVolumeKMSKeyId": keyARN}, - "encryptionInTransit": map[string]any{"clientBroker": "TLS", "inCluster": true}, - }) - - return out + var info map[string]json.RawMessage + if blk := out[encryptionInfoKey]; len(blk) > 0 { + if err := json.Unmarshal(blk, &info); err != nil { + return out + } } - var info map[string]json.RawMessage - if err := json.Unmarshal(blk, &info); err != nil || info == nil { - return out + if info == nil { + info = map[string]json.RawMessage{} } - var atRest map[string]json.RawMessage - if r, has := info["encryptionAtRest"]; has { - _ = json.Unmarshal(r, &atRest) + changed := defaultField(info, "encryptionAtRest", "dataVolumeKMSKeyId", m.managedKafkaKeyARN()) + changed = defaultField(info, "encryptionInTransit", "clientBroker", "TLS") || changed + changed = defaultField(info, "encryptionInTransit", "inCluster", true) || changed + + if changed { + out[encryptionInfoKey] = mustJSON(info) } - if atRest == nil { - atRest = map[string]json.RawMessage{} + return out +} + +// defaultField sets info[block][field] to val when the field is absent, null or +// an empty string, creating the block if needed, and reports whether it did. +func defaultField(info map[string]json.RawMessage, block, field string, val any) bool { + var blk map[string]json.RawMessage + if r, ok := info[block]; ok { + _ = json.Unmarshal(r, &blk) } - var existing string - if k, has := atRest["dataVolumeKMSKeyId"]; has { - _ = json.Unmarshal(k, &existing) + if blk == nil { + blk = map[string]json.RawMessage{} } - if existing != "" { - return out + if v, ok := blk[field]; ok && string(v) != "null" && string(v) != `""` { + return false } - atRest["dataVolumeKMSKeyId"] = mustJSON(keyARN) - info["encryptionAtRest"] = mustJSON(atRest) - out[encryptionInfoKey] = mustJSON(info) + blk[field] = mustJSON(val) + info[block] = mustJSON(blk) - return out + return true } // mustJSON marshals a value built from JSON-safe types (maps, strings, bools, diff --git a/providers/aws/kafka/lifecycle.go b/providers/aws/kafka/lifecycle.go index 383357579..dea6151b1 100644 --- a/providers/aws/kafka/lifecycle.go +++ b/providers/aws/kafka/lifecycle.go @@ -33,11 +33,17 @@ func (m *Mock) settleUntil(d time.Duration) time.Time { return m.now().Add(dur) } -// clusterStateLocked returns the cluster's externally visible state: CREATING +// clusterStateLocked returns the cluster's externally visible state now. The +// caller holds cd.mu (read or write). +func (m *Mock) clusterStateLocked(cd *clusterData) string { + return clusterStateAt(cd, m.now()) +} + +// clusterStateAt returns the cluster's externally visible state at t: CREATING // while its create window is open, otherwise the stored state. The caller holds // cd.mu (read or write). -func (m *Mock) clusterStateLocked(cd *clusterData) string { - if cd.cluster.State == driver.ClusterStateActive && m.now().Before(cd.readyAt) { +func clusterStateAt(cd *clusterData, t time.Time) string { + if cd.cluster.State == driver.ClusterStateActive && t.Before(cd.readyAt) { return driver.ClusterStateCreating } @@ -53,35 +59,42 @@ func (m *Mock) observedClusterLocked(cd *clusterData) driver.Cluster { return out } -// requireActive rejects an update against a cluster that is not ACTIVE. Real -// MSK accepts no update operations while a cluster is CREATING or DELETING +// requireActive rejects a write against a cluster that is not ACTIVE: cluster +// updates, and topic, SCRAM secret and cluster policy writes. Real MSK accepts +// no API operations on a cluster that is CREATING or DELETING // (https://docs.aws.amazon.com/msk/latest/developerguide/msk-cluster-states.html). // Without AsyncSettle a cluster is always ACTIVE, so this never fires. The // caller holds cd.mu. func (m *Mock) requireActive(cd *clusterData) error { if st := m.clusterStateLocked(cd); st != driver.ClusterStateActive { - return badRequest("cluster %s is in %s state; update operations require ACTIVE", + return badRequest("cluster %s is in %s state; this operation requires ACTIVE", cd.cluster.ClusterARN, st) } return nil } -// gone reports whether a stored cluster is DELETING and its delete window has -// elapsed, i.e. real MSK would no longer return it. A DELETING cluster with no -// window (restored from a snapshot that predates windows) counts as elapsed. -func (m *Mock) gone(cd *clusterData) bool { +// goneAt reports whether a stored cluster is DELETING and its delete window has +// elapsed at t, i.e. real MSK would no longer return it. A DELETING cluster with +// no window (restored from a snapshot that predates windows) counts as elapsed. +func goneAt(cd *clusterData, t time.Time) bool { cd.mu.RLock() defer cd.mu.RUnlock() - return cd.cluster.State == driver.ClusterStateDeleting && !m.now().Before(cd.deleteAt) + return cd.cluster.State == driver.ClusterStateDeleting && !t.Before(cd.deleteAt) } -// reapIfGone removes a cluster whose DELETING window has elapsed and frees its -// name, reporting whether it was removed. Removal is lazy (on the next read), -// mirroring how ELBv2 retires a drained target. +// reapIfGone removes a cluster whose DELETING window has elapsed now and frees +// its name, reporting whether it was removed. Request paths call it lazily on +// read; Tick calls reapIfGoneAt so a finished delete is removed (and persisted) +// even when nothing reads the cluster. func (m *Mock) reapIfGone(arn string, cd *clusterData) bool { - if !m.gone(cd) { + return m.reapIfGoneAt(arn, cd, m.now()) +} + +// reapIfGoneAt is reapIfGone judged at t. +func (m *Mock) reapIfGoneAt(arn string, cd *clusterData, t time.Time) bool { + if !goneAt(cd, t) { return false } diff --git a/providers/aws/kafka/metrics.go b/providers/aws/kafka/metrics.go index 5bb746b40..a95622aa5 100644 --- a/providers/aws/kafka/metrics.go +++ b/providers/aws/kafka/metrics.go @@ -55,19 +55,20 @@ func (m *Mock) SetMonitoring(mon mondriver.Monitoring) { m.monitoring = mon } -// Tick publishes the AWS/Kafka metrics that are due at now for every ACTIVE -// provisioned cluster, and reports whether it published anything. It has the -// config.Tickable signature of the shared serve scheduler; library mode never -// calls it, so tests drive it with a FakeClock. Each cluster is sampled under -// its own lock and everything is published after all locks are released. +// Tick is the MSK work of the shared serve scheduler (config.Tickable). At now +// it removes every cluster whose DELETING window has ended, so a finished +// delete is dropped (and persisted) even when nothing reads the cluster, and +// publishes the AWS/Kafka metrics that are due for every ACTIVE provisioned +// cluster. It reports whether it removed or published anything. Every state +// check uses now, not the options clock. Library mode never calls it, so tests +// drive it with a FakeClock. Each cluster is sampled under its own lock and +// everything is published after all locks are released. func (m *Mock) Tick(now time.Time) bool { - if m.monitoring == nil { - return false - } - arns := m.clusters.Keys() sort.Strings(arns) + reaped := false + var data []mondriver.MetricDatum for _, arn := range arns { @@ -76,27 +77,33 @@ func (m *Mock) Tick(now time.Time) bool { continue } - data = append(data, m.dueMetrics(cd, now)...) - } + if m.reapIfGoneAt(arn, cd, now) { + reaped = true - if len(data) == 0 { - return false + continue + } + + if m.monitoring != nil { + data = append(data, m.dueMetrics(cd, now)...) + } } - _ = m.monitoring.PutMetricData(context.Background(), data) + if len(data) > 0 { + _ = m.monitoring.PutMetricData(context.Background(), data) + } - return true + return reaped || len(data) > 0 } // dueMetrics samples cd's metrics if they are due at now and records the sample // time. A cluster that is not ACTIVE, or is serverless, reports nothing. // Callers must not hold cd.mu. -func (m *Mock) dueMetrics(cd *clusterData, now time.Time) []mondriver.MetricDatum { +func (*Mock) dueMetrics(cd *clusterData, now time.Time) []mondriver.MetricDatum { cd.mu.Lock() defer cd.mu.Unlock() if cd.cluster.ClusterType != driver.ClusterTypeProvisioned || - m.clusterStateLocked(cd) != driver.ClusterStateActive { + clusterStateAt(cd, now) != driver.ClusterStateActive { return nil } diff --git a/providers/aws/kafka/parity_test.go b/providers/aws/kafka/parity_test.go index 9032236a5..9b4e5217d 100644 --- a/providers/aws/kafka/parity_test.go +++ b/providers/aws/kafka/parity_test.go @@ -378,6 +378,14 @@ func TestTickSkipsDeletingAndServerless(t *testing.T) { fc.Advance(settleWindow) + // This tick removes the cluster whose DELETING window has ended (a change); + // the next one, with only the serverless cluster left, does nothing. + if !m.Tick(fc.Now()) { + t.Fatal("Tick did not report removing the deleted cluster") + } + + fc.Advance(time.Minute) + if m.Tick(fc.Now()) { t.Fatal("Tick published for a serverless cluster") } diff --git a/providers/aws/kafka/review_test.go b/providers/aws/kafka/review_test.go new file mode 100644 index 000000000..bce0ce4ab --- /dev/null +++ b/providers/aws/kafka/review_test.go @@ -0,0 +1,316 @@ +package kafka_test + +import ( + "context" + "encoding/json" + "strconv" + "sync" + "testing" + + "github.com/stackshy/cloudemu/v2/config" + "github.com/stackshy/cloudemu/v2/providers/aws/cloudwatch" + "github.com/stackshy/cloudemu/v2/providers/aws/kafka" + "github.com/stackshy/cloudemu/v2/services/kafka/driver" +) + +// TestSnapshotConcurrentWithMutations runs Snapshot while topics, SCRAM +// secrets, tags and broker counts change on a cluster, VPC connection, +// configuration and replicator. Snapshot must deep-copy each record under its +// lock, so the JSON encoding never reads a map or slice a writer is changing +// (run with -race). The final snapshot then restores every record. +func TestSnapshotConcurrentWithMutations(t *testing.T) { + m := newMock(t) + ctx := context.Background() + c := createThreeAZCluster(t, m, "busy") + + vpc, err := m.CreateVpcConnection(ctx, json.RawMessage( + `{"targetClusterArn":"`+c.ClusterARN+`","authentication":"SASL_IAM","vpcId":"vpc-1"}`)) + if err != nil { + t.Fatalf("CreateVpcConnection: %v", err) + } + + cfg, err := m.CreateConfiguration(ctx, driver.CreateConfigurationInput{ + Name: "busy-cfg", KafkaVersions: []string{"3.6.0"}, ServerProperties: []byte("a=b"), + }) + if err != nil { + t.Fatalf("CreateConfiguration: %v", err) + } + + repl, err := m.CreateReplicator(ctx, json.RawMessage(`{"replicatorName":"busy-rep","kafkaClusters":[],`+ + `"replicationInfoList":[],"serviceExecutionRoleArn":"arn:aws:iam::123456789012:role/r"}`)) + if err != nil { + t.Fatalf("CreateReplicator: %v", err) + } + + const n = 50 + + var wg sync.WaitGroup + + wg.Add(2) + + go func() { + defer wg.Done() + + for i := range n { + name := "t" + strconv.Itoa(i) + _, _ = m.CreateTopic(ctx, c.ClusterARN, json.RawMessage(`{"topicName":"`+name+`","partitionCount":1}`)) + _, _ = m.UpdateTopic(ctx, c.ClusterARN, name, json.RawMessage(`{"partitionCount":2}`)) + _, _ = m.BatchAssociateScramSecret(ctx, c.ClusterARN, + []string{"arn:aws:secretsmanager:us-east-1:123456789012:secret:AmazonMSK_" + name}) + _ = m.TagResource(ctx, c.ClusterARN, map[string]string{name: "v"}) + _ = m.TagResource(ctx, vpc.VpcConnectionARN, map[string]string{name: "v"}) + _ = m.TagResource(ctx, cfg.ARN, map[string]string{name: "v"}) + _ = m.TagResource(ctx, repl.ReplicatorARN, map[string]string{name: "v"}) + _, _ = m.UpdateBrokerCount(ctx, c.ClusterARN, "", int32(3+i%3)) + } + }() + + go func() { + defer wg.Done() + + for range n { + if _, err := m.Snapshot(ctx, false); err != nil { + t.Errorf("Snapshot: %v", err) + + return + } + } + }() + + wg.Wait() + + raw, err := m.Snapshot(ctx, false) + if err != nil { + t.Fatalf("Snapshot: %v", err) + } + + dst := newMock(t) + if err := dst.Restore(ctx, raw); err != nil { + t.Fatalf("Restore: %v", err) + } + + if topics, _, err := dst.ListTopics(ctx, c.ClusterARN, driver.Page{MaxResults: 100}); err != nil || len(topics) != n { + t.Fatalf("restored topics = %d/%v, want %d", len(topics), err, n) + } + + for _, arn := range []string{vpc.VpcConnectionARN, cfg.ARN, repl.ReplicatorARN} { + if tags, err := dst.ListTagsForResource(ctx, arn); err != nil || len(tags) < n { + t.Fatalf("restored tags on %s = %d/%v", arn, len(tags), err) + } + } +} + +// TestDefaultEncryptionInTransitCompleted asserts a partial encryptionInfo gets +// MSK's documented in-transit defaults (ClientBroker TLS, InCluster true) for +// any field it leaves out, while explicit values, including InCluster false, +// are kept. +func TestDefaultEncryptionInTransitCompleted(t *testing.T) { + m := newMock(t) + ctx := context.Background() + + cases := []struct { + name, block, client string + inCluster bool + }{ + {"at-rest-only", `{"encryptionAtRest":{"dataVolumeKMSKeyId":"arn:aws:kms:us-east-1:123456789012:key/mine"}}`, "TLS", true}, + {"client-only", `{"encryptionInTransit":{"clientBroker":"TLS_PLAINTEXT"}}`, "TLS_PLAINTEXT", true}, + {"in-cluster-only", `{"encryptionInTransit":{"inCluster":false}}`, "TLS", false}, + {"empty-block", `{}`, "TLS", true}, + } + + for _, tc := range cases { + c, err := m.CreateCluster(ctx, driver.CreateClusterInput{ + ClusterName: "enc-" + tc.name, NumberOfBrokerNodes: 3, BrokerNodeGroupInfo: bng(), + RawOptions: map[string]json.RawMessage{"encryptionInfo": json.RawMessage(tc.block)}, + }) + if err != nil { + t.Fatalf("%s: CreateCluster: %v", tc.name, err) + } + + info := encryptionInfo(t, c) + if info["encryptionInTransit"]["clientBroker"] != tc.client || + info["encryptionInTransit"]["inCluster"] != tc.inCluster { + t.Fatalf("%s: encryptionInTransit = %+v, want %s/%v", + tc.name, info["encryptionInTransit"], tc.client, tc.inCluster) + } + + if key, _ := info["encryptionAtRest"]["dataVolumeKMSKeyId"].(string); key == "" { + t.Fatalf("%s: no at-rest key: %+v", tc.name, info) + } + } +} + +// TestTickUsesItsNow asserts Tick judges a cluster's state by the time it is +// given, not the options clock: a Tick at a time past the CREATING window +// publishes even though the options clock has not moved. +func TestTickUsesItsNow(t *testing.T) { + m, fc := newSettleMock(t) + m.SetMonitoring(cloudwatch.New(config.NewOptions(config.WithClock(fc)))) + + createThreeAZCluster(t, m, "ahead") + + if !m.Tick(fc.Now().Add(settleWindow)) { + t.Fatal("Tick at a time past the CREATING window published nothing") + } + + if m.Tick(fc.Now()) { + t.Fatal("Tick before the CREATING window ends published metrics") + } +} + +// TestTickReapsFinishedDelete asserts Tick removes a cluster whose DELETING +// window has ended and reports the change (so serve persists it), even with no +// monitoring backend wired and no read in between. +func TestTickReapsFinishedDelete(t *testing.T) { + m, fc := newSettleMock(t) + ctx := context.Background() + + c := createThreeAZCluster(t, m, "reaped") + fc.Advance(settleWindow) + + if m.Tick(fc.Now()) { + t.Fatal("Tick reported a change with nothing to do") + } + + if _, _, err := m.DeleteCluster(ctx, c.ClusterARN, ""); err != nil { + t.Fatalf("DeleteCluster: %v", err) + } + + if m.Tick(fc.Now()) { + t.Fatal("Tick reaped a cluster inside its DELETING window") + } + + fc.Advance(settleWindow) + + if !m.Tick(fc.Now()) { + t.Fatal("Tick did not report reaping a finished delete") + } + + raw, err := m.Snapshot(ctx, false) + if err != nil { + t.Fatalf("Snapshot: %v", err) + } + + var snap struct { + Clusters map[string]json.RawMessage `json:"clusters"` + ClusterNames map[string]string `json:"clusterNames"` + } + if err := json.Unmarshal(raw, &snap); err != nil { + t.Fatalf("decode snapshot: %v", err) + } + + if len(snap.Clusters) != 0 || len(snap.ClusterNames) != 0 { + t.Fatalf("snapshot after reap still holds the cluster: %s", raw) + } + + if m.Tick(fc.Now()) { + t.Fatal("Tick reported a change after the cluster was gone") + } +} + +// TestClusterWritesRequireActive asserts topic, SCRAM secret and cluster +// policy writes are rejected while a cluster is CREATING or DELETING (real MSK +// accepts no API operations on it in those states) and accepted once ACTIVE. +func TestClusterWritesRequireActive(t *testing.T) { + m, fc := newSettleMock(t) + ctx := context.Background() + + c := createThreeAZCluster(t, m, "guarded") + secret := []string{"arn:aws:secretsmanager:us-east-1:123456789012:secret:AmazonMSK_s"} + + writes := map[string]func() error{ + "CreateTopic": func() error { + _, err := m.CreateTopic(ctx, c.ClusterARN, json.RawMessage(`{"topicName":"t","partitionCount":1}`)) + + return err + }, + "UpdateTopic": func() error { + _, err := m.UpdateTopic(ctx, c.ClusterARN, "t", json.RawMessage(`{"partitionCount":2}`)) + + return err + }, + "BatchAssociateScramSecret": func() error { + _, err := m.BatchAssociateScramSecret(ctx, c.ClusterARN, secret) + + return err + }, + "BatchDisassociateScramSecret": func() error { + _, err := m.BatchDisassociateScramSecret(ctx, c.ClusterARN, secret) + + return err + }, + "PutClusterPolicy": func() error { + _, err := m.PutClusterPolicy(ctx, c.ClusterARN, json.RawMessage(`{"policy":"{}"}`)) + + return err + }, + "DeleteClusterPolicy": func() error { return m.DeleteClusterPolicy(ctx, c.ClusterARN) }, + "DeleteTopic": func() error { return m.DeleteTopic(ctx, c.ClusterARN, "t") }, + } + order := []string{ + "CreateTopic", "UpdateTopic", "BatchAssociateScramSecret", "BatchDisassociateScramSecret", + "PutClusterPolicy", "DeleteClusterPolicy", "DeleteTopic", + } + + for _, op := range order { + wantException(t, writes[op](), driver.ExBadRequest) + } + + fc.Advance(settleWindow) + + for _, op := range order { + if err := writes[op](); err != nil { + t.Fatalf("%s on ACTIVE cluster: %v", op, err) + } + } + + if _, _, err := m.DeleteCluster(ctx, c.ClusterARN, ""); err != nil { + t.Fatalf("DeleteCluster: %v", err) + } + + for _, op := range order { + wantException(t, writes[op](), driver.ExBadRequest) + } +} + +// TestCompatibleVersionsAfterDeleteWindow asserts GetCompatibleKafkaVersions +// stops answering for a cluster once its DELETING window has ended. +func TestCompatibleVersionsAfterDeleteWindow(t *testing.T) { + m, fc := newSettleMock(t) + ctx := context.Background() + + c := createThreeAZCluster(t, m, "versioned") + fc.Advance(settleWindow) + + if _, _, err := m.DeleteCluster(ctx, c.ClusterARN, ""); err != nil { + t.Fatalf("DeleteCluster: %v", err) + } + + if _, err := m.GetCompatibleKafkaVersions(ctx, c.ClusterARN); err != nil { + t.Fatalf("GetCompatibleKafkaVersions while DELETING: %v", err) + } + + fc.Advance(settleWindow) + + _, err := m.GetCompatibleKafkaVersions(ctx, c.ClusterARN) + wantException(t, err, driver.ExNotFound) +} + +// TestCreateReclaimsDanglingName asserts a create succeeds when the name index +// still points at a cluster ARN that is no longer stored (a snapshot taken +// between the two removals). +func TestCreateReclaimsDanglingName(t *testing.T) { + m := kafka.New(config.NewOptions()) + ctx := context.Background() + + snap := `{"clusterNames":{"orphan":"arn:aws:kafka:us-east-1:123456789012:cluster/orphan/gone-1"}}` + if err := m.Restore(ctx, json.RawMessage(snap)); err != nil { + t.Fatalf("Restore: %v", err) + } + + if _, err := m.CreateCluster(ctx, driver.CreateClusterInput{ + ClusterName: "orphan", NumberOfBrokerNodes: 3, BrokerNodeGroupInfo: bng(), + }); err != nil { + t.Fatalf("CreateCluster over a dangling name: %v", err) + } +} diff --git a/providers/aws/kafka/scram_secrets.go b/providers/aws/kafka/scram_secrets.go index 292e23c25..f808d90c2 100644 --- a/providers/aws/kafka/scram_secrets.go +++ b/providers/aws/kafka/scram_secrets.go @@ -24,6 +24,10 @@ func (m *Mock) BatchAssociateScramSecret( cd.mu.Lock() defer cd.mu.Unlock() + if err := m.requireActive(cd); err != nil { + return nil, err + } + var unprocessed []json.RawMessage for _, arn := range secretARNs { @@ -57,6 +61,10 @@ func (m *Mock) BatchDisassociateScramSecret( cd.mu.Lock() defer cd.mu.Unlock() + if err := m.requireActive(cd); err != nil { + return nil, err + } + var unprocessed []json.RawMessage for _, arn := range secretARNs { diff --git a/providers/aws/kafka/snapshot.go b/providers/aws/kafka/snapshot.go index f20ae0380..bfc4583ba 100644 --- a/providers/aws/kafka/snapshot.go +++ b/providers/aws/kafka/snapshot.go @@ -57,23 +57,10 @@ type replicatorSnapshot struct { } // Snapshot captures the mock's entire state as JSON. includeAssets is unused. MSK holds resource -// metadata, not bulk object bodies. +// metadata, not bulk object bodies. Each record is deep-copied under its own +// lock, so the encoding never reads a map or slice a concurrent writer changes. func (m *Mock) Snapshot(_ context.Context, _ bool) (json.RawMessage, error) { - snap := kafkaSnapshot{} - - if m.clusters.Len() > 0 { - snap.Clusters = make(map[string]*clusterSnapshot, m.clusters.Len()) - - for arn, cd := range m.clusters.All() { - cd.mu.RLock() - snap.Clusters[arn] = &clusterSnapshot{ - Cluster: cd.cluster, Operations: cd.operations, Topics: cd.topics, - ScramSecrets: cd.scramSecrets, Policy: cd.policy, PolicyVersion: cd.policyVersion, - ReadyAt: cd.readyAt, DeleteAt: cd.deleteAt, MetricsSampleAt: cd.metricsSampleAt, - } - cd.mu.RUnlock() - } - } + snap := kafkaSnapshot{Clusters: m.clustersForSnapshot()} names, err := m.clusterNames.Snapshot() if err != nil { @@ -81,46 +68,117 @@ func (m *Mock) Snapshot(_ context.Context, _ bool) (json.RawMessage, error) { } snap.ClusterNames = names + snap.Configs = m.configsForSnapshot() + snap.VPCConns, snap.VPCConnReadyAt = m.vpcConnsForSnapshot() + snap.Replicators = m.replicatorsForSnapshot() + + return json.Marshal(snap) +} + +// clustersForSnapshot deep-copies every cluster record, or returns nil when +// there are none. +func (m *Mock) clustersForSnapshot() map[string]*clusterSnapshot { + if m.clusters.Len() == 0 { + return nil + } + + out := make(map[string]*clusterSnapshot, m.clusters.Len()) - if m.configs.Len() > 0 { - snap.Configs = make(map[string]driver.Configuration, m.configs.Len()) + for arn, cd := range m.clusters.All() { + cd.mu.RLock() + out[arn] = cd.snapshotLocked() + cd.mu.RUnlock() + } + + return out +} + +// snapshotLocked deep-copies a cluster record. The caller holds cd.mu. +func (cd *clusterData) snapshotLocked() *clusterSnapshot { + cs := &clusterSnapshot{ + Cluster: snapshotCluster(cd.cluster), + ScramSecrets: copyStrings(cd.scramSecrets), + Policy: cd.policy, PolicyVersion: cd.policyVersion, + ReadyAt: cd.readyAt, DeleteAt: cd.deleteAt, MetricsSampleAt: cd.metricsSampleAt, + } + + if cd.operations != nil { + cs.Operations = make([]driver.ClusterOperation, len(cd.operations)) + for i := range cd.operations { + cs.Operations[i] = copyOperation(cd.operations[i]) + } + } - for arn, cd := range m.configs.All() { - cd.mu.RLock() - snap.Configs[arn] = cd.config - cd.mu.RUnlock() + if cd.topics != nil { + cs.Topics = make(map[string]driver.Topic, len(cd.topics)) + for name, t := range cd.topics { + cs.Topics[name] = snapshotTopic(t) } } - if m.vpcConns.Len() > 0 { - snap.VPCConns = make(map[string]driver.VpcConnection, m.vpcConns.Len()) + return cs +} + +// configsForSnapshot deep-copies every configuration, or returns nil when there +// are none. +func (m *Mock) configsForSnapshot() map[string]driver.Configuration { + if m.configs.Len() == 0 { + return nil + } + + out := make(map[string]driver.Configuration, m.configs.Len()) + + for arn, cd := range m.configs.All() { + cd.mu.RLock() + out[arn] = snapshotConfig(cd.config) + cd.mu.RUnlock() + } + + return out +} + +// vpcConnsForSnapshot deep-copies every VPC connection, plus the CREATING +// deadline of each one still inside its window (nil when none is). +func (m *Mock) vpcConnsForSnapshot() (conns map[string]driver.VpcConnection, readyAt map[string]time.Time) { + if m.vpcConns.Len() == 0 { + return nil, nil + } - for arn, vd := range m.vpcConns.All() { - vd.mu.RLock() - snap.VPCConns[arn] = vd.vpc + conns = make(map[string]driver.VpcConnection, m.vpcConns.Len()) - if !vd.readyAt.IsZero() { - if snap.VPCConnReadyAt == nil { - snap.VPCConnReadyAt = map[string]time.Time{} - } + for arn, vd := range m.vpcConns.All() { + vd.mu.RLock() + conns[arn] = snapshotVpcConnection(vd.vpc) - snap.VPCConnReadyAt[arn] = vd.readyAt + if !vd.readyAt.IsZero() { + if readyAt == nil { + readyAt = map[string]time.Time{} } - vd.mu.RUnlock() + + readyAt[arn] = vd.readyAt } + vd.mu.RUnlock() } - if m.replicators.Len() > 0 { - snap.Replicators = make(map[string]*replicatorSnapshot, m.replicators.Len()) + return conns, readyAt +} - for arn, rd := range m.replicators.All() { - rd.mu.RLock() - snap.Replicators[arn] = &replicatorSnapshot{Replicator: rd.replicator, Version: rd.version} - rd.mu.RUnlock() - } +// replicatorsForSnapshot deep-copies every replicator, or returns nil when +// there are none. +func (m *Mock) replicatorsForSnapshot() map[string]*replicatorSnapshot { + if m.replicators.Len() == 0 { + return nil } - return json.Marshal(snap) + out := make(map[string]*replicatorSnapshot, m.replicators.Len()) + + for arn, rd := range m.replicators.All() { + rd.mu.RLock() + out[arn] = &replicatorSnapshot{Replicator: snapshotReplicator(rd.replicator), Version: rd.version} + rd.mu.RUnlock() + } + + return out } // Restore rebuilds the mock's state under the original identities: every cluster, diff --git a/providers/aws/kafka/topics.go b/providers/aws/kafka/topics.go index c6f767089..f4208523d 100644 --- a/providers/aws/kafka/topics.go +++ b/providers/aws/kafka/topics.go @@ -62,6 +62,10 @@ func (m *Mock) CreateTopic(_ context.Context, clusterARN string, body json.RawMe cd.mu.Lock() defer cd.mu.Unlock() + if err := m.requireActive(cd); err != nil { + return nil, err + } + if cd.topics == nil { cd.topics = map[string]driver.Topic{} } @@ -168,6 +172,10 @@ func (m *Mock) UpdateTopic( cd.mu.Lock() defer cd.mu.Unlock() + if err := m.requireActive(cd); err != nil { + return nil, err + } + t, ok := cd.getTopic(topicName) if !ok { return nil, notFound("topic not found: %s", topicName) @@ -198,6 +206,10 @@ func (m *Mock) DeleteTopic(_ context.Context, clusterARN, topicName string) erro cd.mu.Lock() defer cd.mu.Unlock() + if err := m.requireActive(cd); err != nil { + return err + } + if _, ok := cd.getTopic(topicName); !ok { return notFound("topic not found: %s", topicName) } diff --git a/providers/aws/kafka/versions.go b/providers/aws/kafka/versions.go index 53dcb64f0..5a63196e5 100644 --- a/providers/aws/kafka/versions.go +++ b/providers/aws/kafka/versions.go @@ -92,8 +92,8 @@ func (m *Mock) compatibilitySources(clusterARN string) []string { return all } - cd, ok := m.clusters.Get(clusterARN) - if !ok { + cd, err := m.getCluster(clusterARN) + if err != nil { return nil }