diff --git a/providers/aws/aws.go b/providers/aws/aws.go index 5aafa2483..06427cd47 100644 --- a/providers/aws/aws.go +++ b/providers/aws/aws.go @@ -500,6 +500,9 @@ func newProvider(o *config.Options, shared *GlobalServices) *Provider { // matching real EKS (which auto-creates the cluster SG and infers the VPC). p.EKS.SetSubnetResolver(p.VPC) p.SageMaker.SetMonitoring(p.CloudWatch) + // MSK publishes DEFAULT-level AWS/Kafka metrics for each ACTIVE provisioned + // cluster once a minute, from the serve scheduler (see Tickables). + p.Kafka.SetMonitoring(p.CloudWatch) // CloudWatch alarm -> SNS: an alarm state transition fires its configured // SNS-topic actions, fanning a notification out to the topic's subscribers. p.CloudWatch.SetSNSPublisher(p.SNS) @@ -638,9 +641,10 @@ func awsDrivers(p *Provider) *resourcediscovery.Drivers { // background ticker calls each one on its interval. New time-driven services // register here. func (p *Provider) Tickables() []config.Tickable { - // SQS and EFS run before CloudWatch so the alarms evaluated in the same - // tick see the queue gauges and AWS/EFS metrics they just published. - return []config.Tickable{p.SQS, p.EFS, p.CloudWatch, p.SSM, p.CloudFormation, p.Batch} + // SQS, EFS and Kafka run before CloudWatch so the alarms evaluated in the + // same tick see the queue gauges and AWS/EFS and AWS/Kafka metrics they just + // published. + return []config.Tickable{p.SQS, p.EFS, p.Kafka, p.CloudWatch, p.SSM, p.CloudFormation, p.Batch} } // Close tears down any real engines wired into the provider via diff --git a/providers/aws/kafka/cluster_policy.go b/providers/aws/kafka/cluster_policy.go index 4abf5b3bd..94c47ecf9 100644 --- a/providers/aws/kafka/cluster_policy.go +++ b/providers/aws/kafka/cluster_policy.go @@ -37,6 +37,10 @@ func (m *Mock) PutClusterPolicy(_ context.Context, clusterARN string, body json. cd.mu.Lock() defer cd.mu.Unlock() + if err := m.requireActive(cd); err != nil { + return "", err + } + if req.CurrentVersion != "" && req.CurrentVersion != cd.policyVersion { return "", badRequest( "currentVersion %q does not match policy version %q", req.CurrentVersion, cd.policyVersion) @@ -75,6 +79,10 @@ func (m *Mock) DeleteClusterPolicy(_ context.Context, clusterARN string) error { cd.mu.Lock() defer cd.mu.Unlock() + if err := m.requireActive(cd); err != nil { + return err + } + cd.policy = "" cd.policyVersion = "" diff --git a/providers/aws/kafka/clusters.go b/providers/aws/kafka/clusters.go index 31ca7b830..e580c43fd 100644 --- a/providers/aws/kafka/clusters.go +++ b/providers/aws/kafka/clusters.go @@ -5,8 +5,8 @@ import ( "encoding/json" "sort" "strings" + "time" - "github.com/stackshy/cloudemu/v2/internal/idgen" "github.com/stackshy/cloudemu/v2/services/kafka/driver" ) @@ -150,16 +150,17 @@ func (m *Mock) getClusterBR(arn string) (*clusterData, error) { // with mkErr so each op returns the exception the SDK actually models for it. func (m *Mock) getClusterErr(arn string, mkErr func(string, ...any) error) (*clusterData, error) { cd, ok := m.clusters.Get(arn) - if !ok { + if !ok || m.reapIfGone(arn, cd) { return nil, mkErr("cluster not found: %s", arn) } return cd, nil } -// CreateCluster provisions a provisioned (v1) cluster that is immediately -// Active. The cluster name is claimed atomically under createMu so a duplicate -// name is rejected with ConflictException. +// CreateCluster provisions a provisioned (v1) cluster. It reports ACTIVE, or +// CREATING for a settle window under AsyncSettle. The cluster name is claimed +// atomically under createMu so a duplicate name is rejected with +// ConflictException. // //nolint:gocritic // hugeParam: signature fixed by driver.Kafka (by-value input). func (m *Mock) CreateCluster(_ context.Context, in driver.CreateClusterInput) (*driver.Cluster, error) { @@ -192,37 +193,17 @@ func (m *Mock) CreateCluster(_ context.Context, in driver.CreateClusterInput) (* kafkaVer = defaultKafkaVer } - m.createMu.Lock() - defer m.createMu.Unlock() - - if _, dup := m.clusterNames.Get(in.ClusterName); dup { - return nil, conflict("cluster already exists: %s", in.ClusterName) - } - - arn := m.clusterARN(in.ClusterName) - - cluster := driver.Cluster{ - ClusterARN: arn, + return m.insertCluster(driver.Cluster{ ClusterName: in.ClusterName, ClusterType: driver.ClusterTypeProvisioned, - State: driver.ClusterStateActive, - CurrentVersion: "K" + idgen.GenerateID(""), KafkaVersion: kafkaVer, NumberOfBrokerNodes: in.NumberOfBrokerNodes, BrokerNodeGroupInfo: copyBrokerNodeGroupInfo(in.BrokerNodeGroupInfo), StorageMode: in.StorageMode, EnhancedMonitoring: in.EnhancedMonitoring, Tags: copyTags(in.Tags), - CreationTime: m.now(), RawOptions: copyRaw(in.RawOptions), - } - - m.clusters.Set(arn, &clusterData{cluster: cluster}) - m.clusterNames.Set(in.ClusterName, arn) - - out := snapshotCluster(cluster) - - return &out, nil + }) } // DescribeCluster returns a deep copy of the stored cluster. @@ -233,34 +214,55 @@ func (m *Mock) DescribeCluster(_ context.Context, arn string) (*driver.Cluster, } cd.mu.RLock() - defer cd.mu.RUnlock() - - out := snapshotCluster(cd.cluster) + out := m.observedClusterLocked(cd) + cd.mu.RUnlock() return &out, nil } -// ListClusters lists clusters (optionally filtered by name prefix), sorted by -// name for deterministic paging. -func (m *Mock) ListClusters( - _ context.Context, namePrefix string, page driver.Page, -) (clusters []driver.Cluster, next string, err error) { +// visibleClusters returns observed deep copies of every cluster real MSK would +// still list, lazily removing any whose DELETING window has elapsed. +func (m *Mock) visibleClusters() []driver.Cluster { vals := m.clusters.SortedValues() all := make([]driver.Cluster, 0, len(vals)) for _, cd := range vals { cd.mu.RLock() - snap := snapshotCluster(cd.cluster) + arn := cd.cluster.ClusterARN cd.mu.RUnlock() - if namePrefix != "" && !strings.HasPrefix(snap.ClusterName, namePrefix) { + if m.reapIfGone(arn, cd) { continue } + cd.mu.RLock() + snap := m.observedClusterLocked(cd) + cd.mu.RUnlock() + all = append(all, snap) } + return all +} + +// ListClusters lists clusters (optionally filtered by name prefix), sorted by +// name for deterministic paging. +func (m *Mock) ListClusters( + _ context.Context, namePrefix string, page driver.Page, +) (clusters []driver.Cluster, next string, err error) { + visible := m.visibleClusters() + + all := make([]driver.Cluster, 0, len(visible)) + + for i := range visible { + if namePrefix != "" && !strings.HasPrefix(visible[i].ClusterName, namePrefix) { + continue + } + + all = append(all, visible[i]) + } + sort.Slice(all, func(i, j int) bool { return all[i].ClusterName < all[j].ClusterName }) start, end, nextTok, err := m.paginate(len(all), page) @@ -271,7 +273,11 @@ func (m *Mock) ListClusters( return all[start:end], nextTok, nil } -// DeleteCluster removes a cluster and frees its name for reuse. +// DeleteCluster deletes a cluster. With AsyncSettle off (the default) it is +// removed at once and its name freed. Under AsyncSettle it reports DELETING via +// Describe/List for a settle window and is then removed (lazily, on the next +// read), as real MSK does. Deleting a cluster that is already DELETING returns +// DELETING again. func (m *Mock) DeleteCluster(_ context.Context, arn, currentVersion string) (arnOut, state string, err error) { cd, err := m.getCluster(arn) if err != nil { @@ -280,6 +286,12 @@ func (m *Mock) DeleteCluster(_ context.Context, arn, currentVersion string) (arn cd.mu.Lock() + if cd.cluster.State == driver.ClusterStateDeleting { + cd.mu.Unlock() + + return arn, driver.ClusterStateDeleting, nil + } + // Honor the optimistic-concurrency version: a stale currentVersion is a 400. if currentVersion != "" && cd.cluster.CurrentVersion != currentVersion { cd.mu.Unlock() @@ -290,10 +302,19 @@ func (m *Mock) DeleteCluster(_ context.Context, arn, currentVersion string) (arn name := cd.cluster.ClusterName cd.cluster.State = driver.ClusterStateDeleting + // A delete supersedes any still-open CREATING window. + cd.readyAt = time.Time{} + cd.deleteAt = m.settleUntil(clusterDeleteSettle) + async := !cd.deleteAt.IsZero() cd.mu.Unlock() - m.clusters.Delete(arn) - m.clusterNames.Delete(name) + if async { + return arn, driver.ClusterStateDeleting, nil + } + + m.reapMu.Lock() + m.removeCluster(arn, name) + m.reapMu.Unlock() return arn, driver.ClusterStateDeleting, nil } diff --git a/providers/aws/kafka/clusters_v2.go b/providers/aws/kafka/clusters_v2.go index afaf1acf2..b94d0a3b5 100644 --- a/providers/aws/kafka/clusters_v2.go +++ b/providers/aws/kafka/clusters_v2.go @@ -95,13 +95,18 @@ func (m *Mock) createClusterV2Serverless(in driver.CreateClusterV2Input) (*drive } // insertCluster finishes minting a cluster (ARN, version, state, creation time) -// and stores it atomically under createMu, rejecting a duplicate name. +// and stores it atomically under createMu, rejecting a duplicate name. The +// stored state is ACTIVE; under AsyncSettle a CREATING window is overlaid on +// reads, and the returned cluster reports it. // //nolint:gocritic // hugeParam: takes the assembled cluster by value to finalize. func (m *Mock) insertCluster(cluster driver.Cluster) (*driver.Cluster, error) { m.createMu.Lock() defer m.createMu.Unlock() + // A name held by a cluster that has finished deleting is free again. + m.reapName(cluster.ClusterName) + if _, dup := m.clusterNames.Get(cluster.ClusterName); dup { return nil, conflict("cluster already exists: %s", cluster.ClusterName) } @@ -111,10 +116,18 @@ func (m *Mock) insertCluster(cluster driver.Cluster) (*driver.Cluster, error) { cluster.CurrentVersion = "K" + idgen.GenerateID("") cluster.CreationTime = m.now() - m.clusters.Set(cluster.ClusterARN, &clusterData{cluster: cluster}) + if cluster.ClusterType == driver.ClusterTypeProvisioned { + cluster.RawOptions = m.withDefaultEncryption(cluster.RawOptions) + } + + cd := &clusterData{cluster: cluster, readyAt: m.settleUntil(clusterCreateSettle)} + + m.clusters.Set(cluster.ClusterARN, cd) m.clusterNames.Set(cluster.ClusterName, cluster.ClusterARN) - out := snapshotCluster(cluster) + cd.mu.RLock() + out := m.observedClusterLocked(cd) + cd.mu.RUnlock() return &out, nil } @@ -134,14 +147,12 @@ func (m *Mock) ListClustersV2( return nil, "", vErr } - vals := m.clusters.SortedValues() + visible := m.visibleClusters() - all := make([]driver.Cluster, 0, len(vals)) + all := make([]driver.Cluster, 0, len(visible)) - for _, cd := range vals { - cd.mu.RLock() - snap := snapshotCluster(cd.cluster) - cd.mu.RUnlock() + for i := range visible { + snap := visible[i] if namePrefix != "" && !strings.HasPrefix(snap.ClusterName, namePrefix) { continue diff --git a/providers/aws/kafka/encryption.go b/providers/aws/kafka/encryption.go new file mode 100644 index 000000000..81bfc927e --- /dev/null +++ b/providers/aws/kafka/encryption.go @@ -0,0 +1,101 @@ +package kafka + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + + "github.com/stackshy/cloudemu/v2/internal/idgen" +) + +// encryptionInfoKey is the RawOptions key holding a provisioned cluster's +// EncryptionInfo block (the wire layer stores the request block under it). +const encryptionInfoKey = "encryptionInfo" + +// managedKafkaKeyARN returns the ARN of this account+region's AWS managed key +// for MSK (alias/aws/kafka). Real MSK encrypts data at rest under that key when +// a create names none, and DescribeCluster reports its key ARN in +// encryptionAtRest.dataVolumeKMSKeyId +// (https://docs.aws.amazon.com/msk/latest/developerguide/msk-working-with-encryption.html). +// There is one such key per account and region, so the key id is a UUID derived +// deterministically from both: every cluster in the account+region reports the +// same key, as in AWS. +func (m *Mock) managedKafkaKeyARN() string { + sum := sha256.Sum256([]byte("alias/aws/kafka|" + m.opts.AccountID + "|" + m.opts.Region)) + // Shape the digest as a version-4 UUID, like a real KMS key id. + sum[6] = sum[6]&0x0f | 0x40 + sum[8] = sum[8]&0x3f | 0x80 + h := hex.EncodeToString(sum[:16]) + keyID := h[0:8] + "-" + h[8:12] + "-" + h[12:16] + "-" + h[16:20] + "-" + h[20:32] + + return idgen.AWSARN("kms", m.opts.Region, m.opts.AccountID, "key/"+keyID) +} + +// withDefaultEncryption returns raw with the cluster's EncryptionInfo completed +// the way real MSK completes it. Every field the request leaves out gets its +// documented default: encryptionAtRest.dataVolumeKMSKeyId the AWS managed key, +// encryptionInTransit.clientBroker TLS and encryptionInTransit.inCluster true +// (https://docs.aws.amazon.com/msk/1.0/apireference/clusters.html#clusters-model-encryptionintransit). +// Fields the request sets, including inCluster false, are kept, as is any other +// content of the block. A block that is not a JSON object is left as sent. raw +// is not mutated. +func (m *Mock) withDefaultEncryption(raw map[string]json.RawMessage) map[string]json.RawMessage { + out := copyRaw(raw) + if out == nil { + out = map[string]json.RawMessage{} + } + + var info map[string]json.RawMessage + if blk := out[encryptionInfoKey]; len(blk) > 0 { + if err := json.Unmarshal(blk, &info); err != nil { + return out + } + } + + if info == nil { + info = map[string]json.RawMessage{} + } + + changed := defaultField(info, "encryptionAtRest", "dataVolumeKMSKeyId", m.managedKafkaKeyARN()) + changed = defaultField(info, "encryptionInTransit", "clientBroker", "TLS") || changed + changed = defaultField(info, "encryptionInTransit", "inCluster", true) || changed + + if changed { + out[encryptionInfoKey] = mustJSON(info) + } + + return out +} + +// defaultField sets info[block][field] to val when the field is absent, null or +// an empty string, creating the block if needed, and reports whether it did. +func defaultField(info map[string]json.RawMessage, block, field string, val any) bool { + var blk map[string]json.RawMessage + if r, ok := info[block]; ok { + _ = json.Unmarshal(r, &blk) + } + + if blk == nil { + blk = map[string]json.RawMessage{} + } + + if v, ok := blk[field]; ok && string(v) != "null" && string(v) != `""` { + return false + } + + blk[field] = mustJSON(val) + info[block] = mustJSON(blk) + + return true +} + +// mustJSON marshals a value built from JSON-safe types (maps, strings, bools, +// raw messages), which cannot fail. +func mustJSON(v any) json.RawMessage { + b, err := json.Marshal(v) + if err != nil { + return json.RawMessage("null") + } + + return b +} diff --git a/providers/aws/kafka/kafka.go b/providers/aws/kafka/kafka.go index b4316fbb3..85371dc0e 100644 --- a/providers/aws/kafka/kafka.go +++ b/providers/aws/kafka/kafka.go @@ -10,9 +10,11 @@ // secrets, policy, operations) live on clusterData under its lock so a mutation // is atomic with its reads. // -// Clusters are provisioned immediately Active (deterministic, no wall-clock -// wait), with a server-minted ARN. Cluster names are unique per account -// (ConflictException on a duplicate). All reads deep-copy maps, slices, and +// Clusters report ACTIVE immediately by default (deterministic, no wall-clock +// wait), with a server-minted ARN. Under config.Options.AsyncSettle a new +// cluster reports CREATING and a deleted one DELETING for a settle window (see +// lifecycle.go), and a new VPC connection reports CREATING. Cluster names are +// unique per account (ConflictException on a duplicate). All reads deep-copy maps, slices, and // json.RawMessage so a caller can never mutate stored state through a result. package kafka @@ -26,6 +28,7 @@ import ( "github.com/stackshy/cloudemu/v2/internal/idgen" "github.com/stackshy/cloudemu/v2/internal/memstore" "github.com/stackshy/cloudemu/v2/services/kafka/driver" + mondriver "github.com/stackshy/cloudemu/v2/services/monitoring/driver" ) // Compile-time check that Mock implements driver.Kafka. @@ -52,14 +55,25 @@ type clusterData struct { scramSecrets []string policy string policyVersion string - mu sync.RWMutex + + // readyAt closes the CREATING window of a new cluster and deleteAt the + // DELETING window of a deleted one (both zero unless AsyncSettle is on). + // metricsSampleAt is when Tick last published its AWS/Kafka metrics. + readyAt time.Time + deleteAt time.Time + metricsSampleAt time.Time + + mu sync.RWMutex } // vpcConnData is a stored VPC connection plus its lock. State transitions (e.g. // a client-vpc-connection reject) mutate under mu. type vpcConnData struct { vpc driver.VpcConnection - mu sync.RWMutex + // readyAt closes the CREATING window of a new connection (zero unless + // AsyncSettle is on). + readyAt time.Time + mu sync.RWMutex } // replicatorData is a stored replicator plus its lock. UpdateReplicationInfo @@ -103,6 +117,15 @@ type Mock struct { // serializes replicator creation so the name-scan and ARN insert are atomic. createMu sync.Mutex + // reapMu serializes the lazy removal of a cluster whose DELETING window has + // elapsed, so two readers never both free its name. Lock order: + // createMu -> reapMu -> clusterData.mu. + reapMu sync.Mutex + + // monitoring receives the AWS/Kafka metrics Tick publishes; nil (the + // default) disables publishing. + monitoring mondriver.Monitoring + opts *config.Options } diff --git a/providers/aws/kafka/lifecycle.go b/providers/aws/kafka/lifecycle.go new file mode 100644 index 000000000..dea6151b1 --- /dev/null +++ b/providers/aws/kafka/lifecycle.go @@ -0,0 +1,157 @@ +package kafka + +import ( + "time" + + "github.com/stackshy/cloudemu/v2/internal/settle" + "github.com/stackshy/cloudemu/v2/services/kafka/driver" +) + +// Settle windows for the MSK lifecycle under config.Options.AsyncSettle. Real +// MSK takes many minutes for each; the emulator keeps the transitions short so +// a polling client observes them without slowing tests down. With AsyncSettle +// off, opts.SettleDuration returns 0 and every transition is immediate. +const ( + clusterCreateSettle = settle.DefaultClusterSettle // CREATING -> ACTIVE + clusterDeleteSettle = settle.DefaultClusterSettle // DELETING -> removed + vpcConnCreateSettle = settle.DefaultClusterSettle // CREATING -> AVAILABLE +) + +// vpcConnStateCreating is the transient state a new VPC connection reports +// under AsyncSettle (VpcConnectionState: CREATING | AVAILABLE | INACTIVE | +// UPDATING | DEACTIVATING | DELETING | FAILED | REJECTED | REJECTING). +const vpcConnStateCreating = "CREATING" + +// settleUntil returns the instant a window of the given default length opened +// now closes, or the zero time when AsyncSettle is off (no window). +func (m *Mock) settleUntil(d time.Duration) time.Time { + dur := m.opts.SettleDuration(d) + if dur <= 0 { + return time.Time{} + } + + return m.now().Add(dur) +} + +// clusterStateLocked returns the cluster's externally visible state now. The +// caller holds cd.mu (read or write). +func (m *Mock) clusterStateLocked(cd *clusterData) string { + return clusterStateAt(cd, m.now()) +} + +// clusterStateAt returns the cluster's externally visible state at t: CREATING +// while its create window is open, otherwise the stored state. The caller holds +// cd.mu (read or write). +func clusterStateAt(cd *clusterData, t time.Time) string { + if cd.cluster.State == driver.ClusterStateActive && t.Before(cd.readyAt) { + return driver.ClusterStateCreating + } + + return cd.cluster.State +} + +// observedClusterLocked returns a deep copy of the stored cluster carrying its +// visible state. The caller holds cd.mu (read or write). +func (m *Mock) observedClusterLocked(cd *clusterData) driver.Cluster { + out := snapshotCluster(cd.cluster) + out.State = m.clusterStateLocked(cd) + + return out +} + +// requireActive rejects a write against a cluster that is not ACTIVE: cluster +// updates, and topic, SCRAM secret and cluster policy writes. Real MSK accepts +// no API operations on a cluster that is CREATING or DELETING +// (https://docs.aws.amazon.com/msk/latest/developerguide/msk-cluster-states.html). +// Without AsyncSettle a cluster is always ACTIVE, so this never fires. The +// caller holds cd.mu. +func (m *Mock) requireActive(cd *clusterData) error { + if st := m.clusterStateLocked(cd); st != driver.ClusterStateActive { + return badRequest("cluster %s is in %s state; this operation requires ACTIVE", + cd.cluster.ClusterARN, st) + } + + return nil +} + +// goneAt reports whether a stored cluster is DELETING and its delete window has +// elapsed at t, i.e. real MSK would no longer return it. A DELETING cluster with +// no window (restored from a snapshot that predates windows) counts as elapsed. +func goneAt(cd *clusterData, t time.Time) bool { + cd.mu.RLock() + defer cd.mu.RUnlock() + + return cd.cluster.State == driver.ClusterStateDeleting && !t.Before(cd.deleteAt) +} + +// reapIfGone removes a cluster whose DELETING window has elapsed now and frees +// its name, reporting whether it was removed. Request paths call it lazily on +// read; Tick calls reapIfGoneAt so a finished delete is removed (and persisted) +// even when nothing reads the cluster. +func (m *Mock) reapIfGone(arn string, cd *clusterData) bool { + return m.reapIfGoneAt(arn, cd, m.now()) +} + +// reapIfGoneAt is reapIfGone judged at t. +func (m *Mock) reapIfGoneAt(arn string, cd *clusterData, t time.Time) bool { + if !goneAt(cd, t) { + return false + } + + m.reapMu.Lock() + defer m.reapMu.Unlock() + + // Re-check under reapMu: a concurrent reader may have removed it already. + cur, ok := m.clusters.Get(arn) + if !ok || cur != cd { + return true + } + + cd.mu.RLock() + name := cd.cluster.ClusterName + cd.mu.RUnlock() + + m.removeCluster(arn, name) + + return true +} + +// removeCluster drops a cluster from the store and frees its name, but only +// when the name still points at this ARN (a later cluster may hold it). +func (m *Mock) removeCluster(arn, name string) { + m.clusters.Delete(arn) + + if owner, ok := m.clusterNames.Get(name); ok && owner == arn { + m.clusterNames.Delete(name) + } +} + +// reapName frees name when it is claimed by a cluster that has finished +// deleting, so a create can reuse it. The caller holds createMu. +func (m *Mock) reapName(name string) { + arn, ok := m.clusterNames.Get(name) + if !ok { + return + } + + cd, ok := m.clusters.Get(arn) + if !ok { + m.clusterNames.Delete(name) + + return + } + + m.reapIfGone(arn, cd) +} + +// observedVpcConnectionLocked returns a copy of the stored VPC connection +// carrying its visible state: CREATING while its create window is open. The +// caller holds vd.mu (read or write). +func (m *Mock) observedVpcConnectionLocked(vd *vpcConnData) driver.VpcConnection { + out := snapshotVpcConnection(vd.vpc) + if out.State == vpcConnStateAvailable && m.now().Before(vd.readyAt) { + out.State = vpcConnStateCreating + } + + return out +} diff --git a/providers/aws/kafka/metrics.go b/providers/aws/kafka/metrics.go new file mode 100644 index 000000000..a95622aa5 --- /dev/null +++ b/providers/aws/kafka/metrics.go @@ -0,0 +1,149 @@ +package kafka + +import ( + "context" + "sort" + "strconv" + "time" + + "github.com/stackshy/cloudemu/v2/services/kafka/driver" + mondriver "github.com/stackshy/cloudemu/v2/services/monitoring/driver" +) + +// AWS/Kafka CloudWatch metrics for provisioned clusters +// (https://docs.aws.amazon.com/msk/latest/developerguide/metrics-details.html). +// +// Real MSK pushes its metrics on a schedule, not per request: "The metrics that +// you configure for your MSK Provisioned clusters are automatically collected +// and pushed to CloudWatch at 1 minute intervals." Tick does the same on the +// shared serve scheduler, and only for clusters that are ACTIVE (every DEFAULT +// metric is documented as visible "After the cluster gets to the ACTIVE +// state"). The dimension names contain spaces ("Cluster Name", "Broker ID"). +// +// Only DEFAULT-level metrics whose values cloudemu actually knows are +// published (DEFAULT metrics are published at every monitoring level): +// +// - Cluster Name: ActiveControllerCount (1: exactly one active controller), +// GlobalTopicCount and GlobalPartitionCount (from the cluster's stored +// topics; partitions exclude replicas, as documented) and +// OfflinePartitionsCount (0: no broker is ever unavailable). +// - Cluster Name, Broker ID (brokers 1..NumberOfBrokerNodes, the IDs +// ListNodes reports): UnderReplicatedPartitions and +// UnderMinIsrPartitionCount (0: replicas never fall behind, since no +// broker runs). +// +// The CPU, memory, disk, network, connection and throughput metrics describe +// broker hosts and client traffic that cloudemu does not run, and per-broker +// PartitionCount/LeaderCount need a partition assignment it does not model, +// so none of them is published. Serverless clusters publish a different +// metric set and are skipped. +const ( + metricsNamespace = "AWS/Kafka" + + dimClusterName = "Cluster Name" + dimBrokerID = "Broker ID" + + unitCount = "Count" + + // metricsInterval is how often the AWS/Kafka metrics are published. + metricsInterval = time.Minute +) + +// SetMonitoring wires the CloudWatch backend that receives the AWS/Kafka +// metrics. With no backend wired (the default) nothing is published. +func (m *Mock) SetMonitoring(mon mondriver.Monitoring) { + m.monitoring = mon +} + +// Tick is the MSK work of the shared serve scheduler (config.Tickable). At now +// it removes every cluster whose DELETING window has ended, so a finished +// delete is dropped (and persisted) even when nothing reads the cluster, and +// publishes the AWS/Kafka metrics that are due for every ACTIVE provisioned +// cluster. It reports whether it removed or published anything. Every state +// check uses now, not the options clock. Library mode never calls it, so tests +// drive it with a FakeClock. Each cluster is sampled under its own lock and +// everything is published after all locks are released. +func (m *Mock) Tick(now time.Time) bool { + arns := m.clusters.Keys() + sort.Strings(arns) + + reaped := false + + var data []mondriver.MetricDatum + + for _, arn := range arns { + cd, ok := m.clusters.Get(arn) + if !ok { + continue + } + + if m.reapIfGoneAt(arn, cd, now) { + reaped = true + + continue + } + + if m.monitoring != nil { + data = append(data, m.dueMetrics(cd, now)...) + } + } + + if len(data) > 0 { + _ = m.monitoring.PutMetricData(context.Background(), data) + } + + return reaped || len(data) > 0 +} + +// dueMetrics samples cd's metrics if they are due at now and records the sample +// time. A cluster that is not ACTIVE, or is serverless, reports nothing. +// Callers must not hold cd.mu. +func (*Mock) dueMetrics(cd *clusterData, now time.Time) []mondriver.MetricDatum { + cd.mu.Lock() + defer cd.mu.Unlock() + + if cd.cluster.ClusterType != driver.ClusterTypeProvisioned || + clusterStateAt(cd, now) != driver.ClusterStateActive { + return nil + } + + if !cd.metricsSampleAt.IsZero() && now.Sub(cd.metricsSampleAt) < metricsInterval { + return nil + } + + cd.metricsSampleAt = now + + name := cd.cluster.ClusterName + + var partitions int64 + for _, t := range cd.topics { + partitions += int64(t.NumberOfPartitions) + } + + datum := func(metric string, value float64, dims map[string]string) mondriver.MetricDatum { + return mondriver.MetricDatum{ + Namespace: metricsNamespace, MetricName: metric, Value: value, + Unit: unitCount, Dimensions: dims, Timestamp: now, + } + } + + clusterDims := map[string]string{dimClusterName: name} + + data := []mondriver.MetricDatum{ + datum("ActiveControllerCount", 1, clusterDims), + datum("GlobalTopicCount", float64(len(cd.topics)), clusterDims), + datum("GlobalPartitionCount", float64(partitions), clusterDims), + datum("OfflinePartitionsCount", 0, clusterDims), + } + + for i := 1; i <= int(cd.cluster.NumberOfBrokerNodes); i++ { + dims := map[string]string{dimClusterName: name, dimBrokerID: strconv.Itoa(i)} + + data = append(data, + datum("UnderReplicatedPartitions", 0, dims), + datum("UnderMinIsrPartitionCount", 0, dims), + ) + } + + return data +} diff --git a/providers/aws/kafka/nodes.go b/providers/aws/kafka/nodes.go index 312089942..d6e4baae1 100644 --- a/providers/aws/kafka/nodes.go +++ b/providers/aws/kafka/nodes.go @@ -9,8 +9,9 @@ import ( ) // ListNodes returns one synthetic broker node per broker in the cluster, sized -// to the cluster's NumberOfBrokerNodes. Broker IDs are 1..N; endpoints and the -// client subnet are derived deterministically from the cluster name. +// to the cluster's NumberOfBrokerNodes. Broker IDs are 1..N; endpoints are +// derived deterministically from the cluster name, and each broker's client +// subnet is one of the cluster's own ClientSubnets (see brokerSubnet). func (m *Mock) ListNodes( _ context.Context, arn string, page driver.Page, ) (nodes []driver.Node, next string, err error) { @@ -45,6 +46,11 @@ func (m *Mock) synthNodes(c *driver.Cluster) []driver.Node { instance = c.BrokerNodeGroupInfo.InstanceType } + var subnets []string + if c.BrokerNodeGroupInfo != nil { + subnets = c.BrokerNodeGroupInfo.ClientSubnets + } + nodes := make([]driver.Node, 0, count) for i := 1; i <= count; i++ { @@ -56,7 +62,7 @@ func (m *Mock) synthNodes(c *driver.Cluster) []driver.Node { NodeType: "BROKER", InstanceType: instance, RawOptions: map[string]json.RawMessage{ - "brokerNodeInfo": brokerNodeInfoJSON(i, endpoint, c.KafkaVersion), + "brokerNodeInfo": brokerNodeInfoJSON(i, endpoint, brokerSubnet(subnets, i), c.KafkaVersion), }, }) } @@ -64,13 +70,28 @@ func (m *Mock) synthNodes(c *driver.Cluster) []driver.Node { return nodes } +// brokerSubnet returns the client subnet broker brokerID (1-based) lives in. +// Real MSK distributes brokers evenly across the cluster's client subnets (one +// per AZ), so brokers are assigned round-robin; "" when the cluster has none. +func brokerSubnet(subnets []string, brokerID int) string { + if len(subnets) == 0 { + return "" + } + + return subnets[(brokerID-1)%len(subnets)] +} + // brokerNodeInfoJSON builds the raw brokerNodeInfo block for a synthetic node. -// brokerId is a JSON number (the SDK models it as *float64). -func brokerNodeInfoJSON(brokerID int, endpoint, kafkaVersion string) json.RawMessage { +// brokerId is a JSON number (the SDK models it as *float64). clientSubnet is +// omitted when the cluster records no client subnets. +func brokerNodeInfoJSON(brokerID int, endpoint, subnet, kafkaVersion string) json.RawMessage { info := map[string]any{ - "brokerId": brokerID, - "clientSubnet": "subnet-" + strconv.Itoa(brokerID), - "endpoints": []string{endpoint}, + "brokerId": brokerID, + "endpoints": []string{endpoint}, + } + + if subnet != "" { + info["clientSubnet"] = subnet } if kafkaVersion != "" { diff --git a/providers/aws/kafka/operations.go b/providers/aws/kafka/operations.go index 5ba752736..34e2b9470 100644 --- a/providers/aws/kafka/operations.go +++ b/providers/aws/kafka/operations.go @@ -117,6 +117,10 @@ func (m *Mock) mutateClusterErr( cd.mu.Lock() defer cd.mu.Unlock() + if err := m.requireActive(cd); err != nil { + return nil, err + } + if currentVersion != "" && cd.cluster.CurrentVersion != currentVersion { return nil, badRequest( "currentVersion %q does not match cluster version %q", diff --git a/providers/aws/kafka/parity_test.go b/providers/aws/kafka/parity_test.go new file mode 100644 index 000000000..9b4e5217d --- /dev/null +++ b/providers/aws/kafka/parity_test.go @@ -0,0 +1,525 @@ +package kafka_test + +import ( + "context" + "encoding/json" + "errors" + "strconv" + "strings" + "testing" + "time" + + "github.com/stackshy/cloudemu/v2/config" + "github.com/stackshy/cloudemu/v2/providers/aws/cloudwatch" + "github.com/stackshy/cloudemu/v2/providers/aws/kafka" + "github.com/stackshy/cloudemu/v2/services/kafka/driver" + mondriver "github.com/stackshy/cloudemu/v2/services/monitoring/driver" +) + +// settleWindow comfortably covers the provider's settle windows. +const settleWindow = 10 * time.Second + +func newSettleMock(t *testing.T) (*kafka.Mock, *config.FakeClock) { + t.Helper() + + fc := config.NewFakeClock(time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)) + + return kafka.New(config.NewOptions(config.WithClock(fc), config.WithAsyncSettle())), fc +} + +func createThreeAZCluster(t *testing.T, m *kafka.Mock, name string) *driver.Cluster { + t.Helper() + + c, err := m.CreateCluster(context.Background(), driver.CreateClusterInput{ + ClusterName: name, + NumberOfBrokerNodes: 6, + BrokerNodeGroupInfo: &driver.BrokerNodeGroupInfo{ + ClientSubnets: []string{"subnet-aaa", "subnet-bbb", "subnet-ccc"}, + InstanceType: "kafka.m5.large", + }, + }) + if err != nil { + t.Fatalf("CreateCluster: %v", err) + } + + return c +} + +func wantException(t *testing.T, err error, ex string) { + t.Helper() + + var apiErr *driver.APIError + if !errors.As(err, &apiErr) || apiErr.Exception != ex { + t.Fatalf("want %s, got %v", ex, err) + } +} + +// TestListNodesClientSubnetFromCluster asserts each broker's clientSubnet is +// one of the cluster's ClientSubnets, distributed evenly (round-robin) the way +// MSK spreads brokers across AZs, not a fabricated subnet-. +func TestListNodesClientSubnetFromCluster(t *testing.T) { + m := newMock(t) + c := createThreeAZCluster(t, m, "subnets") + + nodes, _, err := m.ListNodes(context.Background(), c.ClusterARN, driver.Page{}) + if err != nil || len(nodes) != 6 { + t.Fatalf("ListNodes: %v len=%d", err, len(nodes)) + } + + want := []string{"subnet-aaa", "subnet-bbb", "subnet-ccc", "subnet-aaa", "subnet-bbb", "subnet-ccc"} + + for i, n := range nodes { + var info struct { + BrokerID int `json:"brokerId"` + ClientSubnet string `json:"clientSubnet"` + } + + if err := json.Unmarshal(n.RawOptions["brokerNodeInfo"], &info); err != nil { + t.Fatalf("decode brokerNodeInfo: %v", err) + } + + if info.BrokerID != i+1 || info.ClientSubnet != want[i] { + t.Fatalf("node %d: brokerId=%d clientSubnet=%q, want %d/%q", i, info.BrokerID, info.ClientSubnet, i+1, want[i]) + } + } +} + +// TestListNodesNoSubnetsOmitsClientSubnet asserts a cluster with no recorded +// client subnets does not get a fabricated one. +func TestListNodesNoSubnetsOmitsClientSubnet(t *testing.T) { + m := newMock(t) + ctx := context.Background() + + c, _ := m.CreateCluster(ctx, driver.CreateClusterInput{ClusterName: "nosubnet", NumberOfBrokerNodes: 2, BrokerNodeGroupInfo: bng()}) + + nodes, _, err := m.ListNodes(ctx, c.ClusterARN, driver.Page{}) + if err != nil { + t.Fatalf("ListNodes: %v", err) + } + + if strings.Contains(string(nodes[0].RawOptions["brokerNodeInfo"]), "clientSubnet") { + t.Fatalf("clientSubnet fabricated: %s", nodes[0].RawOptions["brokerNodeInfo"]) + } +} + +// TestClusterCreatingThenActive asserts that under AsyncSettle a new cluster is +// CREATING (create response, Describe, List) and rejects updates, then settles +// to ACTIVE. +func TestClusterCreatingThenActive(t *testing.T) { + m, fc := newSettleMock(t) + ctx := context.Background() + + c := createThreeAZCluster(t, m, "settling") + if c.State != driver.ClusterStateCreating { + t.Fatalf("create state = %s, want CREATING", c.State) + } + + d, err := m.DescribeCluster(ctx, c.ClusterARN) + if err != nil || d.State != driver.ClusterStateCreating { + t.Fatalf("describe = %v/%v, want CREATING", d, err) + } + + list, _, _ := m.ListClustersV2(ctx, "", "", driver.Page{}) + if len(list) != 1 || list[0].State != driver.ClusterStateCreating { + t.Fatalf("list = %+v, want one CREATING", list) + } + + _, err = m.UpdateBrokerCount(ctx, c.ClusterARN, c.CurrentVersion, 9) + wantException(t, err, driver.ExBadRequest) + + fc.Advance(settleWindow) + + d, _ = m.DescribeCluster(ctx, c.ClusterARN) + if d.State != driver.ClusterStateActive { + t.Fatalf("after settle state = %s, want ACTIVE", d.State) + } + + if _, err := m.UpdateBrokerCount(ctx, c.ClusterARN, d.CurrentVersion, 9); err != nil { + t.Fatalf("UpdateBrokerCount after ACTIVE: %v", err) + } +} + +// TestClusterActiveImmediatelyWithoutSettle pins the default behavior. +func TestClusterActiveImmediatelyWithoutSettle(t *testing.T) { + m := newMock(t) + + c := createThreeAZCluster(t, m, "instant") + if c.State != driver.ClusterStateActive { + t.Fatalf("state = %s, want ACTIVE", c.State) + } +} + +// TestDeleteClusterDeletingThenGone asserts that under AsyncSettle a deleted +// cluster stays visible as DELETING (Describe and List), keeps its name, and +// is gone once the window elapses. +func TestDeleteClusterDeletingThenGone(t *testing.T) { + m, fc := newSettleMock(t) + ctx := context.Background() + + c := createThreeAZCluster(t, m, "doomed") + fc.Advance(settleWindow) + + _, state, err := m.DeleteCluster(ctx, c.ClusterARN, "") + if err != nil || state != driver.ClusterStateDeleting { + t.Fatalf("DeleteCluster = %s/%v", state, err) + } + + d, err := m.DescribeCluster(ctx, c.ClusterARN) + if err != nil || d.State != driver.ClusterStateDeleting { + t.Fatalf("describe while deleting = %+v/%v, want DELETING", d, err) + } + + list, _, _ := m.ListClusters(ctx, "", driver.Page{}) + if len(list) != 1 || list[0].State != driver.ClusterStateDeleting { + t.Fatalf("list while deleting = %+v", list) + } + + // A second delete is idempotent; updates are rejected. + if _, state, err = m.DeleteCluster(ctx, c.ClusterARN, ""); err != nil || state != driver.ClusterStateDeleting { + t.Fatalf("second DeleteCluster = %s/%v", state, err) + } + + _, err = m.UpdateMonitoring(ctx, c.ClusterARN, "", json.RawMessage(`{"enhancedMonitoring":"PER_BROKER"}`)) + wantException(t, err, driver.ExBadRequest) + + // The name is still held while the cluster is DELETING. + _, err = m.CreateCluster(ctx, driver.CreateClusterInput{ClusterName: "doomed", NumberOfBrokerNodes: 3, BrokerNodeGroupInfo: bng()}) + wantException(t, err, driver.ExConflict) + + fc.Advance(settleWindow) + + _, err = m.DescribeCluster(ctx, c.ClusterARN) + wantException(t, err, driver.ExNotFound) + + if list, _, _ = m.ListClusters(ctx, "", driver.Page{}); len(list) != 0 { + t.Fatalf("list after delete = %+v", list) + } + + if _, err := m.CreateCluster(ctx, driver.CreateClusterInput{ClusterName: "doomed", NumberOfBrokerNodes: 3, BrokerNodeGroupInfo: bng()}); err != nil { + t.Fatalf("name not freed after delete: %v", err) + } +} + +// TestDeleteFreesNameForCreateWithoutRead asserts a create reclaims the name of +// a cluster whose delete window has elapsed even if nothing read it since. +func TestDeleteFreesNameForCreateWithoutRead(t *testing.T) { + m, fc := newSettleMock(t) + ctx := context.Background() + + c := createThreeAZCluster(t, m, "reuse") + fc.Advance(settleWindow) + + if _, _, err := m.DeleteCluster(ctx, c.ClusterARN, ""); err != nil { + t.Fatalf("DeleteCluster: %v", err) + } + + fc.Advance(settleWindow) + + if _, err := m.CreateCluster(ctx, driver.CreateClusterInput{ClusterName: "reuse", NumberOfBrokerNodes: 3, BrokerNodeGroupInfo: bng()}); err != nil { + t.Fatalf("CreateCluster reusing name: %v", err) + } +} + +// TestVpcConnectionCreatingThenAvailable asserts CREATING (the real transient +// VpcConnectionState; there is no PENDING) under AsyncSettle, then AVAILABLE. +func TestVpcConnectionCreatingThenAvailable(t *testing.T) { + m, fc := newSettleMock(t) + ctx := context.Background() + + c := createThreeAZCluster(t, m, "vpc-target") + fc.Advance(settleWindow) + + body := json.RawMessage(`{"targetClusterArn":"` + c.ClusterARN + + `","authentication":"SASL_IAM","vpcId":"vpc-1","clientSubnets":["subnet-x"],"securityGroups":["sg-1"]}`) + + v, err := m.CreateVpcConnection(ctx, body) + if err != nil || v.State != "CREATING" { + t.Fatalf("CreateVpcConnection = %+v/%v, want CREATING", v, err) + } + + d, _ := m.DescribeVpcConnection(ctx, v.VpcConnectionARN) + if d.State != "CREATING" { + t.Fatalf("describe state = %s, want CREATING", d.State) + } + + fc.Advance(settleWindow) + + d, _ = m.DescribeVpcConnection(ctx, v.VpcConnectionARN) + if d.State != "AVAILABLE" { + t.Fatalf("settled state = %s, want AVAILABLE", d.State) + } +} + +// kafkaSamples returns the per-minute SampleCount datapoints of an AWS/Kafka +// metric with exactly dims between from and to. +func kafkaSamples( + t *testing.T, cw *cloudwatch.Mock, metric string, dims map[string]string, from, to time.Time, +) []float64 { + t.Helper() + + res, err := cw.GetMetricData(context.Background(), mondriver.GetMetricInput{ + Namespace: "AWS/Kafka", MetricName: metric, Dimensions: dims, + StartTime: from, EndTime: to, Period: 60, Stat: "SampleCount", + }) + if err != nil { + t.Fatalf("GetMetricData %s: %v", metric, err) + } + + return res.Values +} + +// TestClusterMetricsPublishedByTick asserts the AWS/Kafka metrics come from the +// scheduler, not from requests: nothing is published on create, nothing while +// the cluster is CREATING, then one sample per minute once it is ACTIVE, under +// the documented dimension names ("Cluster Name", "Broker ID"), with the topic +// counts read from the cluster's stored topics. +func TestClusterMetricsPublishedByTick(t *testing.T) { + fc := config.NewFakeClock(time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)) + opts := config.NewOptions(config.WithClock(fc), config.WithAsyncSettle()) + m := kafka.New(opts) + cw := cloudwatch.New(opts) + m.SetMonitoring(cw) + + ctx := context.Background() + start := fc.Now() + c := createThreeAZCluster(t, m, "metered") + + ids, err := cw.ListMetricsDetailed(ctx) + if err != nil || len(ids) != 0 { + t.Fatalf("metrics published on create: %+v/%v", ids, err) + } + + if m.Tick(fc.Now()) { + t.Fatal("Tick published metrics for a CREATING cluster") + } + + fc.Advance(settleWindow) + + if _, err := m.CreateTopic(ctx, c.ClusterARN, json.RawMessage( + `{"topicName":"orders","partitionCount":6,"replicationFactor":3}`)); err != nil { + t.Fatalf("CreateTopic: %v", err) + } + + if !m.Tick(fc.Now()) { + t.Fatal("Tick published nothing for an ACTIVE cluster") + } + + // A second tick inside the same minute is not due. + fc.Advance(30 * time.Second) + + if m.Tick(fc.Now()) { + t.Fatal("Tick republished within the 1-minute interval") + } + + fc.Advance(30 * time.Second) + + if !m.Tick(fc.Now()) { + t.Fatal("Tick did not publish once the minute elapsed") + } + + end := fc.Now().Add(time.Second) + clusterDims := map[string]string{"Cluster Name": c.ClusterName} + + if got := kafkaSamples(t, cw, "ActiveControllerCount", clusterDims, start, end); len(got) != 2 { + t.Fatalf("ActiveControllerCount samples = %v, want 2 one-minute samples", got) + } + + topics, err := cw.GetMetricData(ctx, mondriver.GetMetricInput{ + Namespace: "AWS/Kafka", MetricName: "GlobalPartitionCount", Dimensions: clusterDims, + StartTime: start, EndTime: end, Period: 3600, Stat: "Maximum", + }) + if err != nil || len(topics.Values) != 1 || topics.Values[0] != 6 || topics.Unit != "Count" { + t.Fatalf("GlobalPartitionCount = %+v/%v, want 6 Count", topics, err) + } + + for broker := 1; broker <= 6; broker++ { + dims := map[string]string{"Cluster Name": c.ClusterName, "Broker ID": strconv.Itoa(broker)} + if got := kafkaSamples(t, cw, "UnderReplicatedPartitions", dims, start, end); len(got) != 2 { + t.Fatalf("broker %d UnderReplicatedPartitions samples = %v", broker, got) + } + } + + // Host and traffic metrics cloudemu cannot know are not fabricated. + ids, _ = cw.ListMetricsDetailed(ctx) + for _, id := range ids { + switch id.MetricName { + case "CpuUser", "CpuIdle", "MemoryUsed", "KafkaDataLogsDiskUsed", "BytesInPerSec": + t.Fatalf("fabricated metric published: %+v", id) + } + } +} + +// TestTickSkipsDeletingAndServerless asserts a DELETING cluster stops +// publishing and serverless clusters (a different metric set) never publish. +func TestTickSkipsDeletingAndServerless(t *testing.T) { + fc := config.NewFakeClock(time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)) + opts := config.NewOptions(config.WithClock(fc), config.WithAsyncSettle()) + m := kafka.New(opts) + m.SetMonitoring(cloudwatch.New(opts)) + + ctx := context.Background() + c := createThreeAZCluster(t, m, "short-lived") + fc.Advance(settleWindow) + + if _, _, err := m.DeleteCluster(ctx, c.ClusterARN, ""); err != nil { + t.Fatalf("DeleteCluster: %v", err) + } + + if m.Tick(fc.Now()) { + t.Fatal("Tick published for a DELETING cluster") + } + + if _, err := m.CreateClusterV2(ctx, driver.CreateClusterV2Input{ + ClusterName: "sls", + Serverless: json.RawMessage(`{"vpcConfigs":[{"subnetIds":["subnet-1","subnet-2"]}]}`), + }); err != nil { + t.Fatalf("CreateClusterV2: %v", err) + } + + fc.Advance(settleWindow) + + // This tick removes the cluster whose DELETING window has ended (a change); + // the next one, with only the serverless cluster left, does nothing. + if !m.Tick(fc.Now()) { + t.Fatal("Tick did not report removing the deleted cluster") + } + + fc.Advance(time.Minute) + + if m.Tick(fc.Now()) { + t.Fatal("Tick published for a serverless cluster") + } +} + +func encryptionInfo(t *testing.T, c *driver.Cluster) map[string]map[string]any { + t.Helper() + + var out map[string]map[string]any + if err := json.Unmarshal(c.RawOptions["encryptionInfo"], &out); err != nil { + t.Fatalf("decode encryptionInfo %s: %v", c.RawOptions["encryptionInfo"], err) + } + + return out +} + +// TestDefaultEncryptionAtRestKey asserts the default at-rest key is a KMS key +// ARN in the cluster's own account and region (one AWS managed aws/kafka key +// per account+region), not a hard-coded key/msk-default. +func TestDefaultEncryptionAtRestKey(t *testing.T) { + m := kafka.New(config.NewOptions(config.WithRegion("eu-west-1"), config.WithAccountID("123456789012"))) + ctx := context.Background() + + a := createThreeAZCluster(t, m, "enc-a") + b := createThreeAZCluster(t, m, "enc-b") + + keyA, _ := encryptionInfo(t, a)["encryptionAtRest"]["dataVolumeKMSKeyId"].(string) + keyB, _ := encryptionInfo(t, b)["encryptionAtRest"]["dataVolumeKMSKeyId"].(string) + + if !strings.HasPrefix(keyA, "arn:aws:kms:eu-west-1:123456789012:key/") || strings.Contains(keyA, "msk-default") { + t.Fatalf("default key = %q", keyA) + } + + if keyA != keyB { + t.Fatalf("managed key differs across clusters: %q vs %q", keyA, keyB) + } + + if len(strings.TrimPrefix(keyA, "arn:aws:kms:eu-west-1:123456789012:key/")) != 36 { + t.Fatalf("key id is not a UUID: %q", keyA) + } + + // An explicit key is kept; an in-transit-only block gets the managed key. + explicit := "arn:aws:kms:eu-west-1:123456789012:key/11111111-2222-3333-4444-555555555555" + + c, err := m.CreateCluster(ctx, driver.CreateClusterInput{ + ClusterName: "enc-explicit", NumberOfBrokerNodes: 3, BrokerNodeGroupInfo: bng(), + RawOptions: map[string]json.RawMessage{ + "encryptionInfo": json.RawMessage(`{"encryptionAtRest":{"dataVolumeKMSKeyId":"` + explicit + `"}}`), + }, + }) + if err != nil { + t.Fatalf("CreateCluster explicit: %v", err) + } + + if got := encryptionInfo(t, c)["encryptionAtRest"]["dataVolumeKMSKeyId"]; got != explicit { + t.Fatalf("explicit key = %v, want %s", got, explicit) + } + + d, err := m.CreateCluster(ctx, driver.CreateClusterInput{ + ClusterName: "enc-transit", NumberOfBrokerNodes: 3, BrokerNodeGroupInfo: bng(), + RawOptions: map[string]json.RawMessage{ + "encryptionInfo": json.RawMessage(`{"encryptionInTransit":{"clientBroker":"TLS_PLAINTEXT","inCluster":true}}`), + }, + }) + if err != nil { + t.Fatalf("CreateCluster transit-only: %v", err) + } + + info := encryptionInfo(t, d) + if info["encryptionAtRest"]["dataVolumeKMSKeyId"] != keyA || info["encryptionInTransit"]["clientBroker"] != "TLS_PLAINTEXT" { + t.Fatalf("transit-only block = %+v", info) + } +} + +// TestSnapshotSettleStates asserts the lifecycle windows survive a snapshot: a +// cluster and a VPC connection snapshotted mid-create restore still CREATING +// and settle on schedule, and a cluster snapshotted mid-delete restores +// DELETING and is gone once its window ends. +func TestSnapshotSettleStates(t *testing.T) { + src, fc := newSettleMock(t) + ctx := context.Background() + + target := createThreeAZCluster(t, src, "vpc-target") + deleting := createThreeAZCluster(t, src, "mid-delete") + + fc.Advance(settleWindow) + + creating := createThreeAZCluster(t, src, "mid-create") + + if _, _, err := src.DeleteCluster(ctx, deleting.ClusterARN, ""); err != nil { + t.Fatalf("DeleteCluster: %v", err) + } + + vpc, err := src.CreateVpcConnection(ctx, json.RawMessage( + `{"targetClusterArn":"`+target.ClusterARN+`","authentication":"SASL_IAM","vpcId":"vpc-1"}`)) + if err != nil { + t.Fatalf("CreateVpcConnection: %v", err) + } + + raw, err := src.Snapshot(ctx, true) + if err != nil { + t.Fatalf("Snapshot: %v", err) + } + + dst := kafka.New(config.NewOptions(config.WithClock(fc), config.WithAsyncSettle())) + if err := dst.Restore(ctx, raw); err != nil { + t.Fatalf("Restore: %v", err) + } + + got, err := dst.DescribeCluster(ctx, creating.ClusterARN) + if err != nil || got.State != driver.ClusterStateCreating { + t.Fatalf("restored mid-create = %+v/%v, want CREATING", got, err) + } + + got, err = dst.DescribeCluster(ctx, deleting.ClusterARN) + if err != nil || got.State != driver.ClusterStateDeleting { + t.Fatalf("restored mid-delete = %+v/%v, want DELETING", got, err) + } + + v, err := dst.DescribeVpcConnection(ctx, vpc.VpcConnectionARN) + if err != nil || v.State != "CREATING" { + t.Fatalf("restored vpc connection = %+v/%v, want CREATING", v, err) + } + + fc.Advance(settleWindow) + + if got, _ = dst.DescribeCluster(ctx, creating.ClusterARN); got.State != driver.ClusterStateActive { + t.Fatalf("restored cluster after window = %s, want ACTIVE", got.State) + } + + _, err = dst.DescribeCluster(ctx, deleting.ClusterARN) + wantException(t, err, driver.ExNotFound) + + if v, _ = dst.DescribeVpcConnection(ctx, vpc.VpcConnectionARN); v.State != "AVAILABLE" { + t.Fatalf("restored vpc connection after window = %s, want AVAILABLE", v.State) + } +} diff --git a/providers/aws/kafka/review_test.go b/providers/aws/kafka/review_test.go new file mode 100644 index 000000000..bce0ce4ab --- /dev/null +++ b/providers/aws/kafka/review_test.go @@ -0,0 +1,316 @@ +package kafka_test + +import ( + "context" + "encoding/json" + "strconv" + "sync" + "testing" + + "github.com/stackshy/cloudemu/v2/config" + "github.com/stackshy/cloudemu/v2/providers/aws/cloudwatch" + "github.com/stackshy/cloudemu/v2/providers/aws/kafka" + "github.com/stackshy/cloudemu/v2/services/kafka/driver" +) + +// TestSnapshotConcurrentWithMutations runs Snapshot while topics, SCRAM +// secrets, tags and broker counts change on a cluster, VPC connection, +// configuration and replicator. Snapshot must deep-copy each record under its +// lock, so the JSON encoding never reads a map or slice a writer is changing +// (run with -race). The final snapshot then restores every record. +func TestSnapshotConcurrentWithMutations(t *testing.T) { + m := newMock(t) + ctx := context.Background() + c := createThreeAZCluster(t, m, "busy") + + vpc, err := m.CreateVpcConnection(ctx, json.RawMessage( + `{"targetClusterArn":"`+c.ClusterARN+`","authentication":"SASL_IAM","vpcId":"vpc-1"}`)) + if err != nil { + t.Fatalf("CreateVpcConnection: %v", err) + } + + cfg, err := m.CreateConfiguration(ctx, driver.CreateConfigurationInput{ + Name: "busy-cfg", KafkaVersions: []string{"3.6.0"}, ServerProperties: []byte("a=b"), + }) + if err != nil { + t.Fatalf("CreateConfiguration: %v", err) + } + + repl, err := m.CreateReplicator(ctx, json.RawMessage(`{"replicatorName":"busy-rep","kafkaClusters":[],`+ + `"replicationInfoList":[],"serviceExecutionRoleArn":"arn:aws:iam::123456789012:role/r"}`)) + if err != nil { + t.Fatalf("CreateReplicator: %v", err) + } + + const n = 50 + + var wg sync.WaitGroup + + wg.Add(2) + + go func() { + defer wg.Done() + + for i := range n { + name := "t" + strconv.Itoa(i) + _, _ = m.CreateTopic(ctx, c.ClusterARN, json.RawMessage(`{"topicName":"`+name+`","partitionCount":1}`)) + _, _ = m.UpdateTopic(ctx, c.ClusterARN, name, json.RawMessage(`{"partitionCount":2}`)) + _, _ = m.BatchAssociateScramSecret(ctx, c.ClusterARN, + []string{"arn:aws:secretsmanager:us-east-1:123456789012:secret:AmazonMSK_" + name}) + _ = m.TagResource(ctx, c.ClusterARN, map[string]string{name: "v"}) + _ = m.TagResource(ctx, vpc.VpcConnectionARN, map[string]string{name: "v"}) + _ = m.TagResource(ctx, cfg.ARN, map[string]string{name: "v"}) + _ = m.TagResource(ctx, repl.ReplicatorARN, map[string]string{name: "v"}) + _, _ = m.UpdateBrokerCount(ctx, c.ClusterARN, "", int32(3+i%3)) + } + }() + + go func() { + defer wg.Done() + + for range n { + if _, err := m.Snapshot(ctx, false); err != nil { + t.Errorf("Snapshot: %v", err) + + return + } + } + }() + + wg.Wait() + + raw, err := m.Snapshot(ctx, false) + if err != nil { + t.Fatalf("Snapshot: %v", err) + } + + dst := newMock(t) + if err := dst.Restore(ctx, raw); err != nil { + t.Fatalf("Restore: %v", err) + } + + if topics, _, err := dst.ListTopics(ctx, c.ClusterARN, driver.Page{MaxResults: 100}); err != nil || len(topics) != n { + t.Fatalf("restored topics = %d/%v, want %d", len(topics), err, n) + } + + for _, arn := range []string{vpc.VpcConnectionARN, cfg.ARN, repl.ReplicatorARN} { + if tags, err := dst.ListTagsForResource(ctx, arn); err != nil || len(tags) < n { + t.Fatalf("restored tags on %s = %d/%v", arn, len(tags), err) + } + } +} + +// TestDefaultEncryptionInTransitCompleted asserts a partial encryptionInfo gets +// MSK's documented in-transit defaults (ClientBroker TLS, InCluster true) for +// any field it leaves out, while explicit values, including InCluster false, +// are kept. +func TestDefaultEncryptionInTransitCompleted(t *testing.T) { + m := newMock(t) + ctx := context.Background() + + cases := []struct { + name, block, client string + inCluster bool + }{ + {"at-rest-only", `{"encryptionAtRest":{"dataVolumeKMSKeyId":"arn:aws:kms:us-east-1:123456789012:key/mine"}}`, "TLS", true}, + {"client-only", `{"encryptionInTransit":{"clientBroker":"TLS_PLAINTEXT"}}`, "TLS_PLAINTEXT", true}, + {"in-cluster-only", `{"encryptionInTransit":{"inCluster":false}}`, "TLS", false}, + {"empty-block", `{}`, "TLS", true}, + } + + for _, tc := range cases { + c, err := m.CreateCluster(ctx, driver.CreateClusterInput{ + ClusterName: "enc-" + tc.name, NumberOfBrokerNodes: 3, BrokerNodeGroupInfo: bng(), + RawOptions: map[string]json.RawMessage{"encryptionInfo": json.RawMessage(tc.block)}, + }) + if err != nil { + t.Fatalf("%s: CreateCluster: %v", tc.name, err) + } + + info := encryptionInfo(t, c) + if info["encryptionInTransit"]["clientBroker"] != tc.client || + info["encryptionInTransit"]["inCluster"] != tc.inCluster { + t.Fatalf("%s: encryptionInTransit = %+v, want %s/%v", + tc.name, info["encryptionInTransit"], tc.client, tc.inCluster) + } + + if key, _ := info["encryptionAtRest"]["dataVolumeKMSKeyId"].(string); key == "" { + t.Fatalf("%s: no at-rest key: %+v", tc.name, info) + } + } +} + +// TestTickUsesItsNow asserts Tick judges a cluster's state by the time it is +// given, not the options clock: a Tick at a time past the CREATING window +// publishes even though the options clock has not moved. +func TestTickUsesItsNow(t *testing.T) { + m, fc := newSettleMock(t) + m.SetMonitoring(cloudwatch.New(config.NewOptions(config.WithClock(fc)))) + + createThreeAZCluster(t, m, "ahead") + + if !m.Tick(fc.Now().Add(settleWindow)) { + t.Fatal("Tick at a time past the CREATING window published nothing") + } + + if m.Tick(fc.Now()) { + t.Fatal("Tick before the CREATING window ends published metrics") + } +} + +// TestTickReapsFinishedDelete asserts Tick removes a cluster whose DELETING +// window has ended and reports the change (so serve persists it), even with no +// monitoring backend wired and no read in between. +func TestTickReapsFinishedDelete(t *testing.T) { + m, fc := newSettleMock(t) + ctx := context.Background() + + c := createThreeAZCluster(t, m, "reaped") + fc.Advance(settleWindow) + + if m.Tick(fc.Now()) { + t.Fatal("Tick reported a change with nothing to do") + } + + if _, _, err := m.DeleteCluster(ctx, c.ClusterARN, ""); err != nil { + t.Fatalf("DeleteCluster: %v", err) + } + + if m.Tick(fc.Now()) { + t.Fatal("Tick reaped a cluster inside its DELETING window") + } + + fc.Advance(settleWindow) + + if !m.Tick(fc.Now()) { + t.Fatal("Tick did not report reaping a finished delete") + } + + raw, err := m.Snapshot(ctx, false) + if err != nil { + t.Fatalf("Snapshot: %v", err) + } + + var snap struct { + Clusters map[string]json.RawMessage `json:"clusters"` + ClusterNames map[string]string `json:"clusterNames"` + } + if err := json.Unmarshal(raw, &snap); err != nil { + t.Fatalf("decode snapshot: %v", err) + } + + if len(snap.Clusters) != 0 || len(snap.ClusterNames) != 0 { + t.Fatalf("snapshot after reap still holds the cluster: %s", raw) + } + + if m.Tick(fc.Now()) { + t.Fatal("Tick reported a change after the cluster was gone") + } +} + +// TestClusterWritesRequireActive asserts topic, SCRAM secret and cluster +// policy writes are rejected while a cluster is CREATING or DELETING (real MSK +// accepts no API operations on it in those states) and accepted once ACTIVE. +func TestClusterWritesRequireActive(t *testing.T) { + m, fc := newSettleMock(t) + ctx := context.Background() + + c := createThreeAZCluster(t, m, "guarded") + secret := []string{"arn:aws:secretsmanager:us-east-1:123456789012:secret:AmazonMSK_s"} + + writes := map[string]func() error{ + "CreateTopic": func() error { + _, err := m.CreateTopic(ctx, c.ClusterARN, json.RawMessage(`{"topicName":"t","partitionCount":1}`)) + + return err + }, + "UpdateTopic": func() error { + _, err := m.UpdateTopic(ctx, c.ClusterARN, "t", json.RawMessage(`{"partitionCount":2}`)) + + return err + }, + "BatchAssociateScramSecret": func() error { + _, err := m.BatchAssociateScramSecret(ctx, c.ClusterARN, secret) + + return err + }, + "BatchDisassociateScramSecret": func() error { + _, err := m.BatchDisassociateScramSecret(ctx, c.ClusterARN, secret) + + return err + }, + "PutClusterPolicy": func() error { + _, err := m.PutClusterPolicy(ctx, c.ClusterARN, json.RawMessage(`{"policy":"{}"}`)) + + return err + }, + "DeleteClusterPolicy": func() error { return m.DeleteClusterPolicy(ctx, c.ClusterARN) }, + "DeleteTopic": func() error { return m.DeleteTopic(ctx, c.ClusterARN, "t") }, + } + order := []string{ + "CreateTopic", "UpdateTopic", "BatchAssociateScramSecret", "BatchDisassociateScramSecret", + "PutClusterPolicy", "DeleteClusterPolicy", "DeleteTopic", + } + + for _, op := range order { + wantException(t, writes[op](), driver.ExBadRequest) + } + + fc.Advance(settleWindow) + + for _, op := range order { + if err := writes[op](); err != nil { + t.Fatalf("%s on ACTIVE cluster: %v", op, err) + } + } + + if _, _, err := m.DeleteCluster(ctx, c.ClusterARN, ""); err != nil { + t.Fatalf("DeleteCluster: %v", err) + } + + for _, op := range order { + wantException(t, writes[op](), driver.ExBadRequest) + } +} + +// TestCompatibleVersionsAfterDeleteWindow asserts GetCompatibleKafkaVersions +// stops answering for a cluster once its DELETING window has ended. +func TestCompatibleVersionsAfterDeleteWindow(t *testing.T) { + m, fc := newSettleMock(t) + ctx := context.Background() + + c := createThreeAZCluster(t, m, "versioned") + fc.Advance(settleWindow) + + if _, _, err := m.DeleteCluster(ctx, c.ClusterARN, ""); err != nil { + t.Fatalf("DeleteCluster: %v", err) + } + + if _, err := m.GetCompatibleKafkaVersions(ctx, c.ClusterARN); err != nil { + t.Fatalf("GetCompatibleKafkaVersions while DELETING: %v", err) + } + + fc.Advance(settleWindow) + + _, err := m.GetCompatibleKafkaVersions(ctx, c.ClusterARN) + wantException(t, err, driver.ExNotFound) +} + +// TestCreateReclaimsDanglingName asserts a create succeeds when the name index +// still points at a cluster ARN that is no longer stored (a snapshot taken +// between the two removals). +func TestCreateReclaimsDanglingName(t *testing.T) { + m := kafka.New(config.NewOptions()) + ctx := context.Background() + + snap := `{"clusterNames":{"orphan":"arn:aws:kafka:us-east-1:123456789012:cluster/orphan/gone-1"}}` + if err := m.Restore(ctx, json.RawMessage(snap)); err != nil { + t.Fatalf("Restore: %v", err) + } + + if _, err := m.CreateCluster(ctx, driver.CreateClusterInput{ + ClusterName: "orphan", NumberOfBrokerNodes: 3, BrokerNodeGroupInfo: bng(), + }); err != nil { + t.Fatalf("CreateCluster over a dangling name: %v", err) + } +} diff --git a/providers/aws/kafka/scram_secrets.go b/providers/aws/kafka/scram_secrets.go index 292e23c25..f808d90c2 100644 --- a/providers/aws/kafka/scram_secrets.go +++ b/providers/aws/kafka/scram_secrets.go @@ -24,6 +24,10 @@ func (m *Mock) BatchAssociateScramSecret( cd.mu.Lock() defer cd.mu.Unlock() + if err := m.requireActive(cd); err != nil { + return nil, err + } + var unprocessed []json.RawMessage for _, arn := range secretARNs { @@ -57,6 +61,10 @@ func (m *Mock) BatchDisassociateScramSecret( cd.mu.Lock() defer cd.mu.Unlock() + if err := m.requireActive(cd); err != nil { + return nil, err + } + var unprocessed []json.RawMessage for _, arn := range secretARNs { diff --git a/providers/aws/kafka/snapshot.go b/providers/aws/kafka/snapshot.go index cf989dcc0..bfc4583ba 100644 --- a/providers/aws/kafka/snapshot.go +++ b/providers/aws/kafka/snapshot.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "time" "github.com/stackshy/cloudemu/v2/internal/snapshot" "github.com/stackshy/cloudemu/v2/services/kafka/driver" @@ -19,13 +20,20 @@ var _ snapshot.Snapshottable = (*Mock)(nil) // index of *clusterData pointers shared with the clusters store, rebuilt on // restore from each cluster's operations slice so the shared identity (a // DescribeClusterOperation resolves to the same cluster object) is preserved. -// The per-record mutexes, createMu, and the wired opts are not serialized. +// The AsyncSettle lifecycle windows (a cluster's CREATING/DELETING deadline, a +// VPC connection's CREATING deadline) and the metric sample time are +// serialized, so a restored resource keeps its transient state until its window +// ends and Tick keeps its publishing cadence. The per-record mutexes, +// createMu/reapMu, the wired opts and the monitoring backend are not. type kafkaSnapshot struct { Clusters map[string]*clusterSnapshot `json:"clusters,omitempty"` ClusterNames json.RawMessage `json:"clusterNames,omitempty"` Configs map[string]driver.Configuration `json:"configs,omitempty"` VPCConns map[string]driver.VpcConnection `json:"vpcConns,omitempty"` - Replicators map[string]*replicatorSnapshot `json:"replicators,omitempty"` + // VPCConnReadyAt holds the CREATING deadline of each VPC connection still + // inside its window (kept beside VPCConns so older snapshots still load). + VPCConnReadyAt map[string]time.Time `json:"vpcConnReadyAt,omitempty"` + Replicators map[string]*replicatorSnapshot `json:"replicators,omitempty"` } // clusterSnapshot mirrors clusterData (all fields unexported). @@ -36,6 +44,10 @@ type clusterSnapshot struct { ScramSecrets []string `json:"scramSecrets,omitempty"` Policy string `json:"policy,omitempty"` PolicyVersion string `json:"policyVersion,omitempty"` + // Lifecycle windows (AsyncSettle) and the last metric sample time. + ReadyAt time.Time `json:"readyAt,omitzero"` + DeleteAt time.Time `json:"deleteAt,omitzero"` + MetricsSampleAt time.Time `json:"metricsSampleAt,omitzero"` } // replicatorSnapshot mirrors replicatorData. @@ -45,22 +57,10 @@ type replicatorSnapshot struct { } // Snapshot captures the mock's entire state as JSON. includeAssets is unused. MSK holds resource -// metadata, not bulk object bodies. +// metadata, not bulk object bodies. Each record is deep-copied under its own +// lock, so the encoding never reads a map or slice a concurrent writer changes. func (m *Mock) Snapshot(_ context.Context, _ bool) (json.RawMessage, error) { - snap := kafkaSnapshot{} - - if m.clusters.Len() > 0 { - snap.Clusters = make(map[string]*clusterSnapshot, m.clusters.Len()) - - for arn, cd := range m.clusters.All() { - cd.mu.RLock() - snap.Clusters[arn] = &clusterSnapshot{ - Cluster: cd.cluster, Operations: cd.operations, Topics: cd.topics, - ScramSecrets: cd.scramSecrets, Policy: cd.policy, PolicyVersion: cd.policyVersion, - } - cd.mu.RUnlock() - } - } + snap := kafkaSnapshot{Clusters: m.clustersForSnapshot()} names, err := m.clusterNames.Snapshot() if err != nil { @@ -68,38 +68,117 @@ func (m *Mock) Snapshot(_ context.Context, _ bool) (json.RawMessage, error) { } snap.ClusterNames = names + snap.Configs = m.configsForSnapshot() + snap.VPCConns, snap.VPCConnReadyAt = m.vpcConnsForSnapshot() + snap.Replicators = m.replicatorsForSnapshot() - if m.configs.Len() > 0 { - snap.Configs = make(map[string]driver.Configuration, m.configs.Len()) + return json.Marshal(snap) +} - for arn, cd := range m.configs.All() { - cd.mu.RLock() - snap.Configs[arn] = cd.config - cd.mu.RUnlock() - } +// clustersForSnapshot deep-copies every cluster record, or returns nil when +// there are none. +func (m *Mock) clustersForSnapshot() map[string]*clusterSnapshot { + if m.clusters.Len() == 0 { + return nil } - if m.vpcConns.Len() > 0 { - snap.VPCConns = make(map[string]driver.VpcConnection, m.vpcConns.Len()) + out := make(map[string]*clusterSnapshot, m.clusters.Len()) - for arn, vd := range m.vpcConns.All() { - vd.mu.RLock() - snap.VPCConns[arn] = vd.vpc - vd.mu.RUnlock() + for arn, cd := range m.clusters.All() { + cd.mu.RLock() + out[arn] = cd.snapshotLocked() + cd.mu.RUnlock() + } + + return out +} + +// snapshotLocked deep-copies a cluster record. The caller holds cd.mu. +func (cd *clusterData) snapshotLocked() *clusterSnapshot { + cs := &clusterSnapshot{ + Cluster: snapshotCluster(cd.cluster), + ScramSecrets: copyStrings(cd.scramSecrets), + Policy: cd.policy, PolicyVersion: cd.policyVersion, + ReadyAt: cd.readyAt, DeleteAt: cd.deleteAt, MetricsSampleAt: cd.metricsSampleAt, + } + + if cd.operations != nil { + cs.Operations = make([]driver.ClusterOperation, len(cd.operations)) + for i := range cd.operations { + cs.Operations[i] = copyOperation(cd.operations[i]) + } + } + + if cd.topics != nil { + cs.Topics = make(map[string]driver.Topic, len(cd.topics)) + for name, t := range cd.topics { + cs.Topics[name] = snapshotTopic(t) } } - if m.replicators.Len() > 0 { - snap.Replicators = make(map[string]*replicatorSnapshot, m.replicators.Len()) + return cs +} + +// configsForSnapshot deep-copies every configuration, or returns nil when there +// are none. +func (m *Mock) configsForSnapshot() map[string]driver.Configuration { + if m.configs.Len() == 0 { + return nil + } + + out := make(map[string]driver.Configuration, m.configs.Len()) - for arn, rd := range m.replicators.All() { - rd.mu.RLock() - snap.Replicators[arn] = &replicatorSnapshot{Replicator: rd.replicator, Version: rd.version} - rd.mu.RUnlock() + for arn, cd := range m.configs.All() { + cd.mu.RLock() + out[arn] = snapshotConfig(cd.config) + cd.mu.RUnlock() + } + + return out +} + +// vpcConnsForSnapshot deep-copies every VPC connection, plus the CREATING +// deadline of each one still inside its window (nil when none is). +func (m *Mock) vpcConnsForSnapshot() (conns map[string]driver.VpcConnection, readyAt map[string]time.Time) { + if m.vpcConns.Len() == 0 { + return nil, nil + } + + conns = make(map[string]driver.VpcConnection, m.vpcConns.Len()) + + for arn, vd := range m.vpcConns.All() { + vd.mu.RLock() + conns[arn] = snapshotVpcConnection(vd.vpc) + + if !vd.readyAt.IsZero() { + if readyAt == nil { + readyAt = map[string]time.Time{} + } + + readyAt[arn] = vd.readyAt } + vd.mu.RUnlock() } - return json.Marshal(snap) + return conns, readyAt +} + +// replicatorsForSnapshot deep-copies every replicator, or returns nil when +// there are none. +func (m *Mock) replicatorsForSnapshot() map[string]*replicatorSnapshot { + if m.replicators.Len() == 0 { + return nil + } + + out := make(map[string]*replicatorSnapshot, m.replicators.Len()) + + for arn, rd := range m.replicators.All() { + rd.mu.RLock() + out[arn] = &replicatorSnapshot{Replicator: snapshotReplicator(rd.replicator), Version: rd.version} + rd.mu.RUnlock() + } + + return out } // Restore rebuilds the mock's state under the original identities: every cluster, @@ -112,9 +191,16 @@ func (m *Mock) Restore(_ context.Context, data json.RawMessage) error { } for arn, cs := range snap.Clusters { + // A snapshot from before the provider completed encryptionInfo at create + // lacks it; backfill MSK's default so the restored cluster reports it. + if cs.Cluster.ClusterType == driver.ClusterTypeProvisioned { + cs.Cluster.RawOptions = m.withDefaultEncryption(cs.Cluster.RawOptions) + } + cd := &clusterData{ cluster: cs.Cluster, operations: cs.Operations, topics: cs.Topics, scramSecrets: cs.ScramSecrets, policy: cs.Policy, policyVersion: cs.PolicyVersion, + readyAt: cs.ReadyAt, deleteAt: cs.DeleteAt, metricsSampleAt: cs.MetricsSampleAt, } m.clusters.Set(arn, cd) @@ -137,7 +223,7 @@ func (m *Mock) Restore(_ context.Context, data json.RawMessage) error { } for arn, vpc := range snap.VPCConns { - m.vpcConns.Set(arn, &vpcConnData{vpc: vpc}) + m.vpcConns.Set(arn, &vpcConnData{vpc: vpc, readyAt: snap.VPCConnReadyAt[arn]}) } for arn, rs := range snap.Replicators { diff --git a/providers/aws/kafka/topics.go b/providers/aws/kafka/topics.go index c6f767089..f4208523d 100644 --- a/providers/aws/kafka/topics.go +++ b/providers/aws/kafka/topics.go @@ -62,6 +62,10 @@ func (m *Mock) CreateTopic(_ context.Context, clusterARN string, body json.RawMe cd.mu.Lock() defer cd.mu.Unlock() + if err := m.requireActive(cd); err != nil { + return nil, err + } + if cd.topics == nil { cd.topics = map[string]driver.Topic{} } @@ -168,6 +172,10 @@ func (m *Mock) UpdateTopic( cd.mu.Lock() defer cd.mu.Unlock() + if err := m.requireActive(cd); err != nil { + return nil, err + } + t, ok := cd.getTopic(topicName) if !ok { return nil, notFound("topic not found: %s", topicName) @@ -198,6 +206,10 @@ func (m *Mock) DeleteTopic(_ context.Context, clusterARN, topicName string) erro cd.mu.Lock() defer cd.mu.Unlock() + if err := m.requireActive(cd); err != nil { + return err + } + if _, ok := cd.getTopic(topicName); !ok { return notFound("topic not found: %s", topicName) } diff --git a/providers/aws/kafka/versions.go b/providers/aws/kafka/versions.go index 53dcb64f0..5a63196e5 100644 --- a/providers/aws/kafka/versions.go +++ b/providers/aws/kafka/versions.go @@ -92,8 +92,8 @@ func (m *Mock) compatibilitySources(clusterARN string) []string { return all } - cd, ok := m.clusters.Get(clusterARN) - if !ok { + cd, err := m.getCluster(clusterARN) + if err != nil { return nil } diff --git a/providers/aws/kafka/vpc_connections.go b/providers/aws/kafka/vpc_connections.go index ba3af342a..cbb1a0e36 100644 --- a/providers/aws/kafka/vpc_connections.go +++ b/providers/aws/kafka/vpc_connections.go @@ -38,7 +38,8 @@ type createVpcConnectionRequest struct { // CreateVpcConnection mints a VPC connection in the AVAILABLE state, carrying // the client subnets and security groups verbatim in raw options so a later -// Describe reflects them. +// Describe reflects them. Under AsyncSettle it reports CREATING for a settle +// window first (real VpcConnectionState has CREATING and no PENDING value). func (m *Mock) CreateVpcConnection(_ context.Context, body json.RawMessage) (*driver.VpcConnection, error) { var req createVpcConnectionRequest if len(body) > 0 { @@ -79,11 +80,14 @@ func (m *Mock) CreateVpcConnection(_ context.Context, body json.RawMessage) (*dr // The ARN embeds a fresh UUID; SetIfAbsent keeps the create atomic and guards // the (impossible-but-cheap) collision. - if !m.vpcConns.SetIfAbsent(vpc.VpcConnectionARN, &vpcConnData{vpc: vpc}) { + vd := &vpcConnData{vpc: vpc, readyAt: m.settleUntil(vpcConnCreateSettle)} + if !m.vpcConns.SetIfAbsent(vpc.VpcConnectionARN, vd) { return nil, badRequest("vpc connection already exists: %s", vpc.VpcConnectionARN) } - out := snapshotVpcConnection(vpc) + vd.mu.RLock() + out := m.observedVpcConnectionLocked(vd) + vd.mu.RUnlock() return &out, nil } @@ -139,9 +143,8 @@ func (m *Mock) DescribeVpcConnection(_ context.Context, arn string) (*driver.Vpc } vd.mu.RLock() - defer vd.mu.RUnlock() - - out := snapshotVpcConnection(vd.vpc) + out := m.observedVpcConnectionLocked(vd) + vd.mu.RUnlock() return &out, nil } @@ -154,7 +157,7 @@ func (m *Mock) DeleteVpcConnection(_ context.Context, arn string) (*driver.VpcCo } vd.mu.RLock() - out := snapshotVpcConnection(vd.vpc) + out := m.observedVpcConnectionLocked(vd) vd.mu.RUnlock() m.vpcConns.Delete(arn) @@ -204,7 +207,7 @@ func (m *Mock) snapshotVpcConnections(targetCluster string) []driver.VpcConnecti for _, vd := range vals { vd.mu.RLock() - snap := snapshotVpcConnection(vd.vpc) + snap := m.observedVpcConnectionLocked(vd) vd.mu.RUnlock() if targetCluster != "" && snap.TargetClusterARN != targetCluster { diff --git a/providers/aws/tickables_test.go b/providers/aws/tickables_test.go index ab76f1d5e..6f0dffaec 100644 --- a/providers/aws/tickables_test.go +++ b/providers/aws/tickables_test.go @@ -8,17 +8,23 @@ import ( ) // TestTickablesRegistersTimeDrivenServices checks the serve ticker reaches the -// region's own SQS, EFS, CloudWatch, SSM, CloudFormation and Batch, so queue -// gauges, EFS metrics, due alarms, parameter policies, settling stack operations -// and Batch job state-change events run without a read. SQS and EFS come before -// CloudWatch so alarms see the metrics published in the same tick. +// region's own SQS, EFS, Kafka, CloudWatch, SSM, CloudFormation and Batch, so +// queue gauges, EFS and Kafka metrics, due alarms, parameter policies, settling +// stack operations and Batch job state-change events run without a read. SQS, +// EFS and Kafka come before CloudWatch so alarms see the metrics published in +// the same tick. func TestTickablesRegistersTimeDrivenServices(t *testing.T) { p := aws.New() got := p.Tickables() - if len(got) != 6 || got[0] != config.Tickable(p.SQS) || got[1] != config.Tickable(p.EFS) || - got[2] != config.Tickable(p.CloudWatch) || got[3] != config.Tickable(p.SSM) || - got[4] != config.Tickable(p.CloudFormation) || got[5] != config.Tickable(p.Batch) { - t.Fatalf("Tickables() = %v, want this provider's SQS, EFS, CloudWatch, SSM, CloudFormation and Batch", got) + want := []config.Tickable{p.SQS, p.EFS, p.Kafka, p.CloudWatch, p.SSM, p.CloudFormation, p.Batch} + if len(got) != len(want) { + t.Fatalf("Tickables() = %v, want this provider's SQS, EFS, Kafka, CloudWatch, SSM, CloudFormation and Batch", got) + } + + for i := range want { + if got[i] != want[i] { + t.Fatalf("Tickables()[%d] = %v, want %v", i, got[i], want[i]) + } } } diff --git a/server/aws/kafka/parity_sdk_test.go b/server/aws/kafka/parity_sdk_test.go new file mode 100644 index 000000000..f34fc0932 --- /dev/null +++ b/server/aws/kafka/parity_sdk_test.go @@ -0,0 +1,291 @@ +package kafka_test + +import ( + "context" + "errors" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + awsconfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + awscw "github.com/aws/aws-sdk-go-v2/service/cloudwatch" + cwtypes "github.com/aws/aws-sdk-go-v2/service/cloudwatch/types" + awskafka "github.com/aws/aws-sdk-go-v2/service/kafka" + kafkatypes "github.com/aws/aws-sdk-go-v2/service/kafka/types" + + "github.com/stackshy/cloudemu/v2" + cloudconfig "github.com/stackshy/cloudemu/v2/config" + awsprovider "github.com/stackshy/cloudemu/v2/providers/aws" + awsserver "github.com/stackshy/cloudemu/v2/server/aws" +) + +// parityEnv is an in-process AWS server (Kafka + CloudWatch from one provider, +// so the AWS/Kafka metrics land in the CloudWatch the client reads) with real +// SDK clients, the FakeClock driving settle windows, and the provider whose +// Tickables the serve scheduler would run. +type parityEnv struct { + kafka *awskafka.Client + cw *awscw.Client + clock *cloudconfig.FakeClock + cloud *awsprovider.Provider +} + +// tick runs one round of the serve scheduler at the current FakeClock time. +func (e *parityEnv) tick() { + for _, tk := range e.cloud.Tickables() { + tk.Tick(e.clock.Now()) + } +} + +func newParityEnv(t *testing.T, opts ...cloudconfig.Option) *parityEnv { + t.Helper() + + c, cw, fc, cloud := newParityClientsFull(t, opts...) + + return &parityEnv{kafka: c, cw: cw, clock: fc, cloud: cloud} +} + +func newParityClients(t *testing.T, opts ...cloudconfig.Option) (*awskafka.Client, *awscw.Client, *cloudconfig.FakeClock) { + t.Helper() + + c, cw, fc, _ := newParityClientsFull(t, opts...) + + return c, cw, fc +} + +func newParityClientsFull( + t *testing.T, opts ...cloudconfig.Option, +) (*awskafka.Client, *awscw.Client, *cloudconfig.FakeClock, *awsprovider.Provider) { + t.Helper() + + fc := cloudconfig.NewFakeClock(time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)) + cloud := cloudemu.NewAWS(append([]cloudconfig.Option{cloudconfig.WithClock(fc)}, opts...)...) + srv := awsserver.New(awsserver.Drivers{Kafka: cloud.Kafka, CloudWatch: cloud.CloudWatch}) + + ts := httptest.NewServer(srv) + t.Cleanup(ts.Close) + + cfg, err := awsconfig.LoadDefaultConfig(context.Background(), + awsconfig.WithRegion("us-east-1"), + awsconfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider("test", "test", "")), + ) + if err != nil { + t.Fatalf("aws config: %v", err) + } + + cfg.BaseEndpoint = aws.String(ts.URL) + + return awskafka.NewFromConfig(cfg), awscw.NewFromConfig(cfg), fc, cloud +} + +func createSDKCluster(ctx context.Context, t *testing.T, c *awskafka.Client, name string) string { + t.Helper() + + out, err := c.CreateCluster(ctx, &awskafka.CreateClusterInput{ + ClusterName: aws.String(name), + KafkaVersion: aws.String("3.6.0"), + NumberOfBrokerNodes: aws.Int32(3), + BrokerNodeGroupInfo: &kafkatypes.BrokerNodeGroupInfo{ + ClientSubnets: []string{"subnet-0a1", "subnet-0b2", "subnet-0c3"}, + InstanceType: aws.String("kafka.m5.large"), + }, + }) + if err != nil { + t.Fatalf("CreateCluster: %v", err) + } + + return aws.ToString(out.ClusterArn) +} + +// TestSDKListNodesClientSubnet asserts ListNodes reports the cluster's own +// client subnets through the wire. +func TestSDKListNodesClientSubnet(t *testing.T) { + ctx := context.Background() + c, _, _ := newParityClients(t) + arn := createSDKCluster(ctx, t, c, "sdk-subnets") + + out, err := c.ListNodes(ctx, &awskafka.ListNodesInput{ClusterArn: aws.String(arn)}) + if err != nil { + t.Fatalf("ListNodes: %v", err) + } + + want := []string{"subnet-0a1", "subnet-0b2", "subnet-0c3"} + for i, n := range out.NodeInfoList { + if got := aws.ToString(n.BrokerNodeInfo.ClientSubnet); got != want[i] { + t.Fatalf("node %d clientSubnet = %q, want %q", i, got, want[i]) + } + } +} + +// TestSDKAsyncClusterLifecycle drives CREATING -> ACTIVE -> DELETING -> gone +// through the real SDK under AsyncSettle. +func TestSDKAsyncClusterLifecycle(t *testing.T) { + ctx := context.Background() + c, _, fc := newParityClients(t, cloudconfig.WithAsyncSettle()) + arn := createSDKCluster(ctx, t, c, "sdk-async") + + desc, err := c.DescribeCluster(ctx, &awskafka.DescribeClusterInput{ClusterArn: aws.String(arn)}) + if err != nil || desc.ClusterInfo.State != kafkatypes.ClusterStateCreating { + t.Fatalf("describe while creating = %v/%v", desc, err) + } + + _, err = c.UpdateMonitoring(ctx, &awskafka.UpdateMonitoringInput{ + ClusterArn: aws.String(arn), CurrentVersion: desc.ClusterInfo.CurrentVersion, + EnhancedMonitoring: kafkatypes.EnhancedMonitoringPerBroker, + }) + + var br *kafkatypes.BadRequestException + if !errors.As(err, &br) { + t.Fatalf("update while CREATING = %v, want BadRequestException", err) + } + + fc.Advance(10 * time.Second) + + v2, err := c.DescribeClusterV2(ctx, &awskafka.DescribeClusterV2Input{ClusterArn: aws.String(arn)}) + if err != nil || v2.ClusterInfo.State != kafkatypes.ClusterStateActive { + t.Fatalf("describe v2 after settle = %v/%v", v2, err) + } + + del, err := c.DeleteCluster(ctx, &awskafka.DeleteClusterInput{ClusterArn: aws.String(arn)}) + if err != nil || del.State != kafkatypes.ClusterStateDeleting { + t.Fatalf("DeleteCluster = %v/%v", del, err) + } + + desc, err = c.DescribeCluster(ctx, &awskafka.DescribeClusterInput{ClusterArn: aws.String(arn)}) + if err != nil || desc.ClusterInfo.State != kafkatypes.ClusterStateDeleting { + t.Fatalf("describe while deleting = %v/%v", desc, err) + } + + fc.Advance(10 * time.Second) + + _, err = c.DescribeCluster(ctx, &awskafka.DescribeClusterInput{ClusterArn: aws.String(arn)}) + + var nf *kafkatypes.NotFoundException + if !errors.As(err, &nf) { + t.Fatalf("describe after delete = %v, want NotFoundException", err) + } +} + +// TestSDKAsyncVpcConnectionCreating asserts a VPC connection reports CREATING +// then AVAILABLE through the wire under AsyncSettle. +func TestSDKAsyncVpcConnectionCreating(t *testing.T) { + ctx := context.Background() + c, _, fc := newParityClients(t, cloudconfig.WithAsyncSettle()) + arn := createSDKCluster(ctx, t, c, "sdk-vpc") + + fc.Advance(10 * time.Second) + + v, err := c.CreateVpcConnection(ctx, &awskafka.CreateVpcConnectionInput{ + TargetClusterArn: aws.String(arn), Authentication: aws.String("SASL_IAM"), + VpcId: aws.String("vpc-123"), ClientSubnets: []string{"subnet-x"}, SecurityGroups: []string{"sg-1"}, + }) + if err != nil || v.State != kafkatypes.VpcConnectionStateCreating { + t.Fatalf("CreateVpcConnection = %v/%v, want CREATING", v, err) + } + + fc.Advance(10 * time.Second) + + d, err := c.DescribeVpcConnection(ctx, &awskafka.DescribeVpcConnectionInput{Arn: v.VpcConnectionArn}) + if err != nil || d.State != kafkatypes.VpcConnectionStateAvailable { + t.Fatalf("DescribeVpcConnection = %v/%v, want AVAILABLE", d, err) + } +} + +// TestSDKKafkaMetricsAndDefaultKey asserts the AWS/Kafka metrics are published +// by the serve scheduler (none on create, one sample per minute after), +// readable through the CloudWatch API under the documented dimensions, and the +// default at-rest key is a real-shaped KMS key ARN in the v1 and v2 shapes. +func TestSDKKafkaMetricsAndDefaultKey(t *testing.T) { + ctx := context.Background() + env := newParityEnv(t) + c, cw := env.kafka, env.cw + start := env.clock.Now() + arn := createSDKCluster(ctx, t, c, "sdk-metrics") + + listed := func() []cwtypes.Metric { + out, err := cw.ListMetrics(ctx, &awscw.ListMetricsInput{Namespace: aws.String("AWS/Kafka")}) + if err != nil { + t.Fatalf("ListMetrics: %v", err) + } + + return out.Metrics + } + + if got := listed(); len(got) != 0 { + t.Fatalf("metrics published by a request: %+v", got) + } + + env.tick() + env.clock.Advance(time.Minute) + env.tick() + + var sawController, sawBroker bool + + for _, m := range listed() { + dims := map[string]string{} + for _, d := range m.Dimensions { + dims[aws.ToString(d.Name)] = aws.ToString(d.Value) + } + + if dims["Cluster Name"] != "sdk-metrics" { + continue + } + + switch aws.ToString(m.MetricName) { + case "ActiveControllerCount": + sawController = len(dims) == 1 + case "UnderReplicatedPartitions": + sawBroker = sawBroker || dims["Broker ID"] != "" + } + } + + if !sawController || !sawBroker { + t.Fatalf("AWS/Kafka metrics missing (controller=%v broker=%v)", sawController, sawBroker) + } + + stats, err := cw.GetMetricStatistics(ctx, &awscw.GetMetricStatisticsInput{ + Namespace: aws.String("AWS/Kafka"), + MetricName: aws.String("ActiveControllerCount"), + Dimensions: []cwtypes.Dimension{{Name: aws.String("Cluster Name"), Value: aws.String("sdk-metrics")}}, + StartTime: aws.Time(start), + EndTime: aws.Time(env.clock.Now().Add(time.Minute)), + Period: aws.Int32(60), + Statistics: []cwtypes.Statistic{cwtypes.StatisticSampleCount, cwtypes.StatisticMaximum}, + }) + if err != nil { + t.Fatalf("GetMetricStatistics: %v", err) + } + + if len(stats.Datapoints) != 2 { + t.Fatalf("ActiveControllerCount datapoints = %+v, want 2 one-minute samples", stats.Datapoints) + } + + for _, dp := range stats.Datapoints { + if aws.ToFloat64(dp.SampleCount) != 1 || aws.ToFloat64(dp.Maximum) != 1 || dp.Unit != cwtypes.StandardUnitCount { + t.Fatalf("datapoint = %+v, want one sample of 1 Count", dp) + } + } + + desc, err := c.DescribeCluster(ctx, &awskafka.DescribeClusterInput{ClusterArn: aws.String(arn)}) + if err != nil { + t.Fatalf("DescribeCluster: %v", err) + } + + key := aws.ToString(desc.ClusterInfo.EncryptionInfo.EncryptionAtRest.DataVolumeKMSKeyId) + if !strings.HasPrefix(key, "arn:aws:kms:us-east-1:") || strings.Contains(key, "msk-default") { + t.Fatalf("default key = %q", key) + } + + v2, err := c.DescribeClusterV2(ctx, &awskafka.DescribeClusterV2Input{ClusterArn: aws.String(arn)}) + if err != nil { + t.Fatalf("DescribeClusterV2: %v", err) + } + + enc := v2.ClusterInfo.Provisioned.EncryptionInfo + if enc == nil || aws.ToString(enc.EncryptionAtRest.DataVolumeKMSKeyId) != key { + t.Fatalf("v2 provisioned encryptionInfo = %+v, want key %s", enc, key) + } +} diff --git a/server/aws/kafka/types.go b/server/aws/kafka/types.go index ce35abdcc..651477db4 100644 --- a/server/aws/kafka/types.go +++ b/server/aws/kafka/types.go @@ -136,10 +136,8 @@ func clusterToWire(c *driver.Cluster) map[string]json.RawMessage { base["brokerNodeGroupInfo"] = bngToWire(c.BrokerNodeGroupInfo) } - if _, ok := c.RawOptions["encryptionInfo"]; !ok { - base["encryptionInfo"] = defaultEncryptionInfo() - } - + // encryptionInfo (including MSK's default at-rest key) is completed by the + // provider at create time and carried in RawOptions; mergeRaw surfaces it. addZookeeperConnect(base, c) return mergeRaw(base, c.RawOptions) @@ -191,22 +189,6 @@ func regionFromARN(arn string) string { return parts[regionIndex] } -// defaultEncryptionInfo returns the MSK default encryption configuration that -// real DescribeCluster surfaces when none is specified at create time: -// in-transit TLS (client-broker + in-cluster) and at-rest with an -// AWS-managed data-volume KMS key. -func defaultEncryptionInfo() map[string]any { - return map[string]any{ - "encryptionAtRest": map[string]any{ - "dataVolumeKMSKeyId": "arn:aws:kms:us-east-1:000000000000:key/msk-default", - }, - "encryptionInTransit": map[string]any{ - "clientBroker": "TLS", - "inCluster": true, - }, - } -} - // enhancedMonitoringOrDefault returns the cluster's EnhancedMonitoring level, // falling back to MSK's DEFAULT. Real DescribeCluster always surfaces this enum // (never omits it), so a cluster created without one still reports "DEFAULT". @@ -305,9 +287,9 @@ func provisionedBlock(c *driver.Cluster) map[string]any { block["brokerNodeGroupInfo"] = bngToWire(c.BrokerNodeGroupInfo) } - if _, ok := c.RawOptions["encryptionInfo"]; !ok { - block["encryptionInfo"] = defaultEncryptionInfo() - } + // The provider stores the cluster's (defaulted) encryptionInfo in RawOptions; + // v2 reports it inside the provisioned block. + overlayRaw(block, c.RawOptions, "encryptionInfo", "encryptionInfo") addZookeeperConnect(block, c)