diff --git a/CHANGELOG.md b/CHANGELOG.md index 694e3a5..2283c4d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,11 +21,15 @@ Version format: `-` ### Changed +- **Sensor version** - `VERSION` is now `29.3.0`; the next release is tagged `v29.3.0-1`. + - **aws/sensor, aws/sensor-single, azure/sensor, azure/sensor-single, gcp/sensor** - Stop provisioning `cloud_provider` and `cloud_region`; `corelightctl` discovers them through the CLOUD-1534 instance metadata probe. Module identity, release-stamped module version, and known traffic-mirroring metadata remain unchanged. Existing provisioning region/provider settings are unaffected. - Direct shared-config consumers must remove the retired `deployment_cloud_provider` and `deployment_cloud_region` arguments. - Older modules with updated `corelightctl` use the probe. These modules with older `corelightctl` may report unknown/null provider and region. - Merge/publication depends on delivery of [CLOUD-1534](https://corelight.atlassian.net/browse/CLOUD-1534) / [Elysium !7563](https://gitlab.com/corelight/engineering/elysium/-/merge_requests/7563); compatible released sensor versions and acceptance from a new published Terraform tag remain pending. +- **aws/sensor, aws/sensor-single, azure/sensor, azure/sensor-single, gcp/sensor** - Deployment metadata (module identity, module version, traffic mirroring) is now written as a `corelightctl` Helm values drop-in, `/etc/corelight/corelightctl-values.yaml.d/10-deployment-metadata.yaml`, and applied by the single `corelightctl sensor deploy`. The post-deploy `corelightctl sensor configuration put` and `/etc/corelight/deployment-metadata.yaml` are removed; Fleet-paired sensors rejected that call, so the metadata was never applied. Sensor versions that predate drop-in support for these fields ignore the file and report the metadata as null. + - **examples/aws/flow-sensor** - Rewritten as a single-sensor example with an inline IAM role/instance profile, replacing the previous module-based deployment ### Removed diff --git a/CLAUDE.md b/CLAUDE.md index daad1ef..8c24437 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -5,7 +5,7 @@ This document provides a comprehensive guide for AI assistants and developers wo ## Repository Overview **Repository**: `github.com/corelight/terraform` -**Current Sensor**: `29.0.5` (see `VERSION`; meta is tag-derived) +**Current Sensor**: `29.3.0` (see `VERSION`; meta is tag-derived) **Purpose**: Consolidated Terraform modules for deploying Corelight sensors, enrichment, and Fleet across AWS, GCP, and Azure. ## Core Principles diff --git a/VERSION b/VERSION index c67164e..fffd516 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -29.0.5 +29.3.0 diff --git a/modules/_shared/config/sensor/data.tf b/modules/_shared/config/sensor/data.tf index 6379c5a..84a3763 100644 --- a/modules/_shared/config/sensor/data.tf +++ b/modules/_shared/config/sensor/data.tf @@ -11,16 +11,19 @@ locals { ) deployment_metadata = merge( var.deployment_traffic_mirroring_enabled == null ? {} : { - "deployment_metadata.cloud_traffic_mirroring_enabled" = tostring(var.deployment_traffic_mirroring_enabled) + cloudTrafficMirroringEnabled = tostring(var.deployment_traffic_mirroring_enabled) }, local.terraform_module_version == null ? {} : { - "deployment_metadata.terraform_module_version" = local.terraform_module_version + terraformModuleVersion = local.terraform_module_version }, var.terraform_module == null ? {} : { - "deployment_metadata.terraform_module" = var.terraform_module + terraformModule = var.terraform_module }, ) - deployment_metadata_yaml = length(local.deployment_metadata) == 0 ? "" : yamlencode(local.deployment_metadata) + # Helm values drop-in read by corelightctl sensor deploy. + deployment_metadata_yaml = length(local.deployment_metadata) == 0 ? "" : yamlencode({ + "sensor-api" = { deploymentMetadata = local.deployment_metadata } + }) } data "cloudinit_config" "config" { diff --git a/modules/_shared/config/sensor/templates/cloud-init.yaml.tpl b/modules/_shared/config/sensor/templates/cloud-init.yaml.tpl index 5b628cb..4aa6dd7 100644 --- a/modules/_shared/config/sensor/templates/cloud-init.yaml.tpl +++ b/modules/_shared/config/sensor/templates/cloud-init.yaml.tpl @@ -56,24 +56,14 @@ write_files: %{ endif ~} %{ if deployment_metadata != "" ~} - owner: root:root - path: /etc/corelight/deployment-metadata.yaml + path: /etc/corelight/corelightctl-values.yaml.d/10-deployment-metadata.yaml permissions: '0644' encoding: b64 content: ${deployment_metadata} %{ endif ~} runcmd: -%{ if deployment_metadata != "" ~} - - | - if ! corelightctl sensor deploy -v; then - exit 1 - fi - if ! corelightctl sensor configuration put --file /etc/corelight/deployment-metadata.yaml; then - logger -t corelight-deployment-metadata "Sensor API did not accept deployment metadata; inventory fields will remain null" - fi -%{ else ~} - corelightctl sensor deploy -v -%{ endif ~} %{ if azure_fips_enabled ~} - | timeout=120 diff --git a/modules/_shared/config/sensor/tests/module.tftest.hcl b/modules/_shared/config/sensor/tests/module.tftest.hcl index 498b146..6fc9cdb 100644 --- a/modules/_shared/config/sensor/tests/module.tftest.hcl +++ b/modules/_shared/config/sensor/tests/module.tftest.hcl @@ -126,7 +126,7 @@ run "verify_deployment_metadata" { assert { condition = strcontains( data.cloudinit_config.config.part[0].content, - "path: /etc/corelight/deployment-metadata.yaml", + "path: /etc/corelight/corelightctl-values.yaml.d/10-deployment-metadata.yaml", ) error_message = "Cloud-init should persist deployment metadata" } @@ -135,11 +135,11 @@ run "verify_deployment_metadata" { condition = try( yamldecode(base64decode(one([ for file in yamldecode(data.cloudinit_config.config.part[0].content).write_files : file.content - if file.path == "/etc/corelight/deployment-metadata.yaml" && try(file.encoding, "") == "b64" - ]))) == { - "deployment_metadata.cloud_traffic_mirroring_enabled" = "true" - "deployment_metadata.terraform_module_version" = "v29.0.5-7" - "deployment_metadata.terraform_module" = "aws/sensor" + if file.path == "/etc/corelight/corelightctl-values.yaml.d/10-deployment-metadata.yaml" && try(file.encoding, "") == "b64" + ])))["sensor-api"].deploymentMetadata == { + cloudTrafficMirroringEnabled = "true" + terraformModuleVersion = "v29.0.5-7" + terraformModule = "aws/sensor" }, false, ) @@ -147,11 +147,8 @@ run "verify_deployment_metadata" { } assert { - condition = strcontains( - data.cloudinit_config.config.part[0].content, - "corelightctl sensor configuration put --file /etc/corelight/deployment-metadata.yaml", - ) - error_message = "Cloud-init should apply metadata through the existing configuration command" + condition = !strcontains(data.cloudinit_config.config.part[0].content, "configuration put") + error_message = "Deploy must seed metadata from the values drop-in; Fleet-paired sensors reject configuration put" } } @@ -171,9 +168,9 @@ run "verify_unknown_mirroring_is_omitted" { condition = try( yamldecode(base64decode(one([ for file in yamldecode(data.cloudinit_config.config.part[0].content).write_files : file.content - if file.path == "/etc/corelight/deployment-metadata.yaml" && try(file.encoding, "") == "b64" - ]))) == { - "deployment_metadata.terraform_module" = "azure/sensor" + if file.path == "/etc/corelight/corelightctl-values.yaml.d/10-deployment-metadata.yaml" && try(file.encoding, "") == "b64" + ])))["sensor-api"].deploymentMetadata == { + terraformModule = "azure/sensor" }, false, ) @@ -212,9 +209,9 @@ run "verify_malformed_terraform_module_version_is_omitted" { condition = try( yamldecode(base64decode(one([ for file in yamldecode(data.cloudinit_config.config.part[0].content).write_files : file.content - if file.path == "/etc/corelight/deployment-metadata.yaml" && try(file.encoding, "") == "b64" - ]))) == { - "deployment_metadata.terraform_module" = "gcp/sensor" + if file.path == "/etc/corelight/corelightctl-values.yaml.d/10-deployment-metadata.yaml" && try(file.encoding, "") == "b64" + ])))["sensor-api"].deploymentMetadata == { + terraformModule = "gcp/sensor" }, false, ) @@ -239,9 +236,9 @@ run "verify_overlong_terraform_module_version_is_omitted" { condition = try( yamldecode(base64decode(one([ for file in yamldecode(data.cloudinit_config.config.part[0].content).write_files : file.content - if file.path == "/etc/corelight/deployment-metadata.yaml" && try(file.encoding, "") == "b64" - ]))) == { - "deployment_metadata.terraform_module" = "gcp/sensor" + if file.path == "/etc/corelight/corelightctl-values.yaml.d/10-deployment-metadata.yaml" && try(file.encoding, "") == "b64" + ])))["sensor-api"].deploymentMetadata == { + terraformModule = "gcp/sensor" }, false, ) @@ -266,9 +263,9 @@ run "verify_false_mirroring" { condition = try( yamldecode(base64decode(one([ for file in yamldecode(data.cloudinit_config.config.part[0].content).write_files : file.content - if file.path == "/etc/corelight/deployment-metadata.yaml" && try(file.encoding, "") == "b64" - ]))) == { - "deployment_metadata.cloud_traffic_mirroring_enabled" = "false" + if file.path == "/etc/corelight/corelightctl-values.yaml.d/10-deployment-metadata.yaml" && try(file.encoding, "") == "b64" + ])))["sensor-api"].deploymentMetadata == { + cloudTrafficMirroringEnabled = "false" }, false, ) @@ -291,13 +288,13 @@ run "verify_absent_deployment_metadata" { assert { condition = !strcontains( data.cloudinit_config.config.part[0].content, - "/etc/corelight/deployment-metadata.yaml", + "/etc/corelight/corelightctl-values.yaml.d/10-deployment-metadata.yaml", ) error_message = "Direct callers without metadata should retain the old cloud-init" } } -run "verify_metadata_application_follows_successful_deploy" { +run "verify_metadata_uses_single_deploy" { command = plan variables { @@ -309,11 +306,10 @@ run "verify_metadata_application_follows_successful_deploy" { } assert { - condition = strcontains( - data.cloudinit_config.config.part[0].content, - " if ! corelightctl sensor deploy -v; then\n exit 1\n fi\n if ! corelightctl sensor configuration put --file /etc/corelight/deployment-metadata.yaml; then\n logger -t corelight-deployment-metadata \"Sensor API did not accept deployment metadata; inventory fields will remain null\"\n fi", - ) - error_message = "Metadata application should follow a successful deploy and remain best-effort" + condition = yamldecode(data.cloudinit_config.config.part[0].content).runcmd == [ + "corelightctl sensor deploy -v", + ] + error_message = "Metadata must be applied by the one sensor deploy" } } run "verify_module_version_without_identity_or_mirroring" { @@ -331,9 +327,9 @@ run "verify_module_version_without_identity_or_mirroring" { condition = try( yamldecode(base64decode(one([ for file in yamldecode(data.cloudinit_config.config.part[0].content).write_files : file.content - if file.path == "/etc/corelight/deployment-metadata.yaml" && try(file.encoding, "") == "b64" - ]))) == { - "deployment_metadata.terraform_module_version" = "v29.0.5-7" + if file.path == "/etc/corelight/corelightctl-values.yaml.d/10-deployment-metadata.yaml" && try(file.encoding, "") == "b64" + ])))["sensor-api"].deploymentMetadata == { + terraformModuleVersion = "v29.0.5-7" }, false, ) diff --git a/modules/_shared/config/sensor/tests/paired_metadata.tftest.hcl b/modules/_shared/config/sensor/tests/paired_metadata.tftest.hcl index 51a4507..1da4a20 100644 --- a/modules/_shared/config/sensor/tests/paired_metadata.tftest.hcl +++ b/modules/_shared/config/sensor/tests/paired_metadata.tftest.hcl @@ -41,11 +41,11 @@ run "preserve_fleet_pairing_with_deployment_metadata" { condition = try( yamldecode(base64decode(one([ for file in yamldecode(data.cloudinit_config.config.part[0].content).write_files : file.content - if file.path == "/etc/corelight/deployment-metadata.yaml" && try(file.encoding, "") == "b64" - ]))) == { - "deployment_metadata.cloud_traffic_mirroring_enabled" = "true" - "deployment_metadata.terraform_module_version" = "v29.0.5-7" - "deployment_metadata.terraform_module" = "aws/sensor" + if file.path == "/etc/corelight/corelightctl-values.yaml.d/10-deployment-metadata.yaml" && try(file.encoding, "") == "b64" + ])))["sensor-api"].deploymentMetadata == { + cloudTrafficMirroringEnabled = "true" + terraformModuleVersion = "v29.0.5-7" + terraformModule = "aws/sensor" }, false, ) diff --git a/modules/_shared/config/sensor/tests/published_metadata.tftest.hcl b/modules/_shared/config/sensor/tests/published_metadata.tftest.hcl index 76653bc..073e325 100644 --- a/modules/_shared/config/sensor/tests/published_metadata.tftest.hcl +++ b/modules/_shared/config/sensor/tests/published_metadata.tftest.hcl @@ -16,11 +16,11 @@ run "verify_release_stamped_module_version" { condition = try( yamldecode(base64decode(one([ for file in yamldecode(data.cloudinit_config.config.part[0].content).write_files : file.content - if file.path == "/etc/corelight/deployment-metadata.yaml" && try(file.encoding, "") == "b64" - ]))) == merge( - { "deployment_metadata.terraform_module" = "aws/sensor" }, + if file.path == "/etc/corelight/corelightctl-values.yaml.d/10-deployment-metadata.yaml" && try(file.encoding, "") == "b64" + ])))["sensor-api"].deploymentMetadata == merge( + { terraformModule = "aws/sensor" }, chomp(file("${path.module}/../../../../RELEASE_VERSION")) == "" ? {} : { - "deployment_metadata.terraform_module_version" = chomp(file("${path.module}/../../../../RELEASE_VERSION")) + terraformModuleVersion = chomp(file("${path.module}/../../../../RELEASE_VERSION")) }, ), false, diff --git a/modules/aws/sensor-single/tests/module.tftest.hcl b/modules/aws/sensor-single/tests/module.tftest.hcl index c37b2c4..7c2ab62 100644 --- a/modules/aws/sensor-single/tests/module.tftest.hcl +++ b/modules/aws/sensor-single/tests/module.tftest.hcl @@ -45,9 +45,9 @@ run "test_minimal_configuration" { condition = try( yamldecode(base64decode(one([ for file in yamldecode(module.config[0].cloudinit_config.part[0].content).write_files : file.content - if file.path == "/etc/corelight/deployment-metadata.yaml" && try(file.encoding, "") == "b64" - ]))) == { - "deployment_metadata.terraform_module" = "aws/sensor-single" + if file.path == "/etc/corelight/corelightctl-values.yaml.d/10-deployment-metadata.yaml" && try(file.encoding, "") == "b64" + ])))["sensor-api"].deploymentMetadata == { + terraformModule = "aws/sensor-single" }, false, ) diff --git a/modules/aws/sensor/tests/module.tftest.hcl b/modules/aws/sensor/tests/module.tftest.hcl index b158dcf..f222ee8 100644 --- a/modules/aws/sensor/tests/module.tftest.hcl +++ b/modules/aws/sensor/tests/module.tftest.hcl @@ -58,9 +58,9 @@ run "test_minimal_configuration" { condition = try( yamldecode(base64decode(one([ for file in yamldecode(module.sensor_config.cloudinit_config.part[0].content).write_files : file.content - if file.path == "/etc/corelight/deployment-metadata.yaml" && try(file.encoding, "") == "b64" - ]))) == { - "deployment_metadata.terraform_module" = "aws/sensor" + if file.path == "/etc/corelight/corelightctl-values.yaml.d/10-deployment-metadata.yaml" && try(file.encoding, "") == "b64" + ])))["sensor-api"].deploymentMetadata == { + terraformModule = "aws/sensor" }, false, ) diff --git a/modules/azure/sensor-single/tests/module.tftest.hcl b/modules/azure/sensor-single/tests/module.tftest.hcl index fe5bfb6..64b12a6 100644 --- a/modules/azure/sensor-single/tests/module.tftest.hcl +++ b/modules/azure/sensor-single/tests/module.tftest.hcl @@ -21,9 +21,9 @@ run "verify_terraform_module_identity" { condition = try( yamldecode(base64decode(one([ for file in yamldecode(module.sensor_config[0].cloudinit_config.part[0].content).write_files : file.content - if file.path == "/etc/corelight/deployment-metadata.yaml" && try(file.encoding, "") == "b64" - ]))) == { - "deployment_metadata.terraform_module" = "azure/sensor-single" + if file.path == "/etc/corelight/corelightctl-values.yaml.d/10-deployment-metadata.yaml" && try(file.encoding, "") == "b64" + ])))["sensor-api"].deploymentMetadata == { + terraformModule = "azure/sensor-single" }, false, ) diff --git a/modules/azure/sensor/tests/module.tftest.hcl b/modules/azure/sensor/tests/module.tftest.hcl index d5377f3..37ba8ed 100644 --- a/modules/azure/sensor/tests/module.tftest.hcl +++ b/modules/azure/sensor/tests/module.tftest.hcl @@ -37,9 +37,9 @@ run "test_minimal_configuration" { condition = try( yamldecode(base64decode(one([ for file in yamldecode(module.sensor_config.cloudinit_config.part[0].content).write_files : file.content - if file.path == "/etc/corelight/deployment-metadata.yaml" && try(file.encoding, "") == "b64" - ]))) == { - "deployment_metadata.terraform_module" = "azure/sensor" + if file.path == "/etc/corelight/corelightctl-values.yaml.d/10-deployment-metadata.yaml" && try(file.encoding, "") == "b64" + ])))["sensor-api"].deploymentMetadata == { + terraformModule = "azure/sensor" }, false, ) diff --git a/modules/gcp/sensor/tests/module.tftest.hcl b/modules/gcp/sensor/tests/module.tftest.hcl index b8b5c05..bed3bb5 100644 --- a/modules/gcp/sensor/tests/module.tftest.hcl +++ b/modules/gcp/sensor/tests/module.tftest.hcl @@ -49,10 +49,10 @@ run "verify_required_variables" { condition = try( yamldecode(base64decode(one([ for file in yamldecode(module.sensor_config.cloudinit_config.part[0].content).write_files : file.content - if file.path == "/etc/corelight/deployment-metadata.yaml" && try(file.encoding, "") == "b64" - ]))) == { - "deployment_metadata.terraform_module" = "gcp/sensor" - "deployment_metadata.cloud_traffic_mirroring_enabled" = "true" + if file.path == "/etc/corelight/corelightctl-values.yaml.d/10-deployment-metadata.yaml" && try(file.encoding, "") == "b64" + ])))["sensor-api"].deploymentMetadata == { + terraformModule = "gcp/sensor" + cloudTrafficMirroringEnabled = "true" }, false, )