From c5d7000e22a702f44a2d5791a1f838c4cd75a0f0 Mon Sep 17 00:00:00 2001 From: Zachary Bailey Date: Fri, 18 Sep 2026 12:38:54 -0400 Subject: [PATCH 01/14] fix --- parts/linux/cloud-init/artifacts/cse_cmd.sh | 8 + .../linux/cloud-init/artifacts/cse_helpers.sh | 1 + .../cloud-init/artifacts/init-aks-cloud.sh | 235 +++++++++++++----- pkg/agent/baker_test.go | 6 + .../artifacts/init_aks_cloud_spec.sh | 192 ++++++++++++++ 5 files changed, 376 insertions(+), 66 deletions(-) diff --git a/parts/linux/cloud-init/artifacts/cse_cmd.sh b/parts/linux/cloud-init/artifacts/cse_cmd.sh index 2079e1f1664..69ef46cd623 100644 --- a/parts/linux/cloud-init/artifacts/cse_cmd.sh +++ b/parts/linux/cloud-init/artifacts/cse_cmd.sh @@ -19,6 +19,14 @@ fi; INIT_AKS_CLOUD_FILEPATH="{{GetInitAKSCloudFilepath}}"; if [ -f "${INIT_AKS_CLOUD_FILEPATH}" ]; then REPO_DEPOT_ENDPOINT="{{AKSCustomCloudRepoDepotEndpoint}}" LOCATION={{GetVariable "location"}} "${INIT_AKS_CLOUD_FILEPATH}" >> /var/log/azure/cluster-provision.log 2>&1; + initAKSCloudExitCode=$?; + if [ "$initAKSCloudExitCode" -eq 50 ]; then + echo "NTP not reachable; init-aks-cloud failed with outbound connectivity error code ${initAKSCloudExitCode}" >> ${PROVISION_OUTPUT}; + exit ${initAKSCloudExitCode}; + elif [ "$initAKSCloudExitCode" -eq 244 ]; then + echo "Unable to determine confidential VM platform; init-aks-cloud failed with exit code ${initAKSCloudExitCode}" >> ${PROVISION_OUTPUT}; + exit ${initAKSCloudExitCode}; + fi; fi; {{/* Keep the environment assignments below contiguous through the nohup invocation at the end of this file. */ -}} {{/* The CSE command is flattened into one shell command, so all assignments below are passed to nohup. */ -}} diff --git a/parts/linux/cloud-init/artifacts/cse_helpers.sh b/parts/linux/cloud-init/artifacts/cse_helpers.sh index 8862606a704..60fd4dbbeab 100755 --- a/parts/linux/cloud-init/artifacts/cse_helpers.sh +++ b/parts/linux/cloud-init/artifacts/cse_helpers.sh @@ -162,6 +162,7 @@ ERR_AKS_NODE_CONTROLLER_ERROR=240 # Generic error in AKS Node Controller ERR_AZNFS_RPM_DOWNLOAD_TIMEOUT=241 # Timeout downloading aznfs RPM from PMC ERR_AZNFS_INSTALL_FAIL=242 # Failed to install aznfs RPM package ERR_SECONDARY_NIC_CONFIG_FAIL=243 # Error configuring secondary NIC network interface +ERR_CVM_PLATFORM_DETECTION_FAIL=244 # Unable to distinguish SEV-SNP from TDX # ----------------------------------------------------------------------------- # This probably wasn't launched via a login shell, so ensure the PATH is correct. diff --git a/parts/linux/cloud-init/artifacts/init-aks-cloud.sh b/parts/linux/cloud-init/artifacts/init-aks-cloud.sh index e20b28c055a..0927d338868 100644 --- a/parts/linux/cloud-init/artifacts/init-aks-cloud.sh +++ b/parts/linux/cloud-init/artifacts/init-aks-cloud.sh @@ -81,6 +81,9 @@ IS_ACL=0 IS_MARINER=0 IS_AZURELINUX=0 +ERR_OUTBOUND_CONN_FAIL=50 # Unable to establish outbound connection +ERR_CVM_PLATFORM_DETECTION_FAIL=244 # Unable to distinguish SEV-SNP from TDX + # http://168.63.129.16 is a constant for the host's wireserver endpoint. WIRESERVER_ENDPOINT="http://168.63.129.16" @@ -541,10 +544,164 @@ function determine_cert_endpoint_mode { echo "$mode" } +function is_ubuntu_2604_cvm { + [ "$IS_UBUNTU" -eq 1 ] || return 1 + [ "${VERSION_ID:-}" = "26.04" ] || return 1 + + case "$(uname -r)" in + *-azure-fde*) return 0 ;; + *) return 1 ;; + esac +} + +function detect_confidential_vm_platform { + local platform + + if ! platform="$(systemd-detect-virt --cvm 2>/dev/null)"; then + return 1 + fi + + case "$platform" in + sev-snp|tdx) + echo "$platform" + ;; + *) + return 1 + ;; + esac +} + +function ubuntu_ntp_pools { + cat <<'EOF' +pool ntp.ubuntu.com iburst maxsources 4 +pool 0.ubuntu.pool.ntp.org iburst maxsources 1 +pool 1.ubuntu.pool.ntp.org iburst maxsources 1 +pool 2.ubuntu.pool.ntp.org iburst maxsources 2 +EOF +} + +function configure_chrony { + local time_sources="${1:-refclock PHC /dev/ptp0 poll 3 dpoll -2 offset 0}" + local chrony_conf="${CHRONY_CONF:-/etc/chrony/chrony.conf}" + + if [ "$IS_UBUNTU" -eq 1 ]; then + systemctl stop systemd-timesyncd + systemctl disable systemd-timesyncd + + if [ ! -e "$chrony_conf" ]; then + apt-get update + apt-get install chrony -y + fi + elif [ "$IS_FLATCAR" -eq 1 ]; then + rm -f "$chrony_conf" + fi + + cat > "$chrony_conf" <&2 + emit_event "AKS.CSE.chrony.ntpUnavailable" "NTP not reachable after ${max_attempts} synchronization checks; failing provisioning" "Error" + echo "Chrony source diagnostics:" >&2 + chronyc sources -v >&2 || echo "ERROR: unable to retrieve Chrony source diagnostics" >&2 + echo "Chrony tracking diagnostics:" >&2 + chronyc tracking >&2 || echo "ERROR: unable to retrieve Chrony tracking diagnostics" >&2 + return "$ERR_OUTBOUND_CONN_FAIL" +} + +function configure_ubuntu_2604_cvm_time_sync { + local platform + local ntp_pools + + if ! platform="$(detect_confidential_vm_platform)"; then + echo "ERROR: unable to determine Ubuntu 26.04 CVM platform with systemd-detect-virt --cvm" >&2 + emit_event "AKS.CSE.chrony.platformDetectionFailed" "Unable to distinguish SEV-SNP from TDX using systemd-detect-virt --cvm" "Error" + return "$ERR_CVM_PLATFORM_DETECTION_FAIL" + fi + + case "$platform" in + sev-snp) + echo "AMD SEV-SNP detected; preserving the existing Hyper-V PHC Chrony configuration" + emit_event "AKS.CSE.chrony.usingPHC" "AMD SEV-SNP detected; preserving the existing /dev/ptp0 PHC configuration" + configure_chrony + ;; + tdx) + echo "Intel TDX detected; configuring Chrony to use the Ubuntu NTP pools" + emit_event "AKS.CSE.chrony.usingNTP" "Intel TDX detected; using only the approved Ubuntu NTP pools" + ntp_pools="$(ubuntu_ntp_pools)" + configure_chrony "$ntp_pools" || return 1 + verify_chrony_ntp_sync + ;; + esac +} + # shellcheck disable=SC2317 ${__SOURCED__:+return} set -x +action=${1:-init} + # shellcheck disable=SC3010 if [[ -f /etc/os-release ]]; then . /etc/os-release @@ -570,6 +727,16 @@ fi echo "Running on $NAME" +ubuntu_2604_cvm_chrony_configured=0 +if [ "$action" = "init" ] && is_ubuntu_2604_cvm; then + configure_ubuntu_2604_cvm_time_sync + chrony_result=$? + if [ "$chrony_result" -ne 0 ]; then + exit "$chrony_result" + fi + ubuntu_2604_cvm_chrony_configured=1 +fi + # Certificate refresh behavior summary: # - legacy mode directly attempts certificate download from wireserver and only in ussec and usnat regions. @@ -636,7 +803,6 @@ fi # Action values: # - init (default): full provisioning path # - ca-refresh : periodic refresh path; location is passed as arg to avoid env dependency -action=${1:-init} if [ "$action" = "ca-refresh" ] || [ "$install_ca_refresh_schedule" -eq 0 ]; then exit 0 fi @@ -759,71 +925,8 @@ EOF systemctl restart chronyd else - chrony_conf="/etc/chrony/chrony.conf" - if [ "$IS_UBUNTU" -eq 1 ]; then - systemctl stop systemd-timesyncd - systemctl disable systemd-timesyncd - - if [ ! -e "$chrony_conf" ]; then - apt-get update - apt-get install chrony -y - fi - elif [ "$IS_FLATCAR" -eq 1 ]; then - rm -f ${chrony_conf} - fi - - cat > $chrony_conf <&2 + End + + When call configure_ubuntu_2604_cvm_time_sync + The error should include "unable to determine Ubuntu 26.04 CVM platform" + The error should include "AKS.CSE.chrony.platformDetectionFailed" + The status should equal 244 + End + + It 'preserves the existing PHC configuration for SEV-SNP' + setup_chrony_test + Mock detect_confidential_vm_platform + echo "sev-snp" + End + Mock emit_event + echo "event: $*" + End + + When call configure_ubuntu_2604_cvm_time_sync + The output should include "preserving the existing Hyper-V PHC Chrony configuration" + The output should include "AKS.CSE.chrony.usingPHC" + The contents of file "$CHRONY_CONF" should include "refclock PHC /dev/ptp0 poll 3 dpoll -2 offset 0" + The status should be success + End + + It 'defines exactly the four approved Ubuntu NTP pools' + When call ubuntu_ntp_pools + The lines of output should eq 4 + The line 1 of output should eq "pool ntp.ubuntu.com iburst maxsources 4" + The line 2 of output should eq "pool 0.ubuntu.pool.ntp.org iburst maxsources 1" + The line 3 of output should eq "pool 1.ubuntu.pool.ntp.org iburst maxsources 1" + The line 4 of output should eq "pool 2.ubuntu.pool.ntp.org iburst maxsources 2" + The status should be success + End + + It 'configures TDX with only the fixed Ubuntu NTP pools' + setup_chrony_test + Mock detect_confidential_vm_platform + echo "tdx" + End + Mock verify_chrony_ntp_sync + echo "verified NTP synchronization" + End + Mock emit_event + echo "event: $*" + End + + When call configure_ubuntu_2604_cvm_time_sync + The output should include "Intel TDX detected" + The output should include "verified NTP synchronization" + The contents of file "$CHRONY_CONF" should include "pool ntp.ubuntu.com iburst maxsources 4" + The contents of file "$CHRONY_CONF" should include "pool 0.ubuntu.pool.ntp.org iburst maxsources 1" + The contents of file "$CHRONY_CONF" should include "pool 1.ubuntu.pool.ntp.org iburst maxsources 1" + The contents of file "$CHRONY_CONF" should include "pool 2.ubuntu.pool.ntp.org iburst maxsources 2" + The contents of file "$CHRONY_CONF" should not include "refclock PHC" + The status should be success + End + + It 'waits for Chrony to synchronize successfully' + Mock chronyc + echo "$*" + End + Mock emit_event + echo "event: $*" + End + + When call verify_chrony_ntp_sync + The output should include "waitsync 12 0 0 5" + The output should include "NTP synchronization confirmed through the Ubuntu NTP pools" + The output should include "AKS.CSE.chrony.ntpSynchronized" + The status should be success + End + + It 'returns the outbound connectivity code with Chrony diagnostics when NTP is not reachable' + Mock chronyc + case "$1" in + waitsync) + return 1 + ;; + sources) + echo "mock Chrony sources" + ;; + tracking) + echo "mock Chrony tracking" + ;; + esac + End + Mock emit_event + echo "event: $*" >&2 + End + + When call verify_chrony_ntp_sync + The error should include "NTP not reachable" + The error should include "AKS.CSE.chrony.ntpUnavailable" + The error should include "mock Chrony sources" + The error should include "mock Chrony tracking" + The status should equal 50 + End + + It 'propagates failed TDX NTP synchronization' + Mock detect_confidential_vm_platform + echo "tdx" + End + Mock ubuntu_ntp_pools + echo "fixed pools" + End + Mock configure_chrony + : + End + Mock verify_chrony_ntp_sync + exit 50 + End + Mock emit_event + : + End + + When call configure_ubuntu_2604_cvm_time_sync + The output should include "Intel TDX detected" + The status should equal 50 + End + End + Describe 'init_mariner_repo_depot' It 'creates extended, nvidia, and cloud-native repos and points all at RepoDepot' export YUM_REPOS_DIR="${TEST_DIR}/yum.repos.d" From 4e2452fd57df30f500858588b039a4585ce146b5 Mon Sep 17 00:00:00 2001 From: Zachary Bailey Date: Fri, 18 Sep 2026 14:22:24 -0400 Subject: [PATCH 02/14] fix --- parts/linux/cloud-init/artifacts/cse_cmd.sh | 4 ++-- parts/linux/cloud-init/artifacts/cse_helpers.sh | 1 + parts/linux/cloud-init/artifacts/init-aks-cloud.sh | 4 ++-- pkg/agent/baker_test.go | 2 +- .../linux/cloud-init/artifacts/init_aks_cloud_spec.sh | 8 ++++---- 5 files changed, 10 insertions(+), 9 deletions(-) diff --git a/parts/linux/cloud-init/artifacts/cse_cmd.sh b/parts/linux/cloud-init/artifacts/cse_cmd.sh index 69ef46cd623..8578d58d39d 100644 --- a/parts/linux/cloud-init/artifacts/cse_cmd.sh +++ b/parts/linux/cloud-init/artifacts/cse_cmd.sh @@ -20,8 +20,8 @@ INIT_AKS_CLOUD_FILEPATH="{{GetInitAKSCloudFilepath}}"; if [ -f "${INIT_AKS_CLOUD_FILEPATH}" ]; then REPO_DEPOT_ENDPOINT="{{AKSCustomCloudRepoDepotEndpoint}}" LOCATION={{GetVariable "location"}} "${INIT_AKS_CLOUD_FILEPATH}" >> /var/log/azure/cluster-provision.log 2>&1; initAKSCloudExitCode=$?; - if [ "$initAKSCloudExitCode" -eq 50 ]; then - echo "NTP not reachable; init-aks-cloud failed with outbound connectivity error code ${initAKSCloudExitCode}" >> ${PROVISION_OUTPUT}; + if [ "$initAKSCloudExitCode" -eq 245 ]; then + echo "NTP not reachable; init-aks-cloud failed with NTP synchronization error code ${initAKSCloudExitCode}" >> ${PROVISION_OUTPUT}; exit ${initAKSCloudExitCode}; elif [ "$initAKSCloudExitCode" -eq 244 ]; then echo "Unable to determine confidential VM platform; init-aks-cloud failed with exit code ${initAKSCloudExitCode}" >> ${PROVISION_OUTPUT}; diff --git a/parts/linux/cloud-init/artifacts/cse_helpers.sh b/parts/linux/cloud-init/artifacts/cse_helpers.sh index 60fd4dbbeab..a0e9919e6b9 100755 --- a/parts/linux/cloud-init/artifacts/cse_helpers.sh +++ b/parts/linux/cloud-init/artifacts/cse_helpers.sh @@ -163,6 +163,7 @@ ERR_AZNFS_RPM_DOWNLOAD_TIMEOUT=241 # Timeout downloading aznfs RPM from PMC ERR_AZNFS_INSTALL_FAIL=242 # Failed to install aznfs RPM package ERR_SECONDARY_NIC_CONFIG_FAIL=243 # Error configuring secondary NIC network interface ERR_CVM_PLATFORM_DETECTION_FAIL=244 # Unable to distinguish SEV-SNP from TDX +ERR_NTP_UNREACHABLE=245 # Chrony could not synchronize with the configured NTP pools # ----------------------------------------------------------------------------- # This probably wasn't launched via a login shell, so ensure the PATH is correct. diff --git a/parts/linux/cloud-init/artifacts/init-aks-cloud.sh b/parts/linux/cloud-init/artifacts/init-aks-cloud.sh index 0927d338868..41033f73a98 100644 --- a/parts/linux/cloud-init/artifacts/init-aks-cloud.sh +++ b/parts/linux/cloud-init/artifacts/init-aks-cloud.sh @@ -81,8 +81,8 @@ IS_ACL=0 IS_MARINER=0 IS_AZURELINUX=0 -ERR_OUTBOUND_CONN_FAIL=50 # Unable to establish outbound connection ERR_CVM_PLATFORM_DETECTION_FAIL=244 # Unable to distinguish SEV-SNP from TDX +ERR_NTP_UNREACHABLE=245 # Chrony could not synchronize with the configured NTP pools # http://168.63.129.16 is a constant for the host's wireserver endpoint. WIRESERVER_ENDPOINT="http://168.63.129.16" @@ -667,7 +667,7 @@ function verify_chrony_ntp_sync { chronyc sources -v >&2 || echo "ERROR: unable to retrieve Chrony source diagnostics" >&2 echo "Chrony tracking diagnostics:" >&2 chronyc tracking >&2 || echo "ERROR: unable to retrieve Chrony tracking diagnostics" >&2 - return "$ERR_OUTBOUND_CONN_FAIL" + return "$ERR_NTP_UNREACHABLE" } function configure_ubuntu_2604_cvm_time_sync { diff --git a/pkg/agent/baker_test.go b/pkg/agent/baker_test.go index 07643792889..e23a605d326 100644 --- a/pkg/agent/baker_test.go +++ b/pkg/agent/baker_test.go @@ -1369,7 +1369,7 @@ var _ = Describe("getLinuxNodeCSECommand", func() { Expect(cseCmd).To(ContainSubstring("cloud-init status --wait")) Expect(cseCmd).To(ContainSubstring("cloudInitExitCode=$?")) Expect(cseCmd).To(ContainSubstring("initAKSCloudExitCode=$?")) - Expect(cseCmd).To(ContainSubstring("if [ \"$initAKSCloudExitCode\" -eq 50 ]")) + Expect(cseCmd).To(ContainSubstring("if [ \"$initAKSCloudExitCode\" -eq 245 ]")) Expect(cseCmd).To(ContainSubstring("NTP not reachable")) Expect(cseCmd).To(ContainSubstring("elif [ \"$initAKSCloudExitCode\" -eq 244 ]")) Expect(cseCmd).To(ContainSubstring("Unable to determine confidential VM platform")) diff --git a/spec/parts/linux/cloud-init/artifacts/init_aks_cloud_spec.sh b/spec/parts/linux/cloud-init/artifacts/init_aks_cloud_spec.sh index 0f764dedb9f..eda5402dc15 100644 --- a/spec/parts/linux/cloud-init/artifacts/init_aks_cloud_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/init_aks_cloud_spec.sh @@ -363,7 +363,7 @@ EOF The status should be success End - It 'returns the outbound connectivity code with Chrony diagnostics when NTP is not reachable' + It 'returns the NTP-unreachable code with Chrony diagnostics when NTP is not reachable' Mock chronyc case "$1" in waitsync) @@ -386,7 +386,7 @@ EOF The error should include "AKS.CSE.chrony.ntpUnavailable" The error should include "mock Chrony sources" The error should include "mock Chrony tracking" - The status should equal 50 + The status should equal 245 End It 'propagates failed TDX NTP synchronization' @@ -400,7 +400,7 @@ EOF : End Mock verify_chrony_ntp_sync - exit 50 + exit 245 End Mock emit_event : @@ -408,7 +408,7 @@ EOF When call configure_ubuntu_2604_cvm_time_sync The output should include "Intel TDX detected" - The status should equal 50 + The status should equal 245 End End From 024693f16fb0bbf087bbe643941eafc98bef9de9 Mon Sep 17 00:00:00 2001 From: Zachary Bailey Date: Fri, 18 Sep 2026 15:46:00 -0400 Subject: [PATCH 03/14] fix --- parts/linux/cloud-init/artifacts/cse_cmd.sh | 5 +- .../linux/cloud-init/artifacts/cse_helpers.sh | 1 + .../cloud-init/artifacts/init-aks-cloud.sh | 54 ++++++++++++++---- pkg/agent/baker_test.go | 4 +- .../artifacts/init_aks_cloud_spec.sh | 55 +++++++++++++++++++ 5 files changed, 107 insertions(+), 12 deletions(-) diff --git a/parts/linux/cloud-init/artifacts/cse_cmd.sh b/parts/linux/cloud-init/artifacts/cse_cmd.sh index 8578d58d39d..9ee7c62cf6f 100644 --- a/parts/linux/cloud-init/artifacts/cse_cmd.sh +++ b/parts/linux/cloud-init/artifacts/cse_cmd.sh @@ -20,7 +20,10 @@ INIT_AKS_CLOUD_FILEPATH="{{GetInitAKSCloudFilepath}}"; if [ -f "${INIT_AKS_CLOUD_FILEPATH}" ]; then REPO_DEPOT_ENDPOINT="{{AKSCustomCloudRepoDepotEndpoint}}" LOCATION={{GetVariable "location"}} "${INIT_AKS_CLOUD_FILEPATH}" >> /var/log/azure/cluster-provision.log 2>&1; initAKSCloudExitCode=$?; - if [ "$initAKSCloudExitCode" -eq 245 ]; then + if [ "$initAKSCloudExitCode" -eq 246 ]; then + echo "Chrony configuration failed; init-aks-cloud failed with exit code ${initAKSCloudExitCode}" >> ${PROVISION_OUTPUT}; + exit ${initAKSCloudExitCode}; + elif [ "$initAKSCloudExitCode" -eq 245 ]; then echo "NTP not reachable; init-aks-cloud failed with NTP synchronization error code ${initAKSCloudExitCode}" >> ${PROVISION_OUTPUT}; exit ${initAKSCloudExitCode}; elif [ "$initAKSCloudExitCode" -eq 244 ]; then diff --git a/parts/linux/cloud-init/artifacts/cse_helpers.sh b/parts/linux/cloud-init/artifacts/cse_helpers.sh index a0e9919e6b9..13edf70f0d8 100755 --- a/parts/linux/cloud-init/artifacts/cse_helpers.sh +++ b/parts/linux/cloud-init/artifacts/cse_helpers.sh @@ -164,6 +164,7 @@ ERR_AZNFS_INSTALL_FAIL=242 # Failed to install aznfs RPM package ERR_SECONDARY_NIC_CONFIG_FAIL=243 # Error configuring secondary NIC network interface ERR_CVM_PLATFORM_DETECTION_FAIL=244 # Unable to distinguish SEV-SNP from TDX ERR_NTP_UNREACHABLE=245 # Chrony could not synchronize with the configured NTP pools +ERR_CHRONY_CONFIG_FAIL=246 # Chrony could not be configured for the detected CVM platform # ----------------------------------------------------------------------------- # This probably wasn't launched via a login shell, so ensure the PATH is correct. diff --git a/parts/linux/cloud-init/artifacts/init-aks-cloud.sh b/parts/linux/cloud-init/artifacts/init-aks-cloud.sh index 41033f73a98..90b8871f977 100644 --- a/parts/linux/cloud-init/artifacts/init-aks-cloud.sh +++ b/parts/linux/cloud-init/artifacts/init-aks-cloud.sh @@ -83,6 +83,7 @@ IS_AZURELINUX=0 ERR_CVM_PLATFORM_DETECTION_FAIL=244 # Unable to distinguish SEV-SNP from TDX ERR_NTP_UNREACHABLE=245 # Chrony could not synchronize with the configured NTP pools +ERR_CHRONY_CONFIG_FAIL=246 # Chrony could not be configured for the detected CVM platform # http://168.63.129.16 is a constant for the host's wireserver endpoint. WIRESERVER_ENDPOINT="http://168.63.129.16" @@ -585,18 +586,33 @@ function configure_chrony { local chrony_conf="${CHRONY_CONF:-/etc/chrony/chrony.conf}" if [ "$IS_UBUNTU" -eq 1 ]; then - systemctl stop systemd-timesyncd - systemctl disable systemd-timesyncd + if ! systemctl stop systemd-timesyncd; then + echo "ERROR: failed to stop systemd-timesyncd" >&2 + return 1 + fi + if ! systemctl disable systemd-timesyncd; then + echo "ERROR: failed to disable systemd-timesyncd" >&2 + return 1 + fi if [ ! -e "$chrony_conf" ]; then - apt-get update - apt-get install chrony -y + if ! apt-get update; then + echo "ERROR: failed to update package metadata before installing Chrony" >&2 + return 1 + fi + if ! apt-get install chrony -y; then + echo "ERROR: failed to install Chrony" >&2 + return 1 + fi fi elif [ "$IS_FLATCAR" -eq 1 ]; then - rm -f "$chrony_conf" + if ! rm -f "$chrony_conf"; then + echo "ERROR: failed to remove the existing Flatcar Chrony configuration" >&2 + return 1 + fi fi - cat > "$chrony_conf" < "$chrony_conf" <&2 + return 1 + fi if [ "$IS_UBUNTU" -eq 1 ]; then - systemctl restart chrony + if ! systemctl restart chrony; then + echo "ERROR: failed to restart Chrony" >&2 + return 1 + fi elif [ "$IS_FLATCAR" -eq 1 ]; then - systemctl restart chronyd + if ! systemctl restart chronyd; then + echo "ERROR: failed to restart chronyd" >&2 + return 1 + fi fi } @@ -684,13 +710,21 @@ function configure_ubuntu_2604_cvm_time_sync { sev-snp) echo "AMD SEV-SNP detected; preserving the existing Hyper-V PHC Chrony configuration" emit_event "AKS.CSE.chrony.usingPHC" "AMD SEV-SNP detected; preserving the existing /dev/ptp0 PHC configuration" - configure_chrony + if ! configure_chrony; then + echo "ERROR: failed to configure Chrony with the Hyper-V PHC source for AMD SEV-SNP" >&2 + emit_event "AKS.CSE.chrony.configurationFailed" "Failed to configure Chrony with the Hyper-V PHC source for AMD SEV-SNP" "Error" + return "$ERR_CHRONY_CONFIG_FAIL" + fi ;; tdx) echo "Intel TDX detected; configuring Chrony to use the Ubuntu NTP pools" emit_event "AKS.CSE.chrony.usingNTP" "Intel TDX detected; using only the approved Ubuntu NTP pools" ntp_pools="$(ubuntu_ntp_pools)" - configure_chrony "$ntp_pools" || return 1 + if ! configure_chrony "$ntp_pools"; then + echo "ERROR: failed to configure Chrony with the Ubuntu NTP pools for Intel TDX" >&2 + emit_event "AKS.CSE.chrony.configurationFailed" "Failed to configure Chrony with the Ubuntu NTP pools for Intel TDX" "Error" + return "$ERR_CHRONY_CONFIG_FAIL" + fi verify_chrony_ntp_sync ;; esac diff --git a/pkg/agent/baker_test.go b/pkg/agent/baker_test.go index e23a605d326..220ca745ce8 100644 --- a/pkg/agent/baker_test.go +++ b/pkg/agent/baker_test.go @@ -1369,7 +1369,9 @@ var _ = Describe("getLinuxNodeCSECommand", func() { Expect(cseCmd).To(ContainSubstring("cloud-init status --wait")) Expect(cseCmd).To(ContainSubstring("cloudInitExitCode=$?")) Expect(cseCmd).To(ContainSubstring("initAKSCloudExitCode=$?")) - Expect(cseCmd).To(ContainSubstring("if [ \"$initAKSCloudExitCode\" -eq 245 ]")) + Expect(cseCmd).To(ContainSubstring("if [ \"$initAKSCloudExitCode\" -eq 246 ]")) + Expect(cseCmd).To(ContainSubstring("Chrony configuration failed")) + Expect(cseCmd).To(ContainSubstring("elif [ \"$initAKSCloudExitCode\" -eq 245 ]")) Expect(cseCmd).To(ContainSubstring("NTP not reachable")) Expect(cseCmd).To(ContainSubstring("elif [ \"$initAKSCloudExitCode\" -eq 244 ]")) Expect(cseCmd).To(ContainSubstring("Unable to determine confidential VM platform")) diff --git a/spec/parts/linux/cloud-init/artifacts/init_aks_cloud_spec.sh b/spec/parts/linux/cloud-init/artifacts/init_aks_cloud_spec.sh index eda5402dc15..082bd0631f7 100644 --- a/spec/parts/linux/cloud-init/artifacts/init_aks_cloud_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/init_aks_cloud_spec.sh @@ -315,6 +315,24 @@ EOF The status should be success End + It 'returns the Chrony configuration failure code when SEV-SNP PHC setup fails' + Mock detect_confidential_vm_platform + echo "sev-snp" + End + Mock configure_chrony + return 1 + End + Mock emit_event + echo "event: $*" >&2 + End + + When call configure_ubuntu_2604_cvm_time_sync + The output should include "AMD SEV-SNP detected" + The error should include "failed to configure Chrony with the Hyper-V PHC source" + The error should include "AKS.CSE.chrony.configurationFailed" + The status should equal 246 + End + It 'defines exactly the four approved Ubuntu NTP pools' When call ubuntu_ntp_pools The lines of output should eq 4 @@ -348,6 +366,43 @@ EOF The status should be success End + It 'returns failure when the Chrony service cannot restart' + setup_chrony_test + Mock systemctl + if [ "$1" = "restart" ]; then + return 1 + fi + End + + When call configure_chrony + The error should include "failed to restart Chrony" + The status should equal 1 + End + + It 'returns the Chrony configuration failure code without checking NTP when TDX setup fails' + Mock detect_confidential_vm_platform + echo "tdx" + End + Mock ubuntu_ntp_pools + echo "fixed pools" + End + Mock configure_chrony + return 1 + End + Mock verify_chrony_ntp_sync + echo "unexpected NTP verification" + End + Mock emit_event + echo "event: $*" >&2 + End + + When call configure_ubuntu_2604_cvm_time_sync + The output should not include "unexpected NTP verification" + The error should include "failed to configure Chrony with the Ubuntu NTP pools" + The error should include "AKS.CSE.chrony.configurationFailed" + The status should equal 246 + End + It 'waits for Chrony to synchronize successfully' Mock chronyc echo "$*" From db441348f0add2a904c741e20280c4d8f3b2524d Mon Sep 17 00:00:00 2001 From: Zachary Bailey Date: Fri, 18 Sep 2026 16:32:18 -0400 Subject: [PATCH 04/14] fix --- .../artifacts/init_aks_cloud_spec.sh | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/spec/parts/linux/cloud-init/artifacts/init_aks_cloud_spec.sh b/spec/parts/linux/cloud-init/artifacts/init_aks_cloud_spec.sh index 082bd0631f7..dac62abdf71 100644 --- a/spec/parts/linux/cloud-init/artifacts/init_aks_cloud_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/init_aks_cloud_spec.sh @@ -68,6 +68,28 @@ Describe 'init-aks-cloud.sh refresh mode wiring' End End +Describe 'init-aks-cloud.sh Chrony distro routing' + script_path='./parts/linux/cloud-init/artifacts/init-aks-cloud.sh' + + chrony_routing_block() { + sed -n '/^if \[ "\$IS_ACL" -eq 1 \]; then$/,/^#EOF$/p' "$script_path" + } + + It 'keeps Azure Linux and Mariner on their native chronyd configuration path' + When call chrony_routing_block + The output should include 'elif [ "$IS_MARINER" -eq 1 ] || [ "$IS_AZURELINUX" -eq 1 ]; then' + The output should include 'cat > /etc/chrony.conf < Date: Fri, 18 Sep 2026 16:40:12 -0400 Subject: [PATCH 05/14] fix --- .../cloud-init/artifacts/init-aks-cloud.sh | 20 +++++++---- .../artifacts/init_aks_cloud_spec.sh | 35 +++++++++++++++++++ 2 files changed, 48 insertions(+), 7 deletions(-) diff --git a/parts/linux/cloud-init/artifacts/init-aks-cloud.sh b/parts/linux/cloud-init/artifacts/init-aks-cloud.sh index 90b8871f977..69e948f1528 100644 --- a/parts/linux/cloud-init/artifacts/init-aks-cloud.sh +++ b/parts/linux/cloud-init/artifacts/init-aks-cloud.sh @@ -584,15 +584,21 @@ EOF function configure_chrony { local time_sources="${1:-refclock PHC /dev/ptp0 poll 3 dpoll -2 offset 0}" local chrony_conf="${CHRONY_CONF:-/etc/chrony/chrony.conf}" + local timesyncd_status if [ "$IS_UBUNTU" -eq 1 ]; then - if ! systemctl stop systemd-timesyncd; then - echo "ERROR: failed to stop systemd-timesyncd" >&2 - return 1 - fi - if ! systemctl disable systemd-timesyncd; then - echo "ERROR: failed to disable systemd-timesyncd" >&2 - return 1 + timesyncd_status="$(systemctl show -p SubState --value systemd-timesyncd 2>/dev/null || true)" + if [ "$timesyncd_status" = "dead" ]; then + echo "systemd-timesyncd is removed, no need to disable" + else + if ! systemctl stop systemd-timesyncd; then + echo "ERROR: failed to stop systemd-timesyncd" >&2 + return 1 + fi + if ! systemctl disable systemd-timesyncd; then + echo "ERROR: failed to disable systemd-timesyncd" >&2 + return 1 + fi fi if [ ! -e "$chrony_conf" ]; then diff --git a/spec/parts/linux/cloud-init/artifacts/init_aks_cloud_spec.sh b/spec/parts/linux/cloud-init/artifacts/init_aks_cloud_spec.sh index dac62abdf71..12186072da7 100644 --- a/spec/parts/linux/cloud-init/artifacts/init_aks_cloud_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/init_aks_cloud_spec.sh @@ -422,6 +422,41 @@ EOF The status should equal 1 End + It 'skips stopping and disabling systemd-timesyncd when the unit is removed' + setup_chrony_test + Mock systemctl + if [ "$1" = "show" ]; then + echo "dead" + return 1 + fi + if [ "$1" = "stop" ] || [ "$1" = "disable" ]; then + echo "unexpected systemd-timesyncd operation" + return 1 + fi + End + + When call configure_chrony + The output should include "systemd-timesyncd is removed, no need to disable" + The output should not include "unexpected systemd-timesyncd operation" + The contents of file "$CHRONY_CONF" should include "refclock PHC /dev/ptp0 poll 3 dpoll -2 offset 0" + The status should be success + End + + It 'returns failure when an existing systemd-timesyncd unit cannot be stopped' + setup_chrony_test + Mock systemctl + if [ "$1" = "show" ]; then + echo "running" + elif [ "$1" = "stop" ]; then + return 1 + fi + End + + When call configure_chrony + The error should include "failed to stop systemd-timesyncd" + The status should equal 1 + End + It 'returns the Chrony configuration failure code without checking NTP when TDX setup fails' Mock detect_confidential_vm_platform echo "tdx" From bb8913683b1d818bbd01d6e0adeb50c94c82126b Mon Sep 17 00:00:00 2001 From: Zachary Bailey Date: Fri, 18 Sep 2026 17:01:20 -0400 Subject: [PATCH 06/14] fix --- parts/linux/cloud-init/artifacts/cse_helpers.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/parts/linux/cloud-init/artifacts/cse_helpers.sh b/parts/linux/cloud-init/artifacts/cse_helpers.sh index 13edf70f0d8..7e45cd1f0e8 100755 --- a/parts/linux/cloud-init/artifacts/cse_helpers.sh +++ b/parts/linux/cloud-init/artifacts/cse_helpers.sh @@ -162,9 +162,7 @@ ERR_AKS_NODE_CONTROLLER_ERROR=240 # Generic error in AKS Node Controller ERR_AZNFS_RPM_DOWNLOAD_TIMEOUT=241 # Timeout downloading aznfs RPM from PMC ERR_AZNFS_INSTALL_FAIL=242 # Failed to install aznfs RPM package ERR_SECONDARY_NIC_CONFIG_FAIL=243 # Error configuring secondary NIC network interface -ERR_CVM_PLATFORM_DETECTION_FAIL=244 # Unable to distinguish SEV-SNP from TDX -ERR_NTP_UNREACHABLE=245 # Chrony could not synchronize with the configured NTP pools -ERR_CHRONY_CONFIG_FAIL=246 # Chrony could not be configured for the detected CVM platform +# Exit codes 244-246 are reserved by init-aks-cloud.sh and propagated by cse_cmd.sh. # ----------------------------------------------------------------------------- # This probably wasn't launched via a login shell, so ensure the PATH is correct. From dfa3d99231a66c808505467c3a21a29501cb51e0 Mon Sep 17 00:00:00 2001 From: Zachary Bailey Date: Mon, 21 Sep 2026 11:25:26 -0400 Subject: [PATCH 07/14] fix --- .../cloud-init/artifacts/init-aks-cloud.sh | 6 +++--- .../artifacts/init_aks_cloud_spec.sh | 21 +++++++++++++++++-- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/parts/linux/cloud-init/artifacts/init-aks-cloud.sh b/parts/linux/cloud-init/artifacts/init-aks-cloud.sh index f6765bd9426..12f86a215e4 100644 --- a/parts/linux/cloud-init/artifacts/init-aks-cloud.sh +++ b/parts/linux/cloud-init/artifacts/init-aks-cloud.sh @@ -581,11 +581,11 @@ EOF function configure_chrony { local time_sources="${1:-refclock PHC /dev/ptp0 poll 3 dpoll -2 offset 0}" local chrony_conf="${CHRONY_CONF:-/etc/chrony/chrony.conf}" - local timesyncd_status + local timesyncd_load_state if [ "$IS_UBUNTU" -eq 1 ]; then - timesyncd_status="$(systemctl show -p SubState --value systemd-timesyncd 2>/dev/null || true)" - if [ "$timesyncd_status" = "dead" ]; then + timesyncd_load_state="$(systemctl show -p LoadState --value systemd-timesyncd 2>/dev/null || true)" + if [ "$timesyncd_load_state" = "not-found" ]; then echo "systemd-timesyncd is removed, no need to disable" else if ! systemctl stop systemd-timesyncd; then diff --git a/spec/parts/linux/cloud-init/artifacts/init_aks_cloud_spec.sh b/spec/parts/linux/cloud-init/artifacts/init_aks_cloud_spec.sh index 12186072da7..e0ca977f776 100644 --- a/spec/parts/linux/cloud-init/artifacts/init_aks_cloud_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/init_aks_cloud_spec.sh @@ -426,7 +426,7 @@ EOF setup_chrony_test Mock systemctl if [ "$1" = "show" ]; then - echo "dead" + echo "not-found" return 1 fi if [ "$1" = "stop" ] || [ "$1" = "disable" ]; then @@ -442,11 +442,28 @@ EOF The status should be success End + It 'stops and disables systemd-timesyncd when the unit is loaded but inactive' + setup_chrony_test + Mock systemctl + if [ "$1" = "show" ]; then + echo "loaded" + elif [ "$1" = "stop" ] || [ "$1" = "disable" ]; then + echo "systemctl $*" + fi + End + + When call configure_chrony + The output should include "systemctl stop systemd-timesyncd" + The output should include "systemctl disable systemd-timesyncd" + The contents of file "$CHRONY_CONF" should include "refclock PHC /dev/ptp0 poll 3 dpoll -2 offset 0" + The status should be success + End + It 'returns failure when an existing systemd-timesyncd unit cannot be stopped' setup_chrony_test Mock systemctl if [ "$1" = "show" ]; then - echo "running" + echo "loaded" elif [ "$1" = "stop" ]; then return 1 fi From 4609b62117f1334e9d0942fc86b8b4f10e8bd75d Mon Sep 17 00:00:00 2001 From: Zachary Bailey Date: Mon, 21 Sep 2026 15:02:47 -0400 Subject: [PATCH 08/14] fix --- .../cloud-init/artifacts/init-aks-cloud.sh | 21 +++++++++++-------- .../artifacts/init_aks_cloud_spec.sh | 9 ++++++-- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/parts/linux/cloud-init/artifacts/init-aks-cloud.sh b/parts/linux/cloud-init/artifacts/init-aks-cloud.sh index 12f86a215e4..fffb5df916b 100644 --- a/parts/linux/cloud-init/artifacts/init-aks-cloud.sh +++ b/parts/linux/cloud-init/artifacts/init-aks-cloud.sh @@ -582,6 +582,7 @@ function configure_chrony { local time_sources="${1:-refclock PHC /dev/ptp0 poll 3 dpoll -2 offset 0}" local chrony_conf="${CHRONY_CONF:-/etc/chrony/chrony.conf}" local timesyncd_load_state + local chrony_failed=0 if [ "$IS_UBUNTU" -eq 1 ]; then timesyncd_load_state="$(systemctl show -p LoadState --value systemd-timesyncd 2>/dev/null || true)" @@ -590,28 +591,28 @@ function configure_chrony { else if ! systemctl stop systemd-timesyncd; then echo "ERROR: failed to stop systemd-timesyncd" >&2 - return 1 + chrony_failed=1 fi if ! systemctl disable systemd-timesyncd; then echo "ERROR: failed to disable systemd-timesyncd" >&2 - return 1 + chrony_failed=1 fi fi if [ ! -e "$chrony_conf" ]; then if ! apt-get update; then echo "ERROR: failed to update package metadata before installing Chrony" >&2 - return 1 + chrony_failed=1 fi if ! apt-get install chrony -y; then echo "ERROR: failed to install Chrony" >&2 - return 1 + chrony_failed=1 fi fi elif [ "$IS_FLATCAR" -eq 1 ]; then if ! rm -f "$chrony_conf"; then echo "ERROR: failed to remove the existing Flatcar Chrony configuration" >&2 - return 1 + chrony_failed=1 fi fi @@ -664,20 +665,22 @@ makestep 1.0 -1 EOF then echo "ERROR: failed to write Chrony configuration to ${chrony_conf}" >&2 - return 1 + chrony_failed=1 fi if [ "$IS_UBUNTU" -eq 1 ]; then if ! systemctl restart chrony; then echo "ERROR: failed to restart Chrony" >&2 - return 1 + chrony_failed=1 fi elif [ "$IS_FLATCAR" -eq 1 ]; then if ! systemctl restart chronyd; then echo "ERROR: failed to restart chronyd" >&2 - return 1 + chrony_failed=1 fi fi + + return "$chrony_failed" } function verify_chrony_ntp_sync { @@ -962,7 +965,7 @@ EOF systemctl restart chronyd else if [ "$ubuntu_2604_cvm_chrony_configured" -eq 0 ]; then - configure_chrony + configure_chrony || true fi fi diff --git a/spec/parts/linux/cloud-init/artifacts/init_aks_cloud_spec.sh b/spec/parts/linux/cloud-init/artifacts/init_aks_cloud_spec.sh index e0ca977f776..2e6f8c4b0c0 100644 --- a/spec/parts/linux/cloud-init/artifacts/init_aks_cloud_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/init_aks_cloud_spec.sh @@ -86,7 +86,7 @@ Describe 'init-aks-cloud.sh Chrony distro routing' It 'keeps Ubuntu and Flatcar on configure_chrony unless the CVM path already configured it' When call chrony_routing_block The output should include 'if [ "$ubuntu_2604_cvm_chrony_configured" -eq 0 ]; then' - The output should include 'configure_chrony' + The output should include 'configure_chrony || true' End End @@ -459,18 +459,23 @@ EOF The status should be success End - It 'returns failure when an existing systemd-timesyncd unit cannot be stopped' + It 'continues Chrony setup and returns failure when systemd-timesyncd cannot be stopped' setup_chrony_test Mock systemctl if [ "$1" = "show" ]; then echo "loaded" elif [ "$1" = "stop" ]; then return 1 + elif [ "$1" = "disable" ] || [ "$1" = "restart" ]; then + echo "systemctl $*" fi End When call configure_chrony The error should include "failed to stop systemd-timesyncd" + The output should include "systemctl disable systemd-timesyncd" + The output should include "systemctl restart chrony" + The contents of file "$CHRONY_CONF" should include "refclock PHC /dev/ptp0 poll 3 dpoll -2 offset 0" The status should equal 1 End From 1318f5a5836e16587c123f414597dc09e71859ef Mon Sep 17 00:00:00 2001 From: Zachary Bailey Date: Tue, 22 Sep 2026 17:11:21 -0400 Subject: [PATCH 09/14] fix --- parts/linux/cloud-init/artifacts/cse_cmd.sh | 2 +- .../cloud-init/artifacts/init-aks-cloud.sh | 7 +++++- pkg/agent/baker_test.go | 9 +++++++ .../artifacts/init_aks_cloud_spec.sh | 24 +++++++++++++++++++ 4 files changed, 40 insertions(+), 2 deletions(-) diff --git a/parts/linux/cloud-init/artifacts/cse_cmd.sh b/parts/linux/cloud-init/artifacts/cse_cmd.sh index 9ee7c62cf6f..8291358daea 100644 --- a/parts/linux/cloud-init/artifacts/cse_cmd.sh +++ b/parts/linux/cloud-init/artifacts/cse_cmd.sh @@ -18,7 +18,7 @@ fi; {{end}} INIT_AKS_CLOUD_FILEPATH="{{GetInitAKSCloudFilepath}}"; if [ -f "${INIT_AKS_CLOUD_FILEPATH}" ]; then - REPO_DEPOT_ENDPOINT="{{AKSCustomCloudRepoDepotEndpoint}}" LOCATION={{GetVariable "location"}} "${INIT_AKS_CLOUD_FILEPATH}" >> /var/log/azure/cluster-provision.log 2>&1; + PRE_PROVISION_ONLY="{{GetPreProvisionOnly}}" REPO_DEPOT_ENDPOINT="{{AKSCustomCloudRepoDepotEndpoint}}" LOCATION={{GetVariable "location"}} "${INIT_AKS_CLOUD_FILEPATH}" >> /var/log/azure/cluster-provision.log 2>&1; initAKSCloudExitCode=$?; if [ "$initAKSCloudExitCode" -eq 246 ]; then echo "Chrony configuration failed; init-aks-cloud failed with exit code ${initAKSCloudExitCode}" >> ${PROVISION_OUTPUT}; diff --git a/parts/linux/cloud-init/artifacts/init-aks-cloud.sh b/parts/linux/cloud-init/artifacts/init-aks-cloud.sh index fffb5df916b..538598ddc13 100644 --- a/parts/linux/cloud-init/artifacts/init-aks-cloud.sh +++ b/parts/linux/cloud-init/artifacts/init-aks-cloud.sh @@ -552,6 +552,11 @@ function is_ubuntu_2604_cvm { esac } +function should_configure_ubuntu_2604_cvm_time_sync { + [ "${PRE_PROVISION_ONLY:-false}" != "true" ] || return 1 + is_ubuntu_2604_cvm +} + function detect_confidential_vm_platform { local platform @@ -767,7 +772,7 @@ fi echo "Running on $NAME" ubuntu_2604_cvm_chrony_configured=0 -if [ "$action" = "init" ] && is_ubuntu_2604_cvm; then +if [ "$action" = "init" ] && should_configure_ubuntu_2604_cvm_time_sync; then configure_ubuntu_2604_cvm_time_sync chrony_result=$? if [ "$chrony_result" -ne 0 ]; then diff --git a/pkg/agent/baker_test.go b/pkg/agent/baker_test.go index 220ca745ce8..329f7e4f435 100644 --- a/pkg/agent/baker_test.go +++ b/pkg/agent/baker_test.go @@ -1368,6 +1368,7 @@ var _ = Describe("getLinuxNodeCSECommand", func() { Expect(cseCmd).To(ContainSubstring("handleCloudInitStatus")) Expect(cseCmd).To(ContainSubstring("cloud-init status --wait")) Expect(cseCmd).To(ContainSubstring("cloudInitExitCode=$?")) + Expect(cseCmd).To(ContainSubstring("PRE_PROVISION_ONLY=\"false\" REPO_DEPOT_ENDPOINT=")) Expect(cseCmd).To(ContainSubstring("initAKSCloudExitCode=$?")) Expect(cseCmd).To(ContainSubstring("if [ \"$initAKSCloudExitCode\" -eq 246 ]")) Expect(cseCmd).To(ContainSubstring("Chrony configuration failed")) @@ -1378,6 +1379,14 @@ var _ = Describe("getLinuxNodeCSECommand", func() { Expect(cseCmd).To(ContainSubstring("exit ${initAKSCloudExitCode}")) }) + It("should pass pre-provision mode to init-aks-cloud", func() { + baseConfig.PreProvisionOnly = true + + cseCmd := templateGenerator.getLinuxNodeCSECommand(baseConfig) + + Expect(cseCmd).To(ContainSubstring("PRE_PROVISION_ONLY=\"true\" REPO_DEPOT_ENDPOINT=")) + }) + It("should handle configuration with custom kubelet config", func() { baseConfig.KubeletConfig = map[string]string{ "--max-pods": "110", diff --git a/spec/parts/linux/cloud-init/artifacts/init_aks_cloud_spec.sh b/spec/parts/linux/cloud-init/artifacts/init_aks_cloud_spec.sh index 2e6f8c4b0c0..554929a6704 100644 --- a/spec/parts/linux/cloud-init/artifacts/init_aks_cloud_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/init_aks_cloud_spec.sh @@ -287,6 +287,30 @@ EOF The status should be failure End + It 'skips platform-specific time sync during pre-provision image preparation' + IS_UBUNTU=1 + VERSION_ID="26.04" + PRE_PROVISION_ONLY="true" + Mock uname + echo "7.0.0-1011-azure-fde" + End + + When call should_configure_ubuntu_2604_cvm_time_sync + The status should be failure + End + + It 'selects platform-specific time sync when provisioning the real node' + IS_UBUNTU=1 + VERSION_ID="26.04" + PRE_PROVISION_ONLY="false" + Mock uname + echo "7.0.0-1011-azure-fde" + End + + When call should_configure_ubuntu_2604_cvm_time_sync + The status should be success + End + It 'preserves the PHC default for another Ubuntu release' setup_chrony_test VERSION_ID="24.04" From 0a5044d0207ba9944fbe3ad9f028c7266d826f28 Mon Sep 17 00:00:00 2001 From: Zachary Bailey Date: Wed, 23 Sep 2026 15:23:16 -0400 Subject: [PATCH 10/14] fix --- aks-node-controller/parser/consts.go | 1 + aks-node-controller/parser/helper.go | 4 + aks-node-controller/parser/parser.go | 1 + hotfix/hotfix_generate.py | 1 + hotfix/hotfix_generate_test.py | 6 + parts/linux/cloud-init/artifacts/cse_cmd.sh | 14 +- .../linux/cloud-init/artifacts/cse_config.sh | 1 + .../cloud-init/artifacts/cse_config_chrony.sh | 271 ++++++++++++++++++ .../linux/cloud-init/artifacts/cse_helpers.sh | 4 +- parts/linux/cloud-init/artifacts/cse_main.sh | 21 ++ .../cloud-init/artifacts/init-aks-cloud.sh | 254 ---------------- parts/linux/cloud-init/nodecustomdata.yml | 7 + pkg/agent/baker.go | 3 + pkg/agent/baker_test.go | 21 +- pkg/agent/const.go | 2 + pkg/agent/variables.go | 1 + .../artifacts/cse_main_pis_reconcile_spec.sh | 19 ++ .../artifacts/init_aks_cloud_spec.sh | 66 +++-- vhdbuilder/packer/README.md | 5 +- .../imagecustomizer/azlosguard/azlosguard.yml | 3 + vhdbuilder/packer/packer_source.sh | 2 +- .../packer/vhd-image-builder-acl-arm64.json | 5 + vhdbuilder/packer/vhd-image-builder-acl.json | 5 + .../packer/vhd-image-builder-arm64-gb.json | 5 + .../packer/vhd-image-builder-arm64-gen2.json | 5 + vhdbuilder/packer/vhd-image-builder-base.json | 5 + vhdbuilder/packer/vhd-image-builder-cvm.json | 5 + .../vhd-image-builder-mariner-arm64.json | 5 + .../packer/vhd-image-builder-mariner-cvm.json | 5 + .../packer/vhd-image-builder-mariner.json | 5 + 30 files changed, 439 insertions(+), 313 deletions(-) create mode 100644 parts/linux/cloud-init/artifacts/cse_config_chrony.sh diff --git a/aks-node-controller/parser/consts.go b/aks-node-controller/parser/consts.go index 0e3534b5200..c52803b7189 100644 --- a/aks-node-controller/parser/consts.go +++ b/aks-node-controller/parser/consts.go @@ -28,6 +28,7 @@ const ( cseConfigKubeletScriptFilepath = "/opt/azure/containers/provision_configs_kubelet.sh" cseConfigNetworkScriptFilepath = "/opt/azure/containers/provision_configs_network.sh" cseConfigAddonsScriptFilepath = "/opt/azure/containers/provision_configs_addons.sh" + cseConfigChronyScriptFilepath = "/opt/azure/containers/provision_configs_chrony.sh" customSearchDomainsCSEScriptFilepath = "/opt/azure/containers/setup-custom-search-domains.sh" dhcpV6ServiceCSEScriptFilepath = "/etc/systemd/system/dhcpv6.service" dhcpV6ConfigCSEScriptFilepath = "/opt/azure/containers/enable-dhcpv6.sh" diff --git a/aks-node-controller/parser/helper.go b/aks-node-controller/parser/helper.go index bf8f25ffd62..8516fbc4a82 100644 --- a/aks-node-controller/parser/helper.go +++ b/aks-node-controller/parser/helper.go @@ -314,6 +314,10 @@ func getCSEConfigAddonsFilepath() string { return cseConfigAddonsScriptFilepath } +func getCSEConfigChronyFilepath() string { + return cseConfigChronyScriptFilepath +} + func getCustomSearchDomainFilepath() string { return customSearchDomainsCSEScriptFilepath } diff --git a/aks-node-controller/parser/parser.go b/aks-node-controller/parser/parser.go index cd8124e01f5..60a155897f5 100644 --- a/aks-node-controller/parser/parser.go +++ b/aks-node-controller/parser/parser.go @@ -141,6 +141,7 @@ func getCSEEnv(ctx context.Context, config *aksnodeconfigv1.Configuration, gpuCo "CSE_CONFIG_KUBELET_FILEPATH": getCSEConfigKubeletFilepath(), "CSE_CONFIG_NETWORK_FILEPATH": getCSEConfigNetworkFilepath(), "CSE_CONFIG_ADDONS_FILEPATH": getCSEConfigAddonsFilepath(), + "CSE_CONFIG_CHRONY_FILEPATH": getCSEConfigChronyFilepath(), "AZURE_PRIVATE_REGISTRY_SERVER": config.GetAzurePrivateRegistryServer(), "HAS_CUSTOM_SEARCH_DOMAIN": fmt.Sprintf("%v", getHasSearchDomain(config.GetCustomSearchDomainConfig())), "CUSTOM_SEARCH_DOMAIN_FILEPATH": getCustomSearchDomainFilepath(), diff --git a/hotfix/hotfix_generate.py b/hotfix/hotfix_generate.py index 0f927504853..2623604bac3 100644 --- a/hotfix/hotfix_generate.py +++ b/hotfix/hotfix_generate.py @@ -76,6 +76,7 @@ "cse_config_kubelet.sh": "provisionConfigsKubelet", "cse_config_network.sh": "provisionConfigsNetwork", "cse_config_addons.sh": "provisionConfigsAddons", + "cse_config_chrony.sh": "provisionConfigsChrony", # CSE main "cse_main.sh": "provisionScript", # Other scripts present in traditional nodecustomdata diff --git a/hotfix/hotfix_generate_test.py b/hotfix/hotfix_generate_test.py index 5f7eac72792..1d482c9cd9a 100644 --- a/hotfix/hotfix_generate_test.py +++ b/hotfix/hotfix_generate_test.py @@ -80,6 +80,11 @@ def test_config_modules_are_independently_hotfixable(self): "provisionConfigsKubelet", "GetCSEConfigKubeletScriptFilepath", ), + ( + "cse_config_chrony.sh", + "provisionConfigsChrony", + "GetCSEConfigChronyScriptFilepath", + ), ) artifacts = str(repository / hotfix_generate.ARTIFACTS_DIR) hotfix_generate.validate_source_mappings() @@ -120,6 +125,7 @@ def test_config_refactor_hotfix_selects_parent_and_new_modules(self): "cse_config_gpu.sh", "cse_config_localdns.sh", "cse_config_kubelet.sh", + "cse_config_chrony.sh", ) expected = {hotfix_generate.SOURCE_TO_VARKEY[source] for source in sources} result = subprocess.CompletedProcess( diff --git a/parts/linux/cloud-init/artifacts/cse_cmd.sh b/parts/linux/cloud-init/artifacts/cse_cmd.sh index 8291358daea..f436a57145d 100644 --- a/parts/linux/cloud-init/artifacts/cse_cmd.sh +++ b/parts/linux/cloud-init/artifacts/cse_cmd.sh @@ -18,18 +18,7 @@ fi; {{end}} INIT_AKS_CLOUD_FILEPATH="{{GetInitAKSCloudFilepath}}"; if [ -f "${INIT_AKS_CLOUD_FILEPATH}" ]; then - PRE_PROVISION_ONLY="{{GetPreProvisionOnly}}" REPO_DEPOT_ENDPOINT="{{AKSCustomCloudRepoDepotEndpoint}}" LOCATION={{GetVariable "location"}} "${INIT_AKS_CLOUD_FILEPATH}" >> /var/log/azure/cluster-provision.log 2>&1; - initAKSCloudExitCode=$?; - if [ "$initAKSCloudExitCode" -eq 246 ]; then - echo "Chrony configuration failed; init-aks-cloud failed with exit code ${initAKSCloudExitCode}" >> ${PROVISION_OUTPUT}; - exit ${initAKSCloudExitCode}; - elif [ "$initAKSCloudExitCode" -eq 245 ]; then - echo "NTP not reachable; init-aks-cloud failed with NTP synchronization error code ${initAKSCloudExitCode}" >> ${PROVISION_OUTPUT}; - exit ${initAKSCloudExitCode}; - elif [ "$initAKSCloudExitCode" -eq 244 ]; then - echo "Unable to determine confidential VM platform; init-aks-cloud failed with exit code ${initAKSCloudExitCode}" >> ${PROVISION_OUTPUT}; - exit ${initAKSCloudExitCode}; - fi; + REPO_DEPOT_ENDPOINT="{{AKSCustomCloudRepoDepotEndpoint}}" LOCATION={{GetVariable "location"}} "${INIT_AKS_CLOUD_FILEPATH}" >> /var/log/azure/cluster-provision.log 2>&1; fi; {{/* Keep the environment assignments below contiguous through the nohup invocation at the end of this file. */ -}} {{/* The CSE command is flattened into one shell command, so all assignments below are passed to nohup. */ -}} @@ -144,6 +133,7 @@ CSE_CONFIG_LOCALDNS_FILEPATH="{{GetCSEConfigLocalDNSScriptFilepath}}" CSE_CONFIG_KUBELET_FILEPATH="{{GetCSEConfigKubeletScriptFilepath}}" CSE_CONFIG_NETWORK_FILEPATH="{{GetCSEConfigNetworkScriptFilepath}}" CSE_CONFIG_ADDONS_FILEPATH="{{GetCSEConfigAddonsScriptFilepath}}" +CSE_CONFIG_CHRONY_FILEPATH="{{GetCSEConfigChronyScriptFilepath}}" AZURE_PRIVATE_REGISTRY_SERVER="{{GetPrivateAzureRegistryServer}}" HAS_CUSTOM_SEARCH_DOMAIN="{{HasCustomSearchDomain}}" CUSTOM_SEARCH_DOMAIN_FILEPATH="{{GetCustomSearchDomainsCSEScriptFilepath}}" diff --git a/parts/linux/cloud-init/artifacts/cse_config.sh b/parts/linux/cloud-init/artifacts/cse_config.sh index 0d64344d7b4..aca0e4db673 100755 --- a/parts/linux/cloud-init/artifacts/cse_config.sh +++ b/parts/linux/cloud-init/artifacts/cse_config.sh @@ -8,6 +8,7 @@ source "${CSE_CONFIG_LOCALDNS_FILEPATH:-${BASH_SOURCE[0]%.sh}_localdns.sh}" source "${CSE_CONFIG_KUBELET_FILEPATH:-${BASH_SOURCE[0]%.sh}_kubelet.sh}" source "${CSE_CONFIG_NETWORK_FILEPATH:-${BASH_SOURCE[0]%.sh}_network.sh}" source "${CSE_CONFIG_ADDONS_FILEPATH:-${BASH_SOURCE[0]%.sh}_addons.sh}" +source "${CSE_CONFIG_CHRONY_FILEPATH:-${BASH_SOURCE[0]%.sh}_chrony.sh}" configureAdminUser(){ chage -E -1 -I -1 -m 0 -M 99999 "${ADMINUSER}" diff --git a/parts/linux/cloud-init/artifacts/cse_config_chrony.sh b/parts/linux/cloud-init/artifacts/cse_config_chrony.sh new file mode 100644 index 00000000000..b46f49c394d --- /dev/null +++ b/parts/linux/cloud-init/artifacts/cse_config_chrony.sh @@ -0,0 +1,271 @@ +#!/bin/bash + +chrony_emit_event() { + local task=$1 + local message=$2 + local level=${3:-Informational} + local events_file_name + local timestamp + local json_string + + events_file_name=$(date +%s%3N) + timestamp=$(date +"%F %T.%3N") + json_string=$(jq -n \ + --arg Timestamp "${timestamp}" \ + --arg OperationId "${timestamp}" \ + --arg Version "1.23" \ + --arg TaskName "${task}" \ + --arg EventLevel "${level}" \ + --arg Message "${message}" \ + --arg EventPid "0" \ + --arg EventTid "0" \ + '{Timestamp: $Timestamp, OperationId: $OperationId, Version: $Version, TaskName: $TaskName, EventLevel: $EventLevel, Message: $Message, EventPid: $EventPid, EventTid: $EventTid}' + ) + + mkdir -p "${EVENTS_LOGGING_DIR}" + echo "${json_string}" > "${EVENTS_LOGGING_DIR}${events_file_name}.json" +} + +is_ubuntu_2604_cvm() { + [ "$OS" = "$UBUNTU_OS_NAME" ] || return 1 + [ "${OS_VERSION:-}" = "26.04" ] || return 1 + + case "$(uname -r)" in + *-azure-fde*) return 0 ;; + *) return 1 ;; + esac +} + +should_configure_ubuntu_2604_cvm_time_sync() { + [ "${PRE_PROVISION_ONLY:-false}" != "true" ] || return 1 + is_ubuntu_2604_cvm +} + +detect_confidential_vm_platform() { + local platform + + if ! platform="$(systemd-detect-virt --cvm 2>/dev/null)"; then + return 1 + fi + + case "$platform" in + sev-snp|tdx) + echo "$platform" + ;; + *) + return 1 + ;; + esac +} + +ubuntu_ntp_pools() { + cat <<'EOF' +pool ntp.ubuntu.com iburst maxsources 4 +pool 0.ubuntu.pool.ntp.org iburst maxsources 1 +pool 1.ubuntu.pool.ntp.org iburst maxsources 1 +pool 2.ubuntu.pool.ntp.org iburst maxsources 2 +EOF +} + +configure_chrony() { + local time_sources="${1:-refclock PHC /dev/ptp0 poll 3 dpoll -2 offset 0}" + local chrony_conf="${CHRONY_CONF:-/etc/chrony/chrony.conf}" + local timesyncd_load_state + local chrony_failed=0 + + if [ "$OS" = "$UBUNTU_OS_NAME" ]; then + timesyncd_load_state="$(systemctl show -p LoadState --value systemd-timesyncd 2>/dev/null || true)" + if [ "$timesyncd_load_state" = "not-found" ]; then + echo "systemd-timesyncd is removed, no need to disable" + else + if ! systemctl stop systemd-timesyncd; then + echo "ERROR: failed to stop systemd-timesyncd" >&2 + chrony_failed=1 + fi + if ! systemctl disable systemd-timesyncd; then + echo "ERROR: failed to disable systemd-timesyncd" >&2 + chrony_failed=1 + fi + fi + + if [ ! -e "$chrony_conf" ]; then + if ! apt-get update; then + echo "ERROR: failed to update package metadata before installing Chrony" >&2 + chrony_failed=1 + fi + if ! apt-get install chrony -y; then + echo "ERROR: failed to install Chrony" >&2 + chrony_failed=1 + fi + fi + elif [ "$OS" = "$FLATCAR_OS_NAME" ]; then + if ! rm -f "$chrony_conf"; then + echo "ERROR: failed to remove the existing Flatcar Chrony configuration" >&2 + chrony_failed=1 + fi + fi + + if ! cat > "$chrony_conf" <&2 + chrony_failed=1 + fi + + if [ "$OS" = "$UBUNTU_OS_NAME" ]; then + if ! systemctl restart chrony; then + echo "ERROR: failed to restart Chrony" >&2 + chrony_failed=1 + fi + elif [ "$OS" = "$FLATCAR_OS_NAME" ]; then + if ! systemctl restart chronyd; then + echo "ERROR: failed to restart chronyd" >&2 + chrony_failed=1 + fi + fi + + return "$chrony_failed" +} + +configure_mariner_azurelinux_chrony() { + local chrony_conf="${CHRONY_CONF:-/etc/chrony.conf}" + + cat > "$chrony_conf" <<'EOF' +# This directive specify the location of the file containing ID/key pairs for +# NTP authentication. +keyfile /etc/chrony.keys + +# This directive specify the file into which chronyd will store the rate +# information. +driftfile /var/lib/chrony/drift + +# Uncomment the following line to turn logging on. +#log tracking measurements statistics + +# Log files location. +logdir /var/log/chrony + +# Stop bad estimates upsetting machine clock. +maxupdateskew 100.0 + +# This directive enables kernel synchronisation (every 11 minutes) of the +# real-time clock. Note that it can’t be used along with the 'rtcfile' directive. +rtcsync + +# Settings come from: https://docs.microsoft.com/en-us/azure/virtual-machines/linux/time-sync +refclock PHC /dev/ptp0 poll 3 dpoll -2 offset 0 +makestep 1.0 -1 +EOF + + systemctl restart chronyd +} + +verify_chrony_ntp_sync() { + local max_attempts=12 + local retry_interval_seconds=5 + + if chronyc waitsync "$max_attempts" 0 0 "$retry_interval_seconds"; then + echo "NTP synchronization confirmed through the Ubuntu NTP pools" + chrony_emit_event "AKS.CSE.chrony.ntpSynchronized" "NTP synchronization confirmed through the Ubuntu NTP pools" + return 0 + fi + + echo "ERROR: NTP not reachable; Chrony did not synchronize" >&2 + chrony_emit_event "AKS.CSE.chrony.ntpUnavailable" "NTP not reachable after ${max_attempts} synchronization checks; failing provisioning" "Error" + echo "Chrony source diagnostics:" >&2 + chronyc sources -v >&2 || echo "ERROR: unable to retrieve Chrony source diagnostics" >&2 + echo "Chrony tracking diagnostics:" >&2 + chronyc tracking >&2 || echo "ERROR: unable to retrieve Chrony tracking diagnostics" >&2 + return "$ERR_NTP_UNREACHABLE" +} + +configure_ubuntu_2604_cvm_time_sync() { + local platform + local ntp_pools + + if ! platform="$(detect_confidential_vm_platform)"; then + echo "ERROR: unable to determine Ubuntu 26.04 CVM platform with systemd-detect-virt --cvm" >&2 + chrony_emit_event "AKS.CSE.chrony.platformDetectionFailed" "Unable to distinguish SEV-SNP from TDX using systemd-detect-virt --cvm" "Error" + return "$ERR_CVM_PLATFORM_DETECTION_FAIL" + fi + + case "$platform" in + sev-snp) + echo "AMD SEV-SNP detected; preserving the existing Hyper-V PHC Chrony configuration" + chrony_emit_event "AKS.CSE.chrony.usingPHC" "AMD SEV-SNP detected; preserving the existing /dev/ptp0 PHC configuration" + if ! configure_chrony; then + echo "ERROR: failed to configure Chrony with the Hyper-V PHC source for AMD SEV-SNP" >&2 + chrony_emit_event "AKS.CSE.chrony.configurationFailed" "Failed to configure Chrony with the Hyper-V PHC source for AMD SEV-SNP" "Error" + return "$ERR_CHRONY_CONFIG_FAIL" + fi + ;; + tdx) + echo "Intel TDX detected; configuring Chrony to use the Ubuntu NTP pools" + chrony_emit_event "AKS.CSE.chrony.usingNTP" "Intel TDX detected; using only the approved Ubuntu NTP pools" + ntp_pools="$(ubuntu_ntp_pools)" + if ! configure_chrony "$ntp_pools"; then + echo "ERROR: failed to configure Chrony with the Ubuntu NTP pools for Intel TDX" >&2 + chrony_emit_event "AKS.CSE.chrony.configurationFailed" "Failed to configure Chrony with the Ubuntu NTP pools for Intel TDX" "Error" + return "$ERR_CHRONY_CONFIG_FAIL" + fi + verify_chrony_ntp_sync + ;; + esac +} + +configureChrony() { + if isACL "$OS" "$OS_VARIANT"; then + echo "Skipping chrony configuration for ACL (PTP clock baked into chronyd, no external NTP sources)" + elif isMarinerOrAzureLinux "$OS"; then + configure_mariner_azurelinux_chrony || true + elif should_configure_ubuntu_2604_cvm_time_sync; then + configure_ubuntu_2604_cvm_time_sync + else + configure_chrony || true + fi +} diff --git a/parts/linux/cloud-init/artifacts/cse_helpers.sh b/parts/linux/cloud-init/artifacts/cse_helpers.sh index 7e45cd1f0e8..13edf70f0d8 100755 --- a/parts/linux/cloud-init/artifacts/cse_helpers.sh +++ b/parts/linux/cloud-init/artifacts/cse_helpers.sh @@ -162,7 +162,9 @@ ERR_AKS_NODE_CONTROLLER_ERROR=240 # Generic error in AKS Node Controller ERR_AZNFS_RPM_DOWNLOAD_TIMEOUT=241 # Timeout downloading aznfs RPM from PMC ERR_AZNFS_INSTALL_FAIL=242 # Failed to install aznfs RPM package ERR_SECONDARY_NIC_CONFIG_FAIL=243 # Error configuring secondary NIC network interface -# Exit codes 244-246 are reserved by init-aks-cloud.sh and propagated by cse_cmd.sh. +ERR_CVM_PLATFORM_DETECTION_FAIL=244 # Unable to distinguish SEV-SNP from TDX +ERR_NTP_UNREACHABLE=245 # Chrony could not synchronize with the configured NTP pools +ERR_CHRONY_CONFIG_FAIL=246 # Chrony could not be configured for the detected CVM platform # ----------------------------------------------------------------------------- # This probably wasn't launched via a login shell, so ensure the PATH is correct. diff --git a/parts/linux/cloud-init/artifacts/cse_main.sh b/parts/linux/cloud-init/artifacts/cse_main.sh index ae432248934..4f3820af2e4 100755 --- a/parts/linux/cloud-init/artifacts/cse_main.sh +++ b/parts/linux/cloud-init/artifacts/cse_main.sh @@ -748,6 +748,27 @@ function nodePrep { # In typical deployments, both stages run sequentially during node provisioning. # For VHD image creation workflows, only basePrep runs initially, and nodePrep runs later # when nodes are created from that VHD image. +chronyExitCode=0 +logs_to_events "AKS.CSE.configureChrony" configureChrony || chronyExitCode=$? +if [ "$chronyExitCode" -ne 0 ]; then + case "$chronyExitCode" in + "$ERR_CVM_PLATFORM_DETECTION_FAIL") + chronyErrorMessage="Unable to determine confidential VM platform; Chrony configuration failed with exit code ${chronyExitCode}" + ;; + "$ERR_NTP_UNREACHABLE") + chronyErrorMessage="NTP not reachable; Chrony synchronization failed with exit code ${chronyExitCode}" + ;; + "$ERR_CHRONY_CONFIG_FAIL") + chronyErrorMessage="Chrony configuration failed with exit code ${chronyExitCode}" + ;; + *) + chronyErrorMessage="Unexpected Chrony configuration failure with exit code ${chronyExitCode}" + ;; + esac + echo "$chronyErrorMessage" | tee -a /var/log/azure/cluster-provision-cse-output.log >&2 + exit "$chronyExitCode" +fi + if [ ! -f /opt/azure/containers/base_prep.complete ]; then basePrep else diff --git a/parts/linux/cloud-init/artifacts/init-aks-cloud.sh b/parts/linux/cloud-init/artifacts/init-aks-cloud.sh index 538598ddc13..8056193085e 100644 --- a/parts/linux/cloud-init/artifacts/init-aks-cloud.sh +++ b/parts/linux/cloud-init/artifacts/init-aks-cloud.sh @@ -103,10 +103,6 @@ IS_ACL=0 IS_MARINER=0 IS_AZURELINUX=0 -ERR_CVM_PLATFORM_DETECTION_FAIL=244 # Unable to distinguish SEV-SNP from TDX -ERR_NTP_UNREACHABLE=245 # Chrony could not synchronize with the configured NTP pools -ERR_CHRONY_CONFIG_FAIL=246 # Chrony could not be configured for the detected CVM platform - # http://168.63.129.16 is a constant for the host's wireserver endpoint. WIRESERVER_ENDPOINT="http://168.63.129.16" @@ -542,205 +538,6 @@ function determine_cert_endpoint_mode { echo "$mode" } -function is_ubuntu_2604_cvm { - [ "$IS_UBUNTU" -eq 1 ] || return 1 - [ "${VERSION_ID:-}" = "26.04" ] || return 1 - - case "$(uname -r)" in - *-azure-fde*) return 0 ;; - *) return 1 ;; - esac -} - -function should_configure_ubuntu_2604_cvm_time_sync { - [ "${PRE_PROVISION_ONLY:-false}" != "true" ] || return 1 - is_ubuntu_2604_cvm -} - -function detect_confidential_vm_platform { - local platform - - if ! platform="$(systemd-detect-virt --cvm 2>/dev/null)"; then - return 1 - fi - - case "$platform" in - sev-snp|tdx) - echo "$platform" - ;; - *) - return 1 - ;; - esac -} - -function ubuntu_ntp_pools { - cat <<'EOF' -pool ntp.ubuntu.com iburst maxsources 4 -pool 0.ubuntu.pool.ntp.org iburst maxsources 1 -pool 1.ubuntu.pool.ntp.org iburst maxsources 1 -pool 2.ubuntu.pool.ntp.org iburst maxsources 2 -EOF -} - -function configure_chrony { - local time_sources="${1:-refclock PHC /dev/ptp0 poll 3 dpoll -2 offset 0}" - local chrony_conf="${CHRONY_CONF:-/etc/chrony/chrony.conf}" - local timesyncd_load_state - local chrony_failed=0 - - if [ "$IS_UBUNTU" -eq 1 ]; then - timesyncd_load_state="$(systemctl show -p LoadState --value systemd-timesyncd 2>/dev/null || true)" - if [ "$timesyncd_load_state" = "not-found" ]; then - echo "systemd-timesyncd is removed, no need to disable" - else - if ! systemctl stop systemd-timesyncd; then - echo "ERROR: failed to stop systemd-timesyncd" >&2 - chrony_failed=1 - fi - if ! systemctl disable systemd-timesyncd; then - echo "ERROR: failed to disable systemd-timesyncd" >&2 - chrony_failed=1 - fi - fi - - if [ ! -e "$chrony_conf" ]; then - if ! apt-get update; then - echo "ERROR: failed to update package metadata before installing Chrony" >&2 - chrony_failed=1 - fi - if ! apt-get install chrony -y; then - echo "ERROR: failed to install Chrony" >&2 - chrony_failed=1 - fi - fi - elif [ "$IS_FLATCAR" -eq 1 ]; then - if ! rm -f "$chrony_conf"; then - echo "ERROR: failed to remove the existing Flatcar Chrony configuration" >&2 - chrony_failed=1 - fi - fi - - if ! cat > "$chrony_conf" <&2 - chrony_failed=1 - fi - - if [ "$IS_UBUNTU" -eq 1 ]; then - if ! systemctl restart chrony; then - echo "ERROR: failed to restart Chrony" >&2 - chrony_failed=1 - fi - elif [ "$IS_FLATCAR" -eq 1 ]; then - if ! systemctl restart chronyd; then - echo "ERROR: failed to restart chronyd" >&2 - chrony_failed=1 - fi - fi - - return "$chrony_failed" -} - -function verify_chrony_ntp_sync { - local max_attempts=12 - local retry_interval_seconds=5 - - if chronyc waitsync "$max_attempts" 0 0 "$retry_interval_seconds"; then - echo "NTP synchronization confirmed through the Ubuntu NTP pools" - emit_event "AKS.CSE.chrony.ntpSynchronized" "NTP synchronization confirmed through the Ubuntu NTP pools" - return 0 - fi - - echo "ERROR: NTP not reachable; Chrony did not synchronize" >&2 - emit_event "AKS.CSE.chrony.ntpUnavailable" "NTP not reachable after ${max_attempts} synchronization checks; failing provisioning" "Error" - echo "Chrony source diagnostics:" >&2 - chronyc sources -v >&2 || echo "ERROR: unable to retrieve Chrony source diagnostics" >&2 - echo "Chrony tracking diagnostics:" >&2 - chronyc tracking >&2 || echo "ERROR: unable to retrieve Chrony tracking diagnostics" >&2 - return "$ERR_NTP_UNREACHABLE" -} - -function configure_ubuntu_2604_cvm_time_sync { - local platform - local ntp_pools - - if ! platform="$(detect_confidential_vm_platform)"; then - echo "ERROR: unable to determine Ubuntu 26.04 CVM platform with systemd-detect-virt --cvm" >&2 - emit_event "AKS.CSE.chrony.platformDetectionFailed" "Unable to distinguish SEV-SNP from TDX using systemd-detect-virt --cvm" "Error" - return "$ERR_CVM_PLATFORM_DETECTION_FAIL" - fi - - case "$platform" in - sev-snp) - echo "AMD SEV-SNP detected; preserving the existing Hyper-V PHC Chrony configuration" - emit_event "AKS.CSE.chrony.usingPHC" "AMD SEV-SNP detected; preserving the existing /dev/ptp0 PHC configuration" - if ! configure_chrony; then - echo "ERROR: failed to configure Chrony with the Hyper-V PHC source for AMD SEV-SNP" >&2 - emit_event "AKS.CSE.chrony.configurationFailed" "Failed to configure Chrony with the Hyper-V PHC source for AMD SEV-SNP" "Error" - return "$ERR_CHRONY_CONFIG_FAIL" - fi - ;; - tdx) - echo "Intel TDX detected; configuring Chrony to use the Ubuntu NTP pools" - emit_event "AKS.CSE.chrony.usingNTP" "Intel TDX detected; using only the approved Ubuntu NTP pools" - ntp_pools="$(ubuntu_ntp_pools)" - if ! configure_chrony "$ntp_pools"; then - echo "ERROR: failed to configure Chrony with the Ubuntu NTP pools for Intel TDX" >&2 - emit_event "AKS.CSE.chrony.configurationFailed" "Failed to configure Chrony with the Ubuntu NTP pools for Intel TDX" "Error" - return "$ERR_CHRONY_CONFIG_FAIL" - fi - verify_chrony_ntp_sync - ;; - esac -} - # shellcheck disable=SC2317 ${__SOURCED__:+return} @@ -771,17 +568,6 @@ fi echo "Running on $NAME" -ubuntu_2604_cvm_chrony_configured=0 -if [ "$action" = "init" ] && should_configure_ubuntu_2604_cvm_time_sync; then - configure_ubuntu_2604_cvm_time_sync - chrony_result=$? - if [ "$chrony_result" -ne 0 ]; then - exit "$chrony_result" - fi - ubuntu_2604_cvm_chrony_configured=1 -fi - - # Certificate refresh behavior summary: # - legacy mode directly attempts certificate download from wireserver and only in ussec and usnat regions. # - rcv1p mode first checks IsOptedInForRootCerts, then downloads only when opted in. @@ -934,44 +720,4 @@ elif [ "$IS_MARINER" -eq 1 ] || [ "$IS_AZURELINUX" -eq 1 ]; then fi fi -# Disable systemd-timesyncd and install chrony and uses local time source -# ACL has PTP clock config compiled into chronyd with no config file or sourcedir directives, -# so it uses only the local PTP clock and has no DHCP-injectable NTP sources. -if [ "$IS_ACL" -eq 1 ]; then - echo "Skipping chrony configuration for ACL (PTP clock baked into chronyd, no external NTP sources)" -elif [ "$IS_MARINER" -eq 1 ] || [ "$IS_AZURELINUX" -eq 1 ]; then - cat > /etc/chrony.conf < /etc/chrony.conf < "${SSL_CERTS_DIR}/ca-certificates.crt" # shellcheck disable=SC1090 __SOURCED__=1 . "./parts/linux/cloud-init/artifacts/init-aks-cloud.sh" + UBUNTU_OS_NAME="UBUNTU" + FLATCAR_OS_NAME="FLATCAR" + ERR_CVM_PLATFORM_DETECTION_FAIL=244 + ERR_NTP_UNREACHABLE=245 + ERR_CHRONY_CONFIG_FAIL=246 + # shellcheck disable=SC1091 + . "./parts/linux/cloud-init/artifacts/cse_config_chrony.sh" } cleanup() { @@ -245,8 +251,8 @@ EOF Describe 'Ubuntu 26.04 CVM Chrony configuration' setup_chrony_test() { export CHRONY_CONF="${TEST_DIR}/chrony.conf" - IS_UBUNTU=1 - VERSION_ID="26.04" + OS="$UBUNTU_OS_NAME" + OS_VERSION="26.04" touch "$CHRONY_CONF" } @@ -255,8 +261,8 @@ EOF End It 'scopes the platform-specific behavior to Ubuntu 26.04 FDE images' - IS_UBUNTU=1 - VERSION_ID="26.04" + OS="$UBUNTU_OS_NAME" + OS_VERSION="26.04" Mock uname echo "7.0.0-1011-azure-fde" End @@ -266,8 +272,8 @@ EOF End It 'does not select an Ubuntu 26.04 non-FDE image' - IS_UBUNTU=1 - VERSION_ID="26.04" + OS="$UBUNTU_OS_NAME" + OS_VERSION="26.04" Mock uname echo "7.0.0-1011-azure" End @@ -277,8 +283,8 @@ EOF End It 'does not select another Ubuntu release even when it has an FDE kernel' - IS_UBUNTU=1 - VERSION_ID="24.04" + OS="$UBUNTU_OS_NAME" + OS_VERSION="24.04" Mock uname echo "6.8.0-1065-azure-fde" End @@ -288,8 +294,8 @@ EOF End It 'skips platform-specific time sync during pre-provision image preparation' - IS_UBUNTU=1 - VERSION_ID="26.04" + OS="$UBUNTU_OS_NAME" + OS_VERSION="26.04" PRE_PROVISION_ONLY="true" Mock uname echo "7.0.0-1011-azure-fde" @@ -300,8 +306,8 @@ EOF End It 'selects platform-specific time sync when provisioning the real node' - IS_UBUNTU=1 - VERSION_ID="26.04" + OS="$UBUNTU_OS_NAME" + OS_VERSION="26.04" PRE_PROVISION_ONLY="false" Mock uname echo "7.0.0-1011-azure-fde" @@ -313,7 +319,7 @@ EOF It 'preserves the PHC default for another Ubuntu release' setup_chrony_test - VERSION_ID="24.04" + OS_VERSION="24.04" When call configure_chrony The output should include "systemctl restart chrony" @@ -356,7 +362,7 @@ EOF Mock detect_confidential_vm_platform return 1 End - Mock emit_event + Mock chrony_emit_event echo "event: $*" >&2 End @@ -371,7 +377,7 @@ EOF Mock detect_confidential_vm_platform echo "sev-snp" End - Mock emit_event + Mock chrony_emit_event echo "event: $*" End @@ -389,7 +395,7 @@ EOF Mock configure_chrony return 1 End - Mock emit_event + Mock chrony_emit_event echo "event: $*" >&2 End @@ -418,7 +424,7 @@ EOF Mock verify_chrony_ntp_sync echo "verified NTP synchronization" End - Mock emit_event + Mock chrony_emit_event echo "event: $*" End @@ -516,7 +522,7 @@ EOF Mock verify_chrony_ntp_sync echo "unexpected NTP verification" End - Mock emit_event + Mock chrony_emit_event echo "event: $*" >&2 End @@ -531,7 +537,7 @@ EOF Mock chronyc echo "$*" End - Mock emit_event + Mock chrony_emit_event echo "event: $*" End @@ -556,7 +562,7 @@ EOF ;; esac End - Mock emit_event + Mock chrony_emit_event echo "event: $*" >&2 End @@ -581,7 +587,7 @@ EOF Mock verify_chrony_ntp_sync exit 245 End - Mock emit_event + Mock chrony_emit_event : End diff --git a/vhdbuilder/packer/README.md b/vhdbuilder/packer/README.md index 1d23f04ec18..99bca41d5dc 100644 --- a/vhdbuilder/packer/README.md +++ b/vhdbuilder/packer/README.md @@ -27,11 +27,12 @@ Goal1: remove mariner workflow so things will be simplified. - `cse_config_kubelet.sh` -> `provision_configs_kubelet.sh` - `cse_config_network.sh` -> `provision_configs_network.sh` - `cse_config_addons.sh` -> `provision_configs_addons.sh` (autoscaler, ACI connector, Azure Policy) +- `cse_config_chrony.sh` -> `provision_configs_chrony.sh` `cse_cmd.sh` and the ANC parser provide their paths through `CSE_CONFIG_GPU_FILEPATH`, `CSE_CONFIG_LOCALDNS_FILEPATH`, `CSE_CONFIG_KUBELET_FILEPATH`, `CSE_CONFIG_NETWORK_FILEPATH`, and -`CSE_CONFIG_ADDONS_FILEPATH`; the parent sources each explicitly. +`CSE_CONFIG_ADDONS_FILEPATH`, and `CSE_CONFIG_CHRONY_FILEPATH`; the parent sources each explicitly. If a variable is unset or empty, its path defaults to the corresponding sibling of the sourced parent (`provision_configs_*.sh` on-node, `cse_config_*.sh` in the source tree). Explicit paths take precedence. @@ -51,7 +52,7 @@ unless a hotfix explicitly selects an override. When adding a module, also register its source-to-variable mapping in `hotfix/hotfix_generate.py`. The current keys are `provisionConfigsGPU`, `provisionConfigsLocalDNS`, `provisionConfigsKubelet`, `provisionConfigsNetwork`, -and `provisionConfigsAddons`. Each module can be +`provisionConfigsAddons`, and `provisionConfigsChrony`. Each module can be hotfixed independently; delivering this split to an older VHD requires the updated parent and all new modules together. Keep the immutable VHD baseline so subsequent hotfix payloads remain cumulative. diff --git a/vhdbuilder/packer/imagecustomizer/azlosguard/azlosguard.yml b/vhdbuilder/packer/imagecustomizer/azlosguard/azlosguard.yml index 115e44eb806..837d3dfce68 100644 --- a/vhdbuilder/packer/imagecustomizer/azlosguard/azlosguard.yml +++ b/vhdbuilder/packer/imagecustomizer/azlosguard/azlosguard.yml @@ -115,6 +115,9 @@ os: - source: /AgentBaker/parts/linux/cloud-init/artifacts/cse_config_addons.sh destination: /opt/azure/containers/provision_configs_addons.sh permissions: 744 + - source: /AgentBaker/parts/linux/cloud-init/artifacts/cse_config_chrony.sh + destination: /opt/azure/containers/provision_configs_chrony.sh + permissions: 744 - source: /AgentBaker/parts/linux/cloud-init/artifacts/cse_helpers.sh destination: /opt/azure/containers/provision_source.sh permissions: 744 diff --git a/vhdbuilder/packer/packer_source.sh b/vhdbuilder/packer/packer_source.sh index db70079a35c..73596ff4cd2 100644 --- a/vhdbuilder/packer/packer_source.sh +++ b/vhdbuilder/packer/packer_source.sh @@ -377,7 +377,7 @@ copyPackerFiles() { cpAndMode $CSE_CONFIG_SRC $CSE_CONFIG_DEST 0744 local config_module - for config_module in provision_configs_gpu.sh provision_configs_localdns.sh provision_configs_kubelet.sh provision_configs_network.sh provision_configs_addons.sh; do + for config_module in provision_configs_gpu.sh provision_configs_localdns.sh provision_configs_kubelet.sh provision_configs_network.sh provision_configs_addons.sh provision_configs_chrony.sh; do cpAndMode "/home/packer/${config_module}" "/opt/azure/containers/${config_module}" 0744 done diff --git a/vhdbuilder/packer/vhd-image-builder-acl-arm64.json b/vhdbuilder/packer/vhd-image-builder-acl-arm64.json index 949c37200ad..a015d56dd0f 100644 --- a/vhdbuilder/packer/vhd-image-builder-acl-arm64.json +++ b/vhdbuilder/packer/vhd-image-builder-acl-arm64.json @@ -187,6 +187,11 @@ "source": "parts/linux/cloud-init/artifacts/cse_config_addons.sh", "destination": "/home/packer/provision_configs_addons.sh" }, + { + "type": "file", + "source": "parts/linux/cloud-init/artifacts/cse_config_chrony.sh", + "destination": "/home/packer/provision_configs_chrony.sh" + }, { "type": "file", "source": "parts/linux/cloud-init/artifacts/cse_main.sh", diff --git a/vhdbuilder/packer/vhd-image-builder-acl.json b/vhdbuilder/packer/vhd-image-builder-acl.json index b1f3a08bd53..54e6c226836 100644 --- a/vhdbuilder/packer/vhd-image-builder-acl.json +++ b/vhdbuilder/packer/vhd-image-builder-acl.json @@ -187,6 +187,11 @@ "source": "parts/linux/cloud-init/artifacts/cse_config_addons.sh", "destination": "/home/packer/provision_configs_addons.sh" }, + { + "type": "file", + "source": "parts/linux/cloud-init/artifacts/cse_config_chrony.sh", + "destination": "/home/packer/provision_configs_chrony.sh" + }, { "type": "file", "source": "parts/linux/cloud-init/artifacts/cse_main.sh", diff --git a/vhdbuilder/packer/vhd-image-builder-arm64-gb.json b/vhdbuilder/packer/vhd-image-builder-arm64-gb.json index 488dd8566d2..93806f6e275 100644 --- a/vhdbuilder/packer/vhd-image-builder-arm64-gb.json +++ b/vhdbuilder/packer/vhd-image-builder-arm64-gb.json @@ -212,6 +212,11 @@ "source": "parts/linux/cloud-init/artifacts/cse_config_addons.sh", "destination": "/home/packer/provision_configs_addons.sh" }, + { + "type": "file", + "source": "parts/linux/cloud-init/artifacts/cse_config_chrony.sh", + "destination": "/home/packer/provision_configs_chrony.sh" + }, { "type": "file", "source": "parts/linux/cloud-init/artifacts/cse_main.sh", diff --git a/vhdbuilder/packer/vhd-image-builder-arm64-gen2.json b/vhdbuilder/packer/vhd-image-builder-arm64-gen2.json index 6f0818eb7c8..d8d885f3c96 100644 --- a/vhdbuilder/packer/vhd-image-builder-arm64-gen2.json +++ b/vhdbuilder/packer/vhd-image-builder-arm64-gen2.json @@ -197,6 +197,11 @@ "source": "parts/linux/cloud-init/artifacts/cse_config_addons.sh", "destination": "/home/packer/provision_configs_addons.sh" }, + { + "type": "file", + "source": "parts/linux/cloud-init/artifacts/cse_config_chrony.sh", + "destination": "/home/packer/provision_configs_chrony.sh" + }, { "type": "file", "source": "parts/linux/cloud-init/artifacts/cse_main.sh", diff --git a/vhdbuilder/packer/vhd-image-builder-base.json b/vhdbuilder/packer/vhd-image-builder-base.json index 2ef4ad93a34..0db20d389b7 100644 --- a/vhdbuilder/packer/vhd-image-builder-base.json +++ b/vhdbuilder/packer/vhd-image-builder-base.json @@ -200,6 +200,11 @@ "source": "parts/linux/cloud-init/artifacts/cse_config_addons.sh", "destination": "/home/packer/provision_configs_addons.sh" }, + { + "type": "file", + "source": "parts/linux/cloud-init/artifacts/cse_config_chrony.sh", + "destination": "/home/packer/provision_configs_chrony.sh" + }, { "type": "file", "source": "parts/linux/cloud-init/artifacts/cse_main.sh", diff --git a/vhdbuilder/packer/vhd-image-builder-cvm.json b/vhdbuilder/packer/vhd-image-builder-cvm.json index 08b3b6a3e8c..f9cef1df59e 100644 --- a/vhdbuilder/packer/vhd-image-builder-cvm.json +++ b/vhdbuilder/packer/vhd-image-builder-cvm.json @@ -204,6 +204,11 @@ "source": "parts/linux/cloud-init/artifacts/cse_config_addons.sh", "destination": "/home/packer/provision_configs_addons.sh" }, + { + "type": "file", + "source": "parts/linux/cloud-init/artifacts/cse_config_chrony.sh", + "destination": "/home/packer/provision_configs_chrony.sh" + }, { "type": "file", "source": "parts/linux/cloud-init/artifacts/cse_main.sh", diff --git a/vhdbuilder/packer/vhd-image-builder-mariner-arm64.json b/vhdbuilder/packer/vhd-image-builder-mariner-arm64.json index 1dbfc09b7f4..a1e2bc72117 100644 --- a/vhdbuilder/packer/vhd-image-builder-mariner-arm64.json +++ b/vhdbuilder/packer/vhd-image-builder-mariner-arm64.json @@ -196,6 +196,11 @@ "source": "parts/linux/cloud-init/artifacts/cse_config_addons.sh", "destination": "/home/packer/provision_configs_addons.sh" }, + { + "type": "file", + "source": "parts/linux/cloud-init/artifacts/cse_config_chrony.sh", + "destination": "/home/packer/provision_configs_chrony.sh" + }, { "type": "file", "source": "parts/linux/cloud-init/artifacts/cse_main.sh", diff --git a/vhdbuilder/packer/vhd-image-builder-mariner-cvm.json b/vhdbuilder/packer/vhd-image-builder-mariner-cvm.json index 9d3a8726ee1..56ba149bcd2 100644 --- a/vhdbuilder/packer/vhd-image-builder-mariner-cvm.json +++ b/vhdbuilder/packer/vhd-image-builder-mariner-cvm.json @@ -202,6 +202,11 @@ "source": "parts/linux/cloud-init/artifacts/cse_config_addons.sh", "destination": "/home/packer/provision_configs_addons.sh" }, + { + "type": "file", + "source": "parts/linux/cloud-init/artifacts/cse_config_chrony.sh", + "destination": "/home/packer/provision_configs_chrony.sh" + }, { "type": "file", "source": "parts/linux/cloud-init/artifacts/cse_main.sh", diff --git a/vhdbuilder/packer/vhd-image-builder-mariner.json b/vhdbuilder/packer/vhd-image-builder-mariner.json index 98ec2834ccf..a3c1d159dae 100644 --- a/vhdbuilder/packer/vhd-image-builder-mariner.json +++ b/vhdbuilder/packer/vhd-image-builder-mariner.json @@ -198,6 +198,11 @@ "source": "parts/linux/cloud-init/artifacts/cse_config_addons.sh", "destination": "/home/packer/provision_configs_addons.sh" }, + { + "type": "file", + "source": "parts/linux/cloud-init/artifacts/cse_config_chrony.sh", + "destination": "/home/packer/provision_configs_chrony.sh" + }, { "type": "file", "source": "parts/linux/cloud-init/artifacts/cse_main.sh", From a660743b5a538788b8d94dbc9fdd0f7912dd0be6 Mon Sep 17 00:00:00 2001 From: Zachary Bailey Date: Wed, 23 Sep 2026 15:53:26 -0400 Subject: [PATCH 11/14] fix --- .../cloud-init/artifacts/cse_config_chrony.sh | 33 ---------------- .../artifacts/init_aks_cloud_spec.sh | 38 ------------------- 2 files changed, 71 deletions(-) diff --git a/parts/linux/cloud-init/artifacts/cse_config_chrony.sh b/parts/linux/cloud-init/artifacts/cse_config_chrony.sh index b46f49c394d..0678da3f525 100644 --- a/parts/linux/cloud-init/artifacts/cse_config_chrony.sh +++ b/parts/linux/cloud-init/artifacts/cse_config_chrony.sh @@ -1,31 +1,5 @@ #!/bin/bash -chrony_emit_event() { - local task=$1 - local message=$2 - local level=${3:-Informational} - local events_file_name - local timestamp - local json_string - - events_file_name=$(date +%s%3N) - timestamp=$(date +"%F %T.%3N") - json_string=$(jq -n \ - --arg Timestamp "${timestamp}" \ - --arg OperationId "${timestamp}" \ - --arg Version "1.23" \ - --arg TaskName "${task}" \ - --arg EventLevel "${level}" \ - --arg Message "${message}" \ - --arg EventPid "0" \ - --arg EventTid "0" \ - '{Timestamp: $Timestamp, OperationId: $OperationId, Version: $Version, TaskName: $TaskName, EventLevel: $EventLevel, Message: $Message, EventPid: $EventPid, EventTid: $EventTid}' - ) - - mkdir -p "${EVENTS_LOGGING_DIR}" - echo "${json_string}" > "${EVENTS_LOGGING_DIR}${events_file_name}.json" -} - is_ubuntu_2604_cvm() { [ "$OS" = "$UBUNTU_OS_NAME" ] || return 1 [ "${OS_VERSION:-}" = "26.04" ] || return 1 @@ -211,12 +185,10 @@ verify_chrony_ntp_sync() { if chronyc waitsync "$max_attempts" 0 0 "$retry_interval_seconds"; then echo "NTP synchronization confirmed through the Ubuntu NTP pools" - chrony_emit_event "AKS.CSE.chrony.ntpSynchronized" "NTP synchronization confirmed through the Ubuntu NTP pools" return 0 fi echo "ERROR: NTP not reachable; Chrony did not synchronize" >&2 - chrony_emit_event "AKS.CSE.chrony.ntpUnavailable" "NTP not reachable after ${max_attempts} synchronization checks; failing provisioning" "Error" echo "Chrony source diagnostics:" >&2 chronyc sources -v >&2 || echo "ERROR: unable to retrieve Chrony source diagnostics" >&2 echo "Chrony tracking diagnostics:" >&2 @@ -230,27 +202,22 @@ configure_ubuntu_2604_cvm_time_sync() { if ! platform="$(detect_confidential_vm_platform)"; then echo "ERROR: unable to determine Ubuntu 26.04 CVM platform with systemd-detect-virt --cvm" >&2 - chrony_emit_event "AKS.CSE.chrony.platformDetectionFailed" "Unable to distinguish SEV-SNP from TDX using systemd-detect-virt --cvm" "Error" return "$ERR_CVM_PLATFORM_DETECTION_FAIL" fi case "$platform" in sev-snp) echo "AMD SEV-SNP detected; preserving the existing Hyper-V PHC Chrony configuration" - chrony_emit_event "AKS.CSE.chrony.usingPHC" "AMD SEV-SNP detected; preserving the existing /dev/ptp0 PHC configuration" if ! configure_chrony; then echo "ERROR: failed to configure Chrony with the Hyper-V PHC source for AMD SEV-SNP" >&2 - chrony_emit_event "AKS.CSE.chrony.configurationFailed" "Failed to configure Chrony with the Hyper-V PHC source for AMD SEV-SNP" "Error" return "$ERR_CHRONY_CONFIG_FAIL" fi ;; tdx) echo "Intel TDX detected; configuring Chrony to use the Ubuntu NTP pools" - chrony_emit_event "AKS.CSE.chrony.usingNTP" "Intel TDX detected; using only the approved Ubuntu NTP pools" ntp_pools="$(ubuntu_ntp_pools)" if ! configure_chrony "$ntp_pools"; then echo "ERROR: failed to configure Chrony with the Ubuntu NTP pools for Intel TDX" >&2 - chrony_emit_event "AKS.CSE.chrony.configurationFailed" "Failed to configure Chrony with the Ubuntu NTP pools for Intel TDX" "Error" return "$ERR_CHRONY_CONFIG_FAIL" fi verify_chrony_ntp_sync diff --git a/spec/parts/linux/cloud-init/artifacts/init_aks_cloud_spec.sh b/spec/parts/linux/cloud-init/artifacts/init_aks_cloud_spec.sh index b09a28292ba..17b57169bd4 100644 --- a/spec/parts/linux/cloud-init/artifacts/init_aks_cloud_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/init_aks_cloud_spec.sh @@ -362,13 +362,8 @@ EOF Mock detect_confidential_vm_platform return 1 End - Mock chrony_emit_event - echo "event: $*" >&2 - End - When call configure_ubuntu_2604_cvm_time_sync The error should include "unable to determine Ubuntu 26.04 CVM platform" - The error should include "AKS.CSE.chrony.platformDetectionFailed" The status should equal 244 End @@ -377,13 +372,8 @@ EOF Mock detect_confidential_vm_platform echo "sev-snp" End - Mock chrony_emit_event - echo "event: $*" - End - When call configure_ubuntu_2604_cvm_time_sync The output should include "preserving the existing Hyper-V PHC Chrony configuration" - The output should include "AKS.CSE.chrony.usingPHC" The contents of file "$CHRONY_CONF" should include "refclock PHC /dev/ptp0 poll 3 dpoll -2 offset 0" The status should be success End @@ -395,14 +385,9 @@ EOF Mock configure_chrony return 1 End - Mock chrony_emit_event - echo "event: $*" >&2 - End - When call configure_ubuntu_2604_cvm_time_sync The output should include "AMD SEV-SNP detected" The error should include "failed to configure Chrony with the Hyper-V PHC source" - The error should include "AKS.CSE.chrony.configurationFailed" The status should equal 246 End @@ -424,10 +409,6 @@ EOF Mock verify_chrony_ntp_sync echo "verified NTP synchronization" End - Mock chrony_emit_event - echo "event: $*" - End - When call configure_ubuntu_2604_cvm_time_sync The output should include "Intel TDX detected" The output should include "verified NTP synchronization" @@ -522,14 +503,9 @@ EOF Mock verify_chrony_ntp_sync echo "unexpected NTP verification" End - Mock chrony_emit_event - echo "event: $*" >&2 - End - When call configure_ubuntu_2604_cvm_time_sync The output should not include "unexpected NTP verification" The error should include "failed to configure Chrony with the Ubuntu NTP pools" - The error should include "AKS.CSE.chrony.configurationFailed" The status should equal 246 End @@ -537,14 +513,9 @@ EOF Mock chronyc echo "$*" End - Mock chrony_emit_event - echo "event: $*" - End - When call verify_chrony_ntp_sync The output should include "waitsync 12 0 0 5" The output should include "NTP synchronization confirmed through the Ubuntu NTP pools" - The output should include "AKS.CSE.chrony.ntpSynchronized" The status should be success End @@ -562,13 +533,8 @@ EOF ;; esac End - Mock chrony_emit_event - echo "event: $*" >&2 - End - When call verify_chrony_ntp_sync The error should include "NTP not reachable" - The error should include "AKS.CSE.chrony.ntpUnavailable" The error should include "mock Chrony sources" The error should include "mock Chrony tracking" The status should equal 245 @@ -587,10 +553,6 @@ EOF Mock verify_chrony_ntp_sync exit 245 End - Mock chrony_emit_event - : - End - When call configure_ubuntu_2604_cvm_time_sync The output should include "Intel TDX detected" The status should equal 245 From 63af5be32fafe8bec30ccca03ee862ec56228962 Mon Sep 17 00:00:00 2001 From: Zachary Bailey Date: Wed, 23 Sep 2026 16:42:40 -0400 Subject: [PATCH 12/14] fix --- parts/linux/cloud-init/artifacts/init-aks-cloud.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/parts/linux/cloud-init/artifacts/init-aks-cloud.sh b/parts/linux/cloud-init/artifacts/init-aks-cloud.sh index 8056193085e..a94383e8ad9 100644 --- a/parts/linux/cloud-init/artifacts/init-aks-cloud.sh +++ b/parts/linux/cloud-init/artifacts/init-aks-cloud.sh @@ -541,8 +541,6 @@ function determine_cert_endpoint_mode { # shellcheck disable=SC2317 ${__SOURCED__:+return} -action=${1:-init} - # shellcheck disable=SC3010 if [[ -f /etc/os-release ]]; then . /etc/os-release @@ -633,6 +631,7 @@ fi # Action values: # - init (default): full provisioning path # - ca-refresh : periodic refresh path; location is passed as arg to avoid env dependency +action=${1:-init} if [ "$action" = "ca-refresh" ] || [ "$install_ca_refresh_schedule" -eq 0 ]; then exit 0 fi From 603e4845860d638f05fa486ed6079f542d7b8e95 Mon Sep 17 00:00:00 2001 From: Zachary Bailey Date: Wed, 23 Sep 2026 17:19:13 -0400 Subject: [PATCH 13/14] fix --- parts/linux/cloud-init/artifacts/cse_config_chrony.sh | 3 +-- parts/linux/cloud-init/artifacts/cse_helpers.sh | 2 +- parts/linux/cloud-init/artifacts/cse_main.sh | 2 +- .../linux/cloud-init/artifacts/init_aks_cloud_spec.sh | 7 ++++--- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/parts/linux/cloud-init/artifacts/cse_config_chrony.sh b/parts/linux/cloud-init/artifacts/cse_config_chrony.sh index 0678da3f525..40f6a208871 100644 --- a/parts/linux/cloud-init/artifacts/cse_config_chrony.sh +++ b/parts/linux/cloud-init/artifacts/cse_config_chrony.sh @@ -209,8 +209,7 @@ configure_ubuntu_2604_cvm_time_sync() { sev-snp) echo "AMD SEV-SNP detected; preserving the existing Hyper-V PHC Chrony configuration" if ! configure_chrony; then - echo "ERROR: failed to configure Chrony with the Hyper-V PHC source for AMD SEV-SNP" >&2 - return "$ERR_CHRONY_CONFIG_FAIL" + echo "WARNING: failed to reapply the Hyper-V PHC Chrony configuration for AMD SEV-SNP; continuing provisioning" >&2 fi ;; tdx) diff --git a/parts/linux/cloud-init/artifacts/cse_helpers.sh b/parts/linux/cloud-init/artifacts/cse_helpers.sh index 13edf70f0d8..69d2659705c 100755 --- a/parts/linux/cloud-init/artifacts/cse_helpers.sh +++ b/parts/linux/cloud-init/artifacts/cse_helpers.sh @@ -164,7 +164,7 @@ ERR_AZNFS_INSTALL_FAIL=242 # Failed to install aznfs RPM package ERR_SECONDARY_NIC_CONFIG_FAIL=243 # Error configuring secondary NIC network interface ERR_CVM_PLATFORM_DETECTION_FAIL=244 # Unable to distinguish SEV-SNP from TDX ERR_NTP_UNREACHABLE=245 # Chrony could not synchronize with the configured NTP pools -ERR_CHRONY_CONFIG_FAIL=246 # Chrony could not be configured for the detected CVM platform +ERR_CHRONY_CONFIG_FAIL=246 # Chrony could not be configured for Intel TDX # ----------------------------------------------------------------------------- # This probably wasn't launched via a login shell, so ensure the PATH is correct. diff --git a/parts/linux/cloud-init/artifacts/cse_main.sh b/parts/linux/cloud-init/artifacts/cse_main.sh index 4f3820af2e4..1ded6e4965b 100755 --- a/parts/linux/cloud-init/artifacts/cse_main.sh +++ b/parts/linux/cloud-init/artifacts/cse_main.sh @@ -759,7 +759,7 @@ if [ "$chronyExitCode" -ne 0 ]; then chronyErrorMessage="NTP not reachable; Chrony synchronization failed with exit code ${chronyExitCode}" ;; "$ERR_CHRONY_CONFIG_FAIL") - chronyErrorMessage="Chrony configuration failed with exit code ${chronyExitCode}" + chronyErrorMessage="TDX Chrony configuration failed with exit code ${chronyExitCode}" ;; *) chronyErrorMessage="Unexpected Chrony configuration failure with exit code ${chronyExitCode}" diff --git a/spec/parts/linux/cloud-init/artifacts/init_aks_cloud_spec.sh b/spec/parts/linux/cloud-init/artifacts/init_aks_cloud_spec.sh index 17b57169bd4..15197c2e87c 100644 --- a/spec/parts/linux/cloud-init/artifacts/init_aks_cloud_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/init_aks_cloud_spec.sh @@ -378,7 +378,7 @@ EOF The status should be success End - It 'returns the Chrony configuration failure code when SEV-SNP PHC setup fails' + It 'continues provisioning when SEV-SNP PHC setup fails' Mock detect_confidential_vm_platform echo "sev-snp" End @@ -387,8 +387,9 @@ EOF End When call configure_ubuntu_2604_cvm_time_sync The output should include "AMD SEV-SNP detected" - The error should include "failed to configure Chrony with the Hyper-V PHC source" - The status should equal 246 + The error should include "failed to reapply the Hyper-V PHC Chrony configuration" + The error should include "continuing provisioning" + The status should be success End It 'defines exactly the four approved Ubuntu NTP pools' From 339ecaaa16763b81729f0b53ea13e24f42dd42e6 Mon Sep 17 00:00:00 2001 From: Zachary Bailey Date: Wed, 23 Sep 2026 18:12:45 -0400 Subject: [PATCH 14/14] fix --- .../cloud-init/artifacts/cse_config_chrony.sh | 14 ++++---- parts/linux/cloud-init/artifacts/cse_main.sh | 2 +- .../artifacts/init_aks_cloud_spec.sh | 36 +++++++++++++------ 3 files changed, 33 insertions(+), 19 deletions(-) diff --git a/parts/linux/cloud-init/artifacts/cse_config_chrony.sh b/parts/linux/cloud-init/artifacts/cse_config_chrony.sh index 40f6a208871..5237c3db2b0 100644 --- a/parts/linux/cloud-init/artifacts/cse_config_chrony.sh +++ b/parts/linux/cloud-init/artifacts/cse_config_chrony.sh @@ -41,7 +41,7 @@ pool 2.ubuntu.pool.ntp.org iburst maxsources 2 EOF } -configure_chrony() { +apply_chrony_configuration() { local time_sources="${1:-refclock PHC /dev/ptp0 poll 3 dpoll -2 offset 0}" local chrony_conf="${CHRONY_CONF:-/etc/chrony/chrony.conf}" local timesyncd_load_state @@ -208,30 +208,30 @@ configure_ubuntu_2604_cvm_time_sync() { case "$platform" in sev-snp) echo "AMD SEV-SNP detected; preserving the existing Hyper-V PHC Chrony configuration" - if ! configure_chrony; then + if ! logs_to_events "AKS.CSE.configureChronySEVSNP" apply_chrony_configuration; then echo "WARNING: failed to reapply the Hyper-V PHC Chrony configuration for AMD SEV-SNP; continuing provisioning" >&2 fi ;; tdx) echo "Intel TDX detected; configuring Chrony to use the Ubuntu NTP pools" ntp_pools="$(ubuntu_ntp_pools)" - if ! configure_chrony "$ntp_pools"; then + if ! logs_to_events "AKS.CSE.configureChronyTDX" apply_chrony_configuration "$ntp_pools"; then echo "ERROR: failed to configure Chrony with the Ubuntu NTP pools for Intel TDX" >&2 return "$ERR_CHRONY_CONFIG_FAIL" fi - verify_chrony_ntp_sync + logs_to_events "AKS.CSE.verifyChronyNTPSync" verify_chrony_ntp_sync ;; esac } -configureChrony() { +configure_node_time_sync() { if isACL "$OS" "$OS_VARIANT"; then echo "Skipping chrony configuration for ACL (PTP clock baked into chronyd, no external NTP sources)" elif isMarinerOrAzureLinux "$OS"; then - configure_mariner_azurelinux_chrony || true + logs_to_events "AKS.CSE.configureChronyMarinerAzureLinux" configure_mariner_azurelinux_chrony || true elif should_configure_ubuntu_2604_cvm_time_sync; then configure_ubuntu_2604_cvm_time_sync else - configure_chrony || true + logs_to_events "AKS.CSE.configureChronyDefaultPHC" apply_chrony_configuration || true fi } diff --git a/parts/linux/cloud-init/artifacts/cse_main.sh b/parts/linux/cloud-init/artifacts/cse_main.sh index 1ded6e4965b..4de6c702867 100755 --- a/parts/linux/cloud-init/artifacts/cse_main.sh +++ b/parts/linux/cloud-init/artifacts/cse_main.sh @@ -749,7 +749,7 @@ function nodePrep { # For VHD image creation workflows, only basePrep runs initially, and nodePrep runs later # when nodes are created from that VHD image. chronyExitCode=0 -logs_to_events "AKS.CSE.configureChrony" configureChrony || chronyExitCode=$? +logs_to_events "AKS.CSE.configureChrony" configure_node_time_sync || chronyExitCode=$? if [ "$chronyExitCode" -ne 0 ]; then case "$chronyExitCode" in "$ERR_CVM_PLATFORM_DETECTION_FAIL") diff --git a/spec/parts/linux/cloud-init/artifacts/init_aks_cloud_spec.sh b/spec/parts/linux/cloud-init/artifacts/init_aks_cloud_spec.sh index 15197c2e87c..9ade7317358 100644 --- a/spec/parts/linux/cloud-init/artifacts/init_aks_cloud_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/init_aks_cloud_spec.sh @@ -72,20 +72,30 @@ Describe 'cse_config_chrony.sh distro routing' script_path='./parts/linux/cloud-init/artifacts/cse_config_chrony.sh' chrony_routing_block() { - sed -n '/^configureChrony() {$/,/^}$/p' "$script_path" + sed -n '/^configure_node_time_sync() {$/,/^}$/p' "$script_path" } It 'keeps Azure Linux and Mariner on their native chronyd configuration path' When call chrony_routing_block The output should include 'elif isMarinerOrAzureLinux "$OS"; then' + The output should include 'AKS.CSE.configureChronyMarinerAzureLinux' The output should include 'configure_mariner_azurelinux_chrony || true' End - It 'keeps Ubuntu and Flatcar on configure_chrony unless the CVM path is selected' + It 'keeps Ubuntu and Flatcar on the default PHC configuration unless the CVM path is selected' When call chrony_routing_block The output should include 'elif should_configure_ubuntu_2604_cvm_time_sync; then' The output should include 'configure_ubuntu_2604_cvm_time_sync' - The output should include 'configure_chrony || true' + The output should include 'AKS.CSE.configureChronyDefaultPHC' + The output should include 'apply_chrony_configuration || true' + End + + It 'logs the CVM platform configuration and TDX synchronization sub-operations' + When run grep -E 'AKS.CSE.configureChrony(SEVSNP|TDX)|AKS.CSE.verifyChronyNTPSync' "$script_path" + The line 1 of output should include 'AKS.CSE.configureChronySEVSNP' + The line 2 of output should include 'AKS.CSE.configureChronyTDX' + The line 3 of output should include 'AKS.CSE.verifyChronyNTPSync' + The lines of output should equal 3 End End @@ -113,6 +123,10 @@ Describe 'init-aks-cloud.sh functional tests' ERR_CHRONY_CONFIG_FAIL=246 # shellcheck disable=SC1091 . "./parts/linux/cloud-init/artifacts/cse_config_chrony.sh" + logs_to_events() { + shift + "$@" + } } cleanup() { @@ -321,7 +335,7 @@ EOF setup_chrony_test OS_VERSION="24.04" - When call configure_chrony + When call apply_chrony_configuration The output should include "systemctl restart chrony" The contents of file "$CHRONY_CONF" should include "refclock PHC /dev/ptp0 poll 3 dpoll -2 offset 0" The status should be success @@ -382,7 +396,7 @@ EOF Mock detect_confidential_vm_platform echo "sev-snp" End - Mock configure_chrony + Mock apply_chrony_configuration return 1 End When call configure_ubuntu_2604_cvm_time_sync @@ -429,7 +443,7 @@ EOF fi End - When call configure_chrony + When call apply_chrony_configuration The error should include "failed to restart Chrony" The status should equal 1 End @@ -447,7 +461,7 @@ EOF fi End - When call configure_chrony + When call apply_chrony_configuration The output should include "systemd-timesyncd is removed, no need to disable" The output should not include "unexpected systemd-timesyncd operation" The contents of file "$CHRONY_CONF" should include "refclock PHC /dev/ptp0 poll 3 dpoll -2 offset 0" @@ -464,7 +478,7 @@ EOF fi End - When call configure_chrony + When call apply_chrony_configuration The output should include "systemctl stop systemd-timesyncd" The output should include "systemctl disable systemd-timesyncd" The contents of file "$CHRONY_CONF" should include "refclock PHC /dev/ptp0 poll 3 dpoll -2 offset 0" @@ -483,7 +497,7 @@ EOF fi End - When call configure_chrony + When call apply_chrony_configuration The error should include "failed to stop systemd-timesyncd" The output should include "systemctl disable systemd-timesyncd" The output should include "systemctl restart chrony" @@ -498,7 +512,7 @@ EOF Mock ubuntu_ntp_pools echo "fixed pools" End - Mock configure_chrony + Mock apply_chrony_configuration return 1 End Mock verify_chrony_ntp_sync @@ -548,7 +562,7 @@ EOF Mock ubuntu_ntp_pools echo "fixed pools" End - Mock configure_chrony + Mock apply_chrony_configuration : End Mock verify_chrony_ntp_sync