diff --git a/parts/linux/cloud-init/artifacts/cse_cmd.sh b/parts/linux/cloud-init/artifacts/cse_cmd.sh index 2079e1f1664..8291358daea 100644 --- a/parts/linux/cloud-init/artifacts/cse_cmd.sh +++ b/parts/linux/cloud-init/artifacts/cse_cmd.sh @@ -18,7 +18,18 @@ fi; {{end}} INIT_AKS_CLOUD_FILEPATH="{{GetInitAKSCloudFilepath}}"; if [ -f "${INIT_AKS_CLOUD_FILEPATH}" ]; then - REPO_DEPOT_ENDPOINT="{{AKSCustomCloudRepoDepotEndpoint}}" LOCATION={{GetVariable "location"}} "${INIT_AKS_CLOUD_FILEPATH}" >> /var/log/azure/cluster-provision.log 2>&1; + PRE_PROVISION_ONLY="{{GetPreProvisionOnly}}" REPO_DEPOT_ENDPOINT="{{AKSCustomCloudRepoDepotEndpoint}}" LOCATION={{GetVariable "location"}} "${INIT_AKS_CLOUD_FILEPATH}" >> /var/log/azure/cluster-provision.log 2>&1; + initAKSCloudExitCode=$?; + if [ "$initAKSCloudExitCode" -eq 246 ]; then + echo "Chrony configuration failed; init-aks-cloud failed with exit code ${initAKSCloudExitCode}" >> ${PROVISION_OUTPUT}; + exit ${initAKSCloudExitCode}; + elif [ "$initAKSCloudExitCode" -eq 245 ]; then + echo "NTP not reachable; init-aks-cloud failed with NTP synchronization error code ${initAKSCloudExitCode}" >> ${PROVISION_OUTPUT}; + exit ${initAKSCloudExitCode}; + elif [ "$initAKSCloudExitCode" -eq 244 ]; then + echo "Unable to determine confidential VM platform; init-aks-cloud failed with exit code ${initAKSCloudExitCode}" >> ${PROVISION_OUTPUT}; + exit ${initAKSCloudExitCode}; + fi; fi; {{/* Keep the environment assignments below contiguous through the nohup invocation at the end of this file. */ -}} {{/* The CSE command is flattened into one shell command, so all assignments below are passed to nohup. */ -}} diff --git a/parts/linux/cloud-init/artifacts/cse_helpers.sh b/parts/linux/cloud-init/artifacts/cse_helpers.sh index 8862606a704..7e45cd1f0e8 100755 --- a/parts/linux/cloud-init/artifacts/cse_helpers.sh +++ b/parts/linux/cloud-init/artifacts/cse_helpers.sh @@ -162,6 +162,7 @@ ERR_AKS_NODE_CONTROLLER_ERROR=240 # Generic error in AKS Node Controller ERR_AZNFS_RPM_DOWNLOAD_TIMEOUT=241 # Timeout downloading aznfs RPM from PMC ERR_AZNFS_INSTALL_FAIL=242 # Failed to install aznfs RPM package ERR_SECONDARY_NIC_CONFIG_FAIL=243 # Error configuring secondary NIC network interface +# Exit codes 244-246 are reserved by init-aks-cloud.sh and propagated by cse_cmd.sh. # ----------------------------------------------------------------------------- # This probably wasn't launched via a login shell, so ensure the PATH is correct. diff --git a/parts/linux/cloud-init/artifacts/init-aks-cloud.sh b/parts/linux/cloud-init/artifacts/init-aks-cloud.sh index bf38c0485e8..538598ddc13 100644 --- a/parts/linux/cloud-init/artifacts/init-aks-cloud.sh +++ b/parts/linux/cloud-init/artifacts/init-aks-cloud.sh @@ -103,6 +103,10 @@ IS_ACL=0 IS_MARINER=0 IS_AZURELINUX=0 +ERR_CVM_PLATFORM_DETECTION_FAIL=244 # Unable to distinguish SEV-SNP from TDX +ERR_NTP_UNREACHABLE=245 # Chrony could not synchronize with the configured NTP pools +ERR_CHRONY_CONFIG_FAIL=246 # Chrony could not be configured for the detected CVM platform + # http://168.63.129.16 is a constant for the host's wireserver endpoint. WIRESERVER_ENDPOINT="http://168.63.129.16" @@ -538,9 +542,210 @@ function determine_cert_endpoint_mode { echo "$mode" } +function is_ubuntu_2604_cvm { + [ "$IS_UBUNTU" -eq 1 ] || return 1 + [ "${VERSION_ID:-}" = "26.04" ] || return 1 + + case "$(uname -r)" in + *-azure-fde*) return 0 ;; + *) return 1 ;; + esac +} + +function should_configure_ubuntu_2604_cvm_time_sync { + [ "${PRE_PROVISION_ONLY:-false}" != "true" ] || return 1 + is_ubuntu_2604_cvm +} + +function detect_confidential_vm_platform { + local platform + + if ! platform="$(systemd-detect-virt --cvm 2>/dev/null)"; then + return 1 + fi + + case "$platform" in + sev-snp|tdx) + echo "$platform" + ;; + *) + return 1 + ;; + esac +} + +function ubuntu_ntp_pools { + cat <<'EOF' +pool ntp.ubuntu.com iburst maxsources 4 +pool 0.ubuntu.pool.ntp.org iburst maxsources 1 +pool 1.ubuntu.pool.ntp.org iburst maxsources 1 +pool 2.ubuntu.pool.ntp.org iburst maxsources 2 +EOF +} + +function configure_chrony { + local time_sources="${1:-refclock PHC /dev/ptp0 poll 3 dpoll -2 offset 0}" + local chrony_conf="${CHRONY_CONF:-/etc/chrony/chrony.conf}" + local timesyncd_load_state + local chrony_failed=0 + + if [ "$IS_UBUNTU" -eq 1 ]; then + timesyncd_load_state="$(systemctl show -p LoadState --value systemd-timesyncd 2>/dev/null || true)" + if [ "$timesyncd_load_state" = "not-found" ]; then + echo "systemd-timesyncd is removed, no need to disable" + else + if ! systemctl stop systemd-timesyncd; then + echo "ERROR: failed to stop systemd-timesyncd" >&2 + chrony_failed=1 + fi + if ! systemctl disable systemd-timesyncd; then + echo "ERROR: failed to disable systemd-timesyncd" >&2 + chrony_failed=1 + fi + fi + + if [ ! -e "$chrony_conf" ]; then + if ! apt-get update; then + echo "ERROR: failed to update package metadata before installing Chrony" >&2 + chrony_failed=1 + fi + if ! apt-get install chrony -y; then + echo "ERROR: failed to install Chrony" >&2 + chrony_failed=1 + fi + fi + elif [ "$IS_FLATCAR" -eq 1 ]; then + if ! rm -f "$chrony_conf"; then + echo "ERROR: failed to remove the existing Flatcar Chrony configuration" >&2 + chrony_failed=1 + fi + fi + + if ! cat > "$chrony_conf" <&2 + chrony_failed=1 + fi + + if [ "$IS_UBUNTU" -eq 1 ]; then + if ! systemctl restart chrony; then + echo "ERROR: failed to restart Chrony" >&2 + chrony_failed=1 + fi + elif [ "$IS_FLATCAR" -eq 1 ]; then + if ! systemctl restart chronyd; then + echo "ERROR: failed to restart chronyd" >&2 + chrony_failed=1 + fi + fi + + return "$chrony_failed" +} + +function verify_chrony_ntp_sync { + local max_attempts=12 + local retry_interval_seconds=5 + + if chronyc waitsync "$max_attempts" 0 0 "$retry_interval_seconds"; then + echo "NTP synchronization confirmed through the Ubuntu NTP pools" + emit_event "AKS.CSE.chrony.ntpSynchronized" "NTP synchronization confirmed through the Ubuntu NTP pools" + return 0 + fi + + echo "ERROR: NTP not reachable; Chrony did not synchronize" >&2 + emit_event "AKS.CSE.chrony.ntpUnavailable" "NTP not reachable after ${max_attempts} synchronization checks; failing provisioning" "Error" + echo "Chrony source diagnostics:" >&2 + chronyc sources -v >&2 || echo "ERROR: unable to retrieve Chrony source diagnostics" >&2 + echo "Chrony tracking diagnostics:" >&2 + chronyc tracking >&2 || echo "ERROR: unable to retrieve Chrony tracking diagnostics" >&2 + return "$ERR_NTP_UNREACHABLE" +} + +function configure_ubuntu_2604_cvm_time_sync { + local platform + local ntp_pools + + if ! platform="$(detect_confidential_vm_platform)"; then + echo "ERROR: unable to determine Ubuntu 26.04 CVM platform with systemd-detect-virt --cvm" >&2 + emit_event "AKS.CSE.chrony.platformDetectionFailed" "Unable to distinguish SEV-SNP from TDX using systemd-detect-virt --cvm" "Error" + return "$ERR_CVM_PLATFORM_DETECTION_FAIL" + fi + + case "$platform" in + sev-snp) + echo "AMD SEV-SNP detected; preserving the existing Hyper-V PHC Chrony configuration" + emit_event "AKS.CSE.chrony.usingPHC" "AMD SEV-SNP detected; preserving the existing /dev/ptp0 PHC configuration" + if ! configure_chrony; then + echo "ERROR: failed to configure Chrony with the Hyper-V PHC source for AMD SEV-SNP" >&2 + emit_event "AKS.CSE.chrony.configurationFailed" "Failed to configure Chrony with the Hyper-V PHC source for AMD SEV-SNP" "Error" + return "$ERR_CHRONY_CONFIG_FAIL" + fi + ;; + tdx) + echo "Intel TDX detected; configuring Chrony to use the Ubuntu NTP pools" + emit_event "AKS.CSE.chrony.usingNTP" "Intel TDX detected; using only the approved Ubuntu NTP pools" + ntp_pools="$(ubuntu_ntp_pools)" + if ! configure_chrony "$ntp_pools"; then + echo "ERROR: failed to configure Chrony with the Ubuntu NTP pools for Intel TDX" >&2 + emit_event "AKS.CSE.chrony.configurationFailed" "Failed to configure Chrony with the Ubuntu NTP pools for Intel TDX" "Error" + return "$ERR_CHRONY_CONFIG_FAIL" + fi + verify_chrony_ntp_sync + ;; + esac +} + # shellcheck disable=SC2317 ${__SOURCED__:+return} +action=${1:-init} + # shellcheck disable=SC3010 if [[ -f /etc/os-release ]]; then . /etc/os-release @@ -566,6 +771,16 @@ fi echo "Running on $NAME" +ubuntu_2604_cvm_chrony_configured=0 +if [ "$action" = "init" ] && should_configure_ubuntu_2604_cvm_time_sync; then + configure_ubuntu_2604_cvm_time_sync + chrony_result=$? + if [ "$chrony_result" -ne 0 ]; then + exit "$chrony_result" + fi + ubuntu_2604_cvm_chrony_configured=1 +fi + # Certificate refresh behavior summary: # - legacy mode directly attempts certificate download from wireserver and only in ussec and usnat regions. @@ -632,7 +847,6 @@ fi # Action values: # - init (default): full provisioning path # - ca-refresh : periodic refresh path; location is passed as arg to avoid env dependency -action=${1:-init} if [ "$action" = "ca-refresh" ] || [ "$install_ca_refresh_schedule" -eq 0 ]; then exit 0 fi @@ -755,71 +969,8 @@ EOF systemctl restart chronyd else - chrony_conf="/etc/chrony/chrony.conf" - if [ "$IS_UBUNTU" -eq 1 ]; then - systemctl stop systemd-timesyncd - systemctl disable systemd-timesyncd - - if [ ! -e "$chrony_conf" ]; then - apt-get update - apt-get install chrony -y - fi - elif [ "$IS_FLATCAR" -eq 1 ]; then - rm -f ${chrony_conf} - fi - - cat > $chrony_conf < /etc/chrony.conf <&2 + End + + When call configure_ubuntu_2604_cvm_time_sync + The error should include "unable to determine Ubuntu 26.04 CVM platform" + The error should include "AKS.CSE.chrony.platformDetectionFailed" + The status should equal 244 + End + + It 'preserves the existing PHC configuration for SEV-SNP' + setup_chrony_test + Mock detect_confidential_vm_platform + echo "sev-snp" + End + Mock emit_event + echo "event: $*" + End + + When call configure_ubuntu_2604_cvm_time_sync + The output should include "preserving the existing Hyper-V PHC Chrony configuration" + The output should include "AKS.CSE.chrony.usingPHC" + The contents of file "$CHRONY_CONF" should include "refclock PHC /dev/ptp0 poll 3 dpoll -2 offset 0" + The status should be success + End + + It 'returns the Chrony configuration failure code when SEV-SNP PHC setup fails' + Mock detect_confidential_vm_platform + echo "sev-snp" + End + Mock configure_chrony + return 1 + End + Mock emit_event + echo "event: $*" >&2 + End + + When call configure_ubuntu_2604_cvm_time_sync + The output should include "AMD SEV-SNP detected" + The error should include "failed to configure Chrony with the Hyper-V PHC source" + The error should include "AKS.CSE.chrony.configurationFailed" + The status should equal 246 + End + + It 'defines exactly the four approved Ubuntu NTP pools' + When call ubuntu_ntp_pools + The lines of output should eq 4 + The line 1 of output should eq "pool ntp.ubuntu.com iburst maxsources 4" + The line 2 of output should eq "pool 0.ubuntu.pool.ntp.org iburst maxsources 1" + The line 3 of output should eq "pool 1.ubuntu.pool.ntp.org iburst maxsources 1" + The line 4 of output should eq "pool 2.ubuntu.pool.ntp.org iburst maxsources 2" + The status should be success + End + + It 'configures TDX with only the fixed Ubuntu NTP pools' + setup_chrony_test + Mock detect_confidential_vm_platform + echo "tdx" + End + Mock verify_chrony_ntp_sync + echo "verified NTP synchronization" + End + Mock emit_event + echo "event: $*" + End + + When call configure_ubuntu_2604_cvm_time_sync + The output should include "Intel TDX detected" + The output should include "verified NTP synchronization" + The contents of file "$CHRONY_CONF" should include "pool ntp.ubuntu.com iburst maxsources 4" + The contents of file "$CHRONY_CONF" should include "pool 0.ubuntu.pool.ntp.org iburst maxsources 1" + The contents of file "$CHRONY_CONF" should include "pool 1.ubuntu.pool.ntp.org iburst maxsources 1" + The contents of file "$CHRONY_CONF" should include "pool 2.ubuntu.pool.ntp.org iburst maxsources 2" + The contents of file "$CHRONY_CONF" should not include "refclock PHC" + The status should be success + End + + It 'returns failure when the Chrony service cannot restart' + setup_chrony_test + Mock systemctl + if [ "$1" = "restart" ]; then + return 1 + fi + End + + When call configure_chrony + The error should include "failed to restart Chrony" + The status should equal 1 + End + + It 'skips stopping and disabling systemd-timesyncd when the unit is removed' + setup_chrony_test + Mock systemctl + if [ "$1" = "show" ]; then + echo "not-found" + return 1 + fi + if [ "$1" = "stop" ] || [ "$1" = "disable" ]; then + echo "unexpected systemd-timesyncd operation" + return 1 + fi + End + + When call configure_chrony + The output should include "systemd-timesyncd is removed, no need to disable" + The output should not include "unexpected systemd-timesyncd operation" + The contents of file "$CHRONY_CONF" should include "refclock PHC /dev/ptp0 poll 3 dpoll -2 offset 0" + The status should be success + End + + It 'stops and disables systemd-timesyncd when the unit is loaded but inactive' + setup_chrony_test + Mock systemctl + if [ "$1" = "show" ]; then + echo "loaded" + elif [ "$1" = "stop" ] || [ "$1" = "disable" ]; then + echo "systemctl $*" + fi + End + + When call configure_chrony + The output should include "systemctl stop systemd-timesyncd" + The output should include "systemctl disable systemd-timesyncd" + The contents of file "$CHRONY_CONF" should include "refclock PHC /dev/ptp0 poll 3 dpoll -2 offset 0" + The status should be success + End + + It 'continues Chrony setup and returns failure when systemd-timesyncd cannot be stopped' + setup_chrony_test + Mock systemctl + if [ "$1" = "show" ]; then + echo "loaded" + elif [ "$1" = "stop" ]; then + return 1 + elif [ "$1" = "disable" ] || [ "$1" = "restart" ]; then + echo "systemctl $*" + fi + End + + When call configure_chrony + The error should include "failed to stop systemd-timesyncd" + The output should include "systemctl disable systemd-timesyncd" + The output should include "systemctl restart chrony" + The contents of file "$CHRONY_CONF" should include "refclock PHC /dev/ptp0 poll 3 dpoll -2 offset 0" + The status should equal 1 + End + + It 'returns the Chrony configuration failure code without checking NTP when TDX setup fails' + Mock detect_confidential_vm_platform + echo "tdx" + End + Mock ubuntu_ntp_pools + echo "fixed pools" + End + Mock configure_chrony + return 1 + End + Mock verify_chrony_ntp_sync + echo "unexpected NTP verification" + End + Mock emit_event + echo "event: $*" >&2 + End + + When call configure_ubuntu_2604_cvm_time_sync + The output should not include "unexpected NTP verification" + The error should include "failed to configure Chrony with the Ubuntu NTP pools" + The error should include "AKS.CSE.chrony.configurationFailed" + The status should equal 246 + End + + It 'waits for Chrony to synchronize successfully' + Mock chronyc + echo "$*" + End + Mock emit_event + echo "event: $*" + End + + When call verify_chrony_ntp_sync + The output should include "waitsync 12 0 0 5" + The output should include "NTP synchronization confirmed through the Ubuntu NTP pools" + The output should include "AKS.CSE.chrony.ntpSynchronized" + The status should be success + End + + It 'returns the NTP-unreachable code with Chrony diagnostics when NTP is not reachable' + Mock chronyc + case "$1" in + waitsync) + return 1 + ;; + sources) + echo "mock Chrony sources" + ;; + tracking) + echo "mock Chrony tracking" + ;; + esac + End + Mock emit_event + echo "event: $*" >&2 + End + + When call verify_chrony_ntp_sync + The error should include "NTP not reachable" + The error should include "AKS.CSE.chrony.ntpUnavailable" + The error should include "mock Chrony sources" + The error should include "mock Chrony tracking" + The status should equal 245 + End + + It 'propagates failed TDX NTP synchronization' + Mock detect_confidential_vm_platform + echo "tdx" + End + Mock ubuntu_ntp_pools + echo "fixed pools" + End + Mock configure_chrony + : + End + Mock verify_chrony_ntp_sync + exit 245 + End + Mock emit_event + : + End + + When call configure_ubuntu_2604_cvm_time_sync + The output should include "Intel TDX detected" + The status should equal 245 + End + End + Describe 'init_mariner_repo_depot' It 'creates extended, nvidia, and cloud-native repos and points all at RepoDepot' export YUM_REPOS_DIR="${TEST_DIR}/yum.repos.d"