diff --git a/parts/linux/cloud-init/artifacts/aks-node-controller-hotfix.json b/parts/linux/cloud-init/artifacts/aks-node-controller-hotfix.json new file mode 100644 index 00000000000..1422f2cbb20 --- /dev/null +++ b/parts/linux/cloud-init/artifacts/aks-node-controller-hotfix.json @@ -0,0 +1,3 @@ +{ + "scripts_version": "202610.03.2" +} diff --git a/parts/linux/cloud-init/artifacts/init-aks-cloud.sh b/parts/linux/cloud-init/artifacts/init-aks-cloud.sh index bf38c0485e8..d2091ae31a6 100644 --- a/parts/linux/cloud-init/artifacts/init-aks-cloud.sh +++ b/parts/linux/cloud-init/artifacts/init-aks-cloud.sh @@ -422,13 +422,23 @@ function add_ms_keys { function aptget_update { echo "apt-get updating..." echo "note: depending on how many sources have been added this may take a couple minutes..." - if apt-get update | grep -q "404 Not Found"; then - echo "ERROR: apt-get update failed to find all sources. Please validate the sources or remove bad sources from your sources and try again." - emit_event "AKS.CSE.customCloudRepoInit.aptgetUpdateFailed" "apt-get update returned 404 for one or more sources" "Error" - exit 1 - else - echo "apt-get update complete!" - fi + local retries=10 + local i + for i in $(seq 1 "$retries"); do + echo "Running apt-get update (attempt $i/$retries)" + if apt-get update; then + echo "apt-get update complete!" + return 0 + fi + + if [ "$i" -lt "$retries" ]; then + sleep 5 + fi + done + + echo "ERROR: apt-get update failed after $retries attempts" >&2 + emit_event "AKS.CSE.customCloudRepoInit.aptgetUpdateFailed" "apt-get update failed after $retries attempts" "Error" + return 1 } function init_ubuntu_pmc_repo_depot { @@ -511,16 +521,22 @@ function dnf_makecache { local dnf_makecache_output=/tmp/dnf-makecache.out local i for i in $(seq 1 $retries); do - ! (dnf makecache -y 2>&1 | tee $dnf_makecache_output | grep -E "^([WE]:.*)|([eE]rr.*)$") && \ - cat $dnf_makecache_output && break || \ - cat $dnf_makecache_output - if [ $i -eq $retries ]; then - return 1 - else + dnf makecache -y > "$dnf_makecache_output" 2>&1 + cat "$dnf_makecache_output" + + if ! grep -Eq "^([WE]:.*)|([eE]rr.*)$" "$dnf_makecache_output"; then + echo "Executed dnf makecache -y $i times" + return 0 + fi + + if [ "$i" -lt "$retries" ]; then sleep 5 fi done - echo "Executed dnf makecache -y $i times" + + echo "ERROR: dnf_makecache failed after retries; Please validate the sources or remove bad sources from your sources and try again" >&2 + emit_event "AKS.CSE.customCloudRepoInit.dnfMakecacheFailed" "dnf_makecache failed after retries" "Error" + return 1 } # Determines the certificate endpoint mode based on location. @@ -633,6 +649,32 @@ fi # - init (default): full provisioning path # - ca-refresh : periodic refresh path; location is passed as arg to avoid env dependency action=${1:-init} +if [ "$action" = "init" ]; then + rootRepoDepotEndpoint="$(echo "${REPO_DEPOT_ENDPOINT}" | sed 's/\/ubuntu//')" + if [ -z "$rootRepoDepotEndpoint" ]; then + echo "repo depot endpoint empty, skipping repodepot initialization" + else + if [ "$IS_UBUNTU" -eq 1 ]; then + ubuntuRel=$(lsb_release --release | awk '{print $2}') + ubuntuDist=$(lsb_release -c | awk '{print $2}') + init_ubuntu_main_repo_depot ${rootRepoDepotEndpoint} + init_ubuntu_pmc_repo_depot ${rootRepoDepotEndpoint} + echo "Running apt-get update" + aptget_update + elif [ "$IS_MARINER" -eq 1 ] || [ "$IS_AZURELINUX" -eq 1 ]; then + if [ "$IS_MARINER" -eq 1 ]; then + echo "Initializing Mariner repo depot settings..." + init_mariner_repo_depot ${rootRepoDepotEndpoint} + else + echo "Initializing Azure Linux repo depot settings..." + init_azurelinux_repo_depot ${rootRepoDepotEndpoint} + fi + echo "Running dnf_makecache" + dnf_makecache + fi + fi +fi + if [ "$action" = "ca-refresh" ] || [ "$install_ca_refresh_schedule" -eq 0 ]; then exit 0 fi @@ -688,38 +730,6 @@ EOF systemctl enable --now azure-ca-refresh.timer fi -if [ "$IS_UBUNTU" -eq 1 ]; then - rootRepoDepotEndpoint="$(echo "${REPO_DEPOT_ENDPOINT}" | sed 's/\/ubuntu//')" - if [ -n "$rootRepoDepotEndpoint" ]; then - cloud-init status --wait - ubuntuRel=$(lsb_release --release | awk '{print $2}') - ubuntuDist=$(lsb_release -c | awk '{print $2}') - init_ubuntu_main_repo_depot ${rootRepoDepotEndpoint} - init_ubuntu_pmc_repo_depot ${rootRepoDepotEndpoint} - echo "Running apt-get update" - aptget_update - else - echo "REPO_DEPOT_ENDPOINT empty, skipping Ubuntu RepoDepot initialization" - fi -elif [ "$IS_MARINER" -eq 1 ] || [ "$IS_AZURELINUX" -eq 1 ]; then - cloud-init status --wait - - marinerRepoDepotEndpoint="$(echo "${REPO_DEPOT_ENDPOINT}" | sed 's/\/ubuntu//')" - if [ -z "$marinerRepoDepotEndpoint" ]; then - >&2 echo "repo depot endpoint empty while running custom-cloud init script" - else - if [ "$IS_MARINER" -eq 1 ]; then - echo "Initializing Mariner repo depot settings..." - init_mariner_repo_depot ${marinerRepoDepotEndpoint} - dnf_makecache || { echo "ERROR: dnf_makecache failed after retries; aborting custom cloud repo init (Mariner)"; emit_event "AKS.CSE.customCloudRepoInit.dnfMakecacheFailed" "dnf_makecache failed after retries (Mariner)" "Error"; exit 1; } - else - echo "Initializing Azure Linux repo depot settings..." - init_azurelinux_repo_depot ${marinerRepoDepotEndpoint} - dnf_makecache || { echo "ERROR: dnf_makecache failed after retries; aborting custom cloud repo init (Azure Linux)"; emit_event "AKS.CSE.customCloudRepoInit.dnfMakecacheFailed" "dnf_makecache failed after retries (Azure Linux)" "Error"; exit 1; } - fi - fi -fi - # Disable systemd-timesyncd and install chrony and uses local time source # ACL has PTP clock config compiled into chronyd with no config file or sourcedir directives, # so it uses only the local PTP clock and has no DHCP-injectable NTP sources. diff --git a/spec/parts/linux/cloud-init/artifacts/init_aks_cloud_spec.sh b/spec/parts/linux/cloud-init/artifacts/init_aks_cloud_spec.sh index 472f9b78de2..111406ef511 100644 --- a/spec/parts/linux/cloud-init/artifacts/init_aks_cloud_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/init_aks_cloud_spec.sh @@ -2,9 +2,10 @@ # Tests for parts/linux/cloud-init/artifacts/init-aks-cloud.sh # -# Covers two areas: +# Covers three areas: # 1. Structural wiring tests (grep-based) for the ca-refresh mode added by #8096. -# 2. Functional tests that source the script and exercise: +# 2. Repo initialization dispatch, independent of certificate refresh scheduling. +# 3. Functional tests that source the script and exercise: # - repo-depot helpers (init_ubuntu_main_repo_depot, init_ubuntu_pmc_repo_depot, # init_mariner_repo_depot, init_azurelinux_repo_depot, check_url) # - cloud mode selection helper (determine_cert_endpoint_mode) @@ -66,6 +67,174 @@ Describe 'init-aks-cloud.sh refresh mode wiring' When run grep -Eq '^ExecStart=\$script_path ca-refresh \$LOCATION$' "$script_path" The status should eq 0 End + + It 'does not wait for cloud-init completion' + When run grep -Eq '^[[:space:]]*cloud-init status --wait' "$script_path" + The status should eq 1 + End +End + +Describe 'init-aks-cloud.sh repo initialization dispatch' + script_path='./parts/linux/cloud-init/artifacts/init-aks-cloud.sh' + + setup_repo_dispatch() { + IS_UBUNTU=0 + IS_MARINER=0 + IS_AZURELINUX=0 + install_ca_refresh_schedule=0 + REPO_DEPOT_ENDPOINT='https://repodepot.example.com/ubuntu' + } + BeforeEach 'setup_repo_dispatch' + + run_repo_dispatch() { + # Run the actual action dispatch and early-exit guard, but not host scheduling. + . <(sed -n '/^action=/,/^if \[ "\$IS_UBUNTU"/p' "$script_path" | sed '$d') "$@" + echo 'refresh scheduling reached' + } + + Mock lsb_release + case "$1" in + --release) printf 'Release:\t22.04\n' ;; + -c) printf 'Codename:\tjammy\n' ;; + *) exit 1 ;; + esac + End + + Mock init_ubuntu_main_repo_depot + echo "ubuntu-main $1" + End + + Mock init_ubuntu_pmc_repo_depot + echo "ubuntu-pmc $1" + End + + Mock init_mariner_repo_depot + echo "mariner $1" + End + + Mock init_azurelinux_repo_depot + echo "azurelinux $1" + End + + Mock aptget_update + echo 'apt update' + End + + Mock dnf_makecache + echo 'dnf cache' + End + + Mock emit_event + echo "event $1" + End + + It 'initializes Ubuntu repos without certificate opt-in when action is omitted' + IS_UBUNTU=1 + + When run run_repo_dispatch + The output should include 'ubuntu-main https://repodepot.example.com' + The output should include 'ubuntu-pmc https://repodepot.example.com' + The output should include 'apt update' + The output should not include 'dnf cache' + The output should not include 'refresh scheduling reached' + The status should be success + End + + Describe 'explicit init' + Parameters + 1 0 0 'ubuntu-main' 'apt update' 0 + 1 0 0 'ubuntu-main' 'apt update' 1 + 0 1 0 'mariner' 'dnf cache' 0 + 0 1 0 'mariner' 'dnf cache' 1 + 0 0 1 'azurelinux' 'dnf cache' 0 + 0 0 1 'azurelinux' 'dnf cache' 1 + End + + It 'routes repo setup independently of certificate refresh eligibility' + IS_UBUNTU=$1 + IS_MARINER=$2 + IS_AZURELINUX=$3 + install_ca_refresh_schedule=$6 + + When run run_repo_dispatch init + The output should include "$4 https://repodepot.example.com" + The output should include "$5" + if [ "$6" -eq 1 ]; then + The output should include 'refresh scheduling reached' + else + The output should not include 'refresh scheduling reached' + fi + The status should be success + End + End + + Describe 'ca-refresh' + Parameters + 0 + 1 + End + + It 'skips repo initialization and scheduling regardless of certificate opt-in' + IS_UBUNTU=1 + install_ca_refresh_schedule=$1 + + When run run_repo_dispatch ca-refresh + The output should be blank + The status should be success + End + End + + Describe 'empty endpoint' + Parameters + 1 0 0 + 0 1 0 + 0 0 1 + End + + It 'skips repo initialization and package cache updates' + IS_UBUNTU=$1 + IS_MARINER=$2 + IS_AZURELINUX=$3 + REPO_DEPOT_ENDPOINT='' + + When run run_repo_dispatch init + The output should equal 'repo depot endpoint empty, skipping repodepot initialization' + The status should be success + End + End + + Describe 'non-fatal package cache failure' + Parameters + 1 0 0 0 + 1 0 0 1 + 0 1 0 0 + 0 1 0 1 + 0 0 1 0 + 0 0 1 1 + End + + run_failed_repo_dispatch() { + aptget_update() { return 1; } + dnf_makecache() { return 1; } + run_repo_dispatch init + } + + It 'continues to refresh scheduling only for eligible nodes after cache failure' + IS_UBUNTU=$1 + IS_MARINER=$2 + IS_AZURELINUX=$3 + install_ca_refresh_schedule=$4 + + When run run_failed_repo_dispatch + The output should be present + if [ "$4" -eq 1 ]; then + The output should include 'refresh scheduling reached' + else + The output should not include 'refresh scheduling reached' + fi + The status should be success + End + End End Describe 'init-aks-cloud.sh functional tests' @@ -94,6 +263,126 @@ Describe 'init-aks-cloud.sh functional tests' BeforeEach 'setup' AfterEach 'cleanup' + Describe 'package cache retries' + setup_cache_retries() { + RETRY_ATTEMPTS_FILE="${TEST_DIR}/attempts" + RETRY_SLEEPS_FILE="${TEST_DIR}/sleeps" + RETRY_EVENTS_FILE="${TEST_DIR}/events" + printf '0' > "$RETRY_ATTEMPTS_FILE" + printf '0' > "$RETRY_SLEEPS_FILE" + : > "$RETRY_EVENTS_FILE" + RETRY_FAILURES=0 + RETRY_FAILURE_STATUS=1 + RETRY_FAILURE_TEXT='Error: 404 Not Found' + # Keep DNF's output file inside the fixture rather than shared /tmp. + __SOURCED__=1 . <(sed "s|/tmp/dnf-makecache.out|${TEST_DIR}/dnf-makecache.out|g" \ + ./parts/linux/cloud-init/artifacts/init-aks-cloud.sh) + } + BeforeEach 'setup_cache_retries' + + cache_attempt() { + local attempt + attempt=$(cat "$RETRY_ATTEMPTS_FILE") + attempt=$((attempt + 1)) + printf '%s' "$attempt" > "$RETRY_ATTEMPTS_FILE" + if [ "$attempt" -le "$RETRY_FAILURES" ]; then + echo "$RETRY_FAILURE_TEXT" + return "$RETRY_FAILURE_STATUS" + fi + return 0 + } + + run_cache_update() { + apt-get() { cache_attempt; } + dnf() { cache_attempt; } + sleep() { + local sleeps + sleeps=$(cat "$RETRY_SLEEPS_FILE") + printf '%s' "$((sleeps + 1))" > "$RETRY_SLEEPS_FILE" + } + emit_event() { echo "$1" >> "$RETRY_EVENTS_FILE"; } + "$1" + } + + Describe 'successful updates' + Parameters + aptget_update 0 1 0 + aptget_update 2 3 2 + aptget_update 9 10 9 + dnf_makecache 0 1 0 + dnf_makecache 2 3 2 + dnf_makecache 9 10 9 + End + + It 'stops at the first success and sleeps only between attempts' + RETRY_FAILURES=$2 + + When run run_cache_update "$1" + The output should be present + The status should be success + The contents of file "$RETRY_ATTEMPTS_FILE" should equal "$3" + The contents of file "$RETRY_SLEEPS_FILE" should equal "$4" + The contents of file "$RETRY_EVENTS_FILE" should be blank + End + End + + Describe 'exhausted retries' + Parameters + aptget_update aptgetUpdateFailed + dnf_makecache dnfMakecacheFailed + End + + It 'returns failure and emits an event after ten attempts and nine sleeps' + RETRY_FAILURES=10 + + When run run_cache_update "$1" + The output should be present + The error should include 'ERROR:' + The status should eq 1 + The contents of file "$RETRY_ATTEMPTS_FILE" should equal 10 + The contents of file "$RETRY_SLEEPS_FILE" should equal 9 + The contents of file "$RETRY_EVENTS_FILE" should equal "AKS.CSE.customCloudRepoInit.$2" + End + End + + It 'accepts a successful APT exit status regardless of diagnostic text' + RETRY_FAILURES=10 + RETRY_FAILURE_STATUS=0 + + When run run_cache_update aptget_update + The output should be present + The status should be success + The contents of file "$RETRY_ATTEMPTS_FILE" should equal 1 + The contents of file "$RETRY_SLEEPS_FILE" should equal 0 + The contents of file "$RETRY_EVENTS_FILE" should be blank + End + + It 'retries non-404 APT failures until attempts are exhausted' + RETRY_FAILURES=10 + RETRY_FAILURE_STATUS=100 + RETRY_FAILURE_TEXT='Temporary failure resolving repository' + + When run run_cache_update aptget_update + The output should be present + The error should include 'ERROR: apt-get update failed after 10 attempts' + The status should eq 1 + The contents of file "$RETRY_ATTEMPTS_FILE" should equal 10 + The contents of file "$RETRY_SLEEPS_FILE" should equal 9 + The contents of file "$RETRY_EVENTS_FILE" should equal 'AKS.CSE.customCloudRepoInit.aptgetUpdateFailed' + End + + It 'retains DNF text-based detection rather than failing on exit status alone' + RETRY_FAILURES=10 + RETRY_FAILURE_TEXT='unmatched diagnostic' + + When run run_cache_update dnf_makecache + The output should include 'unmatched diagnostic' + The status should be success + The contents of file "$RETRY_ATTEMPTS_FILE" should equal 1 + The contents of file "$RETRY_SLEEPS_FILE" should equal 0 + End + End + write_ubuntu_os_release() { cat > "${OS_RELEASE_FILE}" <