diff --git a/parts/linux/cloud-init/artifacts/cse_config.sh b/parts/linux/cloud-init/artifacts/cse_config.sh index d6395852aa0..7e360e5ba6d 100755 --- a/parts/linux/cloud-init/artifacts/cse_config.sh +++ b/parts/linux/cloud-init/artifacts/cse_config.sh @@ -506,7 +506,17 @@ configureK8s() { if [ "${ENABLE_SECURE_TLS_BOOTSTRAPPING}" = "false" ] && [ -z "${TLS_BOOTSTRAP_TOKEN:-}" ]; then # only create the client cert and key if we're not using vanilla/secure TLS bootstrapping if [ -n "${KUBELET_CLIENT_CONTENT}" ]; then - echo "${KUBELET_CLIENT_CONTENT}" | base64 -d > /etc/kubernetes/certs/client.key + # client.key is the kubelet client private key; path is overridable for testing. + # Create under a restrictive umask so the file is never briefly world-readable + # between creation and chmod (avoids a TOCTOU window), then chmod again to cover + # the case where the file already existed with looser permissions. + local client_key_path="${CLIENT_KEY_PATH:-/etc/kubernetes/certs/client.key}" + # Guard both operations: if either fails, stop before writing the private key + # rather than silently falling through and writing it with whatever (possibly + # world-readable) permissions the file ended up with. + (umask 077 && touch "${client_key_path}") || return "$ERR_KUBELET_CLIENT_KEY_PERM_FAIL" + chmod 0600 "${client_key_path}" || return "$ERR_KUBELET_CLIENT_KEY_PERM_FAIL" + echo "${KUBELET_CLIENT_CONTENT}" | base64 -d > "${client_key_path}" fi if [ -n "${KUBELET_CLIENT_CERT_CONTENT}" ]; then echo "${KUBELET_CLIENT_CERT_CONTENT}" | base64 -d > /etc/kubernetes/certs/client.crt diff --git a/parts/linux/cloud-init/artifacts/cse_helpers.sh b/parts/linux/cloud-init/artifacts/cse_helpers.sh index 3dbc587f3e0..a6e3076dc6f 100755 --- a/parts/linux/cloud-init/artifacts/cse_helpers.sh +++ b/parts/linux/cloud-init/artifacts/cse_helpers.sh @@ -170,6 +170,10 @@ ERR_CHRONY_NTP_SYNC_FAIL=245 # Chrony could not synchronize with the configured ERR_CHRONY_CONFIG_FAIL=246 # Chrony could not be configured for Intel TDX # ----------------------------------------------------------------------------- +# ----------------------- Kubernetes client credentials ------------------------ +ERR_KUBELET_CLIENT_KEY_PERM_FAIL=247 # Could not create/chmod the kubelet client private key under a safe umask +# ----------------------------------------------------------------------------- + # This probably wasn't launched via a login shell, so ensure the PATH is correct. [ -f /etc/profile.d/path.sh ] && . /etc/profile.d/path.sh diff --git a/parts/linux/cloud-init/artifacts/cse_main.sh b/parts/linux/cloud-init/artifacts/cse_main.sh index 0c3ffe92253..5075003c238 100755 --- a/parts/linux/cloud-init/artifacts/cse_main.sh +++ b/parts/linux/cloud-init/artifacts/cse_main.sh @@ -449,7 +449,7 @@ function nodePrep { # This function first creates the systemd drop-in directory for kubelet.service. # Pay attention to ordering relative to other functions that create kubelet drop-ins. - logs_to_events "AKS.CSE.configureK8s" configureK8s + logs_to_events "AKS.CSE.configureK8s" configureK8s || exit "$ERR_KUBELET_CLIENT_KEY_PERM_FAIL" if [ "${ENABLE_SECURE_TLS_BOOTSTRAPPING}" = "true" ]; then # Depends on configureK8s, ensureKubeCACert, and installSecureTLSBootstrapClient diff --git a/parts/linux/cloud-init/artifacts/setup-custom-search-domains.sh b/parts/linux/cloud-init/artifacts/setup-custom-search-domains.sh index e8472d62ad4..0899fb4e1d2 100755 --- a/parts/linux/cloud-init/artifacts/setup-custom-search-domains.sh +++ b/parts/linux/cloud-init/artifacts/setup-custom-search-domains.sh @@ -9,4 +9,9 @@ wait_for_apt_locks # this script is called from `cse_main.sh` where we return ERR_CUSTOM_SEARCH_DOMAINS_FAIL in case this script ends with non-zero exit code retrycmd_if_failure 10 5 120 apt-get -y install realmd sssd sssd-tools samba-common samba samba-common python2.7 samba-libs packagekit || exit 1 wait_for_apt_locks +# disable command tracing so the realm password isn't written to provisioning logs +set +x echo "${CUSTOM_SEARCH_REALM_PASSWORD}" | realm join -U ${CUSTOM_SEARCH_REALM_USER}@$(echo "${CUSTOM_SEARCH_DOMAIN_NAME}" | tr /a-z/ /A-Z/) $(echo "${CUSTOM_SEARCH_DOMAIN_NAME}" | tr /a-z/ /A-Z/) +realm_join_status=$? +set -x +exit "${realm_join_status}" diff --git a/spec/parts/linux/cloud-init/artifacts/cse_config_spec.sh b/spec/parts/linux/cloud-init/artifacts/cse_config_spec.sh index 1df096ea45c..8ce084d5d33 100755 --- a/spec/parts/linux/cloud-init/artifacts/cse_config_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/cse_config_spec.sh @@ -854,4 +854,76 @@ PubkeyAuthentication no" The output should include "ensureSwapFileFstabEntry /swapfile" End End + Describe 'configureK8s client key permissions' + # Use a throwaway temp dir rather than the real /etc/kubernetes/certs path: when + # ShellSpec is installed natively (spec/README.md's "Option 2") instead of via the + # recommended container, tests run directly against the host filesystem, and this + # test would otherwise delete/overwrite a real node's kubelet credential. + setup_client_key() { + CLIENT_KEY_TEST_DIR="$(mktemp -d "${SHELLSPEC_TMPBASE:-${TMPDIR:-/tmp}}/client-key.XXXXXX")" + CLIENT_KEY_PATH="${CLIENT_KEY_TEST_DIR}/client.key" + APISERVER_PUBLIC_KEY="" + ENABLE_SECURE_TLS_BOOTSTRAPPING="false" + unset TLS_BOOTSTRAP_TOKEN + KUBELET_CLIENT_CONTENT="Y2xpZW50LWtleQ==" + KUBELET_CLIENT_CERT_CONTENT="" + KUBELET_CONFIG_FILE_ENABLED="false" + } + + cleanup_client_key() { + set +x + command rm -rf "${CLIENT_KEY_TEST_DIR}" + } + + invoke_configureK8s() { + { + configureK8s + local status=$? + set +x + return "${status}" + } 2>/dev/null + } + + BeforeEach 'setup_client_key' + AfterEach 'cleanup_client_key' + + It 'writes the kubelet client private key with mode 0600' + When call invoke_configureK8s + + The status should be success + The contents of file "${CLIENT_KEY_PATH}" should equal 'client-key' + The value "$(stat -c '%a' "${CLIENT_KEY_PATH}")" should equal '600' + End + + It 'creates the client key as 0600 at creation time, not only after a later chmod' + # Stub chmod as a no-op so the only thing that can have set a restrictive mode + # is the umask-scoped touch itself; this proves there is no world-readable window. + chmod() { :; } + + When call invoke_configureK8s + + The status should be success + The value "$(stat -c '%a' "${CLIENT_KEY_PATH}")" should equal '600' + End + + It 'does not write the private key if chmod fails' + chmod() { return 1; } + + When call invoke_configureK8s + + The status should equal "$ERR_KUBELET_CLIENT_KEY_PERM_FAIL" + # touch still ran before the guarded chmod, so an empty file may exist; + # what matters is the secret content was never written into it. + The contents of file "${CLIENT_KEY_PATH}" should not equal 'client-key' + End + + It 'does not write the private key if touch fails' + touch() { return 1; } + + When call invoke_configureK8s + + The status should equal "$ERR_KUBELET_CLIENT_KEY_PERM_FAIL" + The path "${CLIENT_KEY_PATH}" should not be exist + End + End End diff --git a/spec/parts/linux/cloud-init/artifacts/setup_custom_search_domains_spec.sh b/spec/parts/linux/cloud-init/artifacts/setup_custom_search_domains_spec.sh new file mode 100644 index 00000000000..33331649900 --- /dev/null +++ b/spec/parts/linux/cloud-init/artifacts/setup_custom_search_domains_spec.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +Describe 'setup-custom-search-domains.sh' + setup() { + TEST_DIR=$(mktemp -d) + mkdir -p "${TEST_DIR}/bin" + cat > "${TEST_DIR}/helpers.sh" <<'EOF' +wait_for_apt_locks() { :; } +systemctl_restart() { :; } +retrycmd_if_failure() { return 0; } +EOF + : > "${TEST_DIR}/distro_helpers.sh" + cat > "${TEST_DIR}/bin/realm" <<'EOF' +#!/bin/sh +cat >/dev/null +exit "${REALM_EXIT_CODE:-0}" +EOF + cat > "${TEST_DIR}/bin/tee" <<'EOF' +#!/bin/sh +cat >/dev/null +EOF + cat > "${TEST_DIR}/bin/run-script" <"${TEST_DIR}/stderr" +EOF + chmod +x "${TEST_DIR}/bin/realm" "${TEST_DIR}/bin/tee" "${TEST_DIR}/bin/run-script" + } + + cleanup() { + rm -rf "${TEST_DIR}" + } + + BeforeEach 'setup' + AfterEach 'cleanup' + + It 'preserves realm join failures without tracing the password' + When run env \ + PATH="${TEST_DIR}/bin:${PATH}" \ + CSE_HELPERS_FILEPATH="${TEST_DIR}/helpers.sh" \ + CSE_DISTRO_HELPERS_FILEPATH="${TEST_DIR}/distro_helpers.sh" \ + CUSTOM_SEARCH_REALM_PASSWORD='realm-password-sentinel' \ + CUSTOM_SEARCH_REALM_USER='test-user' \ + CUSTOM_SEARCH_DOMAIN_NAME='example.com' \ + REALM_EXIT_CODE=37 \ + "${TEST_DIR}/bin/run-script" + + The status should equal 37 + The value "$(grep -F 'realm-password-sentinel' "${TEST_DIR}/stderr")" should equal '' + End +End