From 839c05094cb3ecc32b06cb56912339b1624b8c59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Corr=C3=AAa?= Date: Wed, 26 Aug 2026 12:28:02 -0300 Subject: [PATCH] dhcp: fix boot time sync when a BMC USB NIC wins the DHCP race MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On machines whose BMC exposes a virtual USB NIC, the one shot dhcpcd call exits as soon as that interface gets its link local lease from the BMC, which happens well before the real NICs finish negotiating carrier. The boot then continues with no default route, no DNS and no DHCP option 42, so the ntpd call that follows dies on "bad address 'pool.ntp.org'" and the clock is never set. The dhcpcd service picks up the real lease a moment later, but nothing retries the time sync, and openntpd cannot recover on its own because it only slews. A wrong clock then hard fails the tink-worker image pull on TLS certificate validity. Three fixes to files/dhcp.sh: - Catch SIGALRM. busybox ntpd uses it for its own timeouts, and the alarm terminated this script along with it, so the existing retry and manual fallback logic never ran. Catching rather than ignoring keeps ntpd's own timeout working: a caught signal is reset to its default disposition in the child, while an ignored one is inherited. - Prefer the NTP servers leased via DHCP option 42, falling back to pool.ntp.org. This drops the dependency on name resolution for setting the clock and makes time sync work on isolated provisioning networks that have no outbound DNS, or no internet access at all. - Retry the one shot dhcpcd until some interface actually holds the default route, excluding interfaces that hold a lease without one. A plain re-run cannot help: -1 exits after the first interface is configured, `persistent` leaves the BMC lease in place on exit, and the lease directory survives across invocations, so the next call is satisfied from that cached lease sooner than the real NICs can finish negotiating. `waitip 4` does not help either, as the BMC lease already supplies an IPv4 address. Excluding the routeless interfaces forces dhcpcd to race the ones still unconfigured; an interface that has not leased yet is never excluded, so a slow real NIC is still picked up. The time sync logic is factored into sync_time/try_ntp_servers/ ntp_servers/default_route_iface so both the option 42 preference and the manual date fallback share one retry path. Signed-off-by: José Corrêa --- files/dhcp.sh | 127 +++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 106 insertions(+), 21 deletions(-) diff --git a/files/dhcp.sh b/files/dhcp.sh index 6b59b86e..04e8e052 100755 --- a/files/dhcp.sh +++ b/files/dhcp.sh @@ -7,6 +7,82 @@ # false: run the dhcp client as a service set -x +# busybox ntpd uses SIGALRM for its own timeouts, and the alarm terminates this script +# along with it, so the retry and fallback logic below never runs. Catching the signal +# rather than ignoring it keeps ntpd's own timeout working: a caught signal is reset to +# its default disposition in the child, while an ignored one is inherited. +trap ':' ALRM + +# Print the interface holding the default route, if there is one. +default_route_iface() { + awk 'NR > 1 && $2 == "00000000" && $8 == "00000000" { print $1; exit }' /proc/net/route +} + +# Print the interfaces dhcpcd has already leased, from its lease directory. Handles both +# the .lease and dhcpcd-.lease naming used across dhcpcd versions. +leased_ifaces() { + for lease in /var/lib/dhcpcd/*.lease; do + [ -e "${lease}" ] || continue + iface="${lease##*/}" + iface="${iface%.lease}" + printf '%s ' "${iface#dhcpcd-}" + done +} + +# Print the NTP servers leased via DHCP option 42 on the given interface, followed by the +# public pool as a fallback. Preferring the leased servers keeps time sync working on +# provisioning networks that have no outbound DNS or no internet access at all, and it +# avoids depending on name resolution to set the clock. +ntp_servers() { + leased=$(/sbin/dhcpcd -U "$1" 2>/dev/null | sed -n "s/^ntp_servers='\(.*\)'$/\1/p" | tr ',' ' ') + echo "${leased} pool.ntp.org" +} + +# Try each server in turn, returning as soon as one of them sets the clock. +try_ntp_servers() { + for server in $1; do + if /usr/sbin/ntpd -n -q -dd -p "$server"; then + echo "time synced from ${server}; date is now: '$(date)'" + return 0 + fi + done + + return 1 +} + +sync_time() { + iface=$(default_route_iface) + if [ -z "$iface" ]; then + echo "no default route; not syncing time" + return 1 + fi + + servers=$(ntp_servers "$iface") + echo "syncing time from '${servers}' via ${iface}; date: '$(date)'" + if try_ntp_servers "$servers"; then + return 0 + fi + + echo "ntpd call failed; setting time manually and retrying" + # set system time to the date of the dhcpd binary file + # this should recover from ntpd failures due to time being too far off + date -s "$(stat -c %y /sbin/dhcpcd | cut -d'.' -f1)" || true + + tries=1 # retry up to 5 times + while [ $tries -le 5 ]; do + echo "waiting 1 second before retrying ntpd call; try #$tries ; date is now: '$(date)'" + sleep 1 + if try_ntp_servers "$servers"; then + echo "ntpd retry call succeeded on try #$tries" + return 0 + fi + echo "ntpd retry call failed on try #$tries" + tries=$((tries + 1)) + done + + return 1 +} + run_dhcp_client() { one_shot="$1" al="e*" @@ -25,30 +101,39 @@ run_dhcp_client() { # always return true for the one shot dhcp call so it doesn't block Hook from starting up. # the --nobackground is not used here because when it is used, dhcpcd doesn't honor the --timeout option # and waits indefinitely for a response. For one shot, we want to timeout after the 30 second default. - /sbin/dhcpcd -f /dhcpcd.conf --allowinterfaces "${al}" -1 || true + # + # One shot mode exits as soon as any allowed interface gets a lease. On machines whose + # BMC exposes a virtual USB NIC, that first lease is often a link local one handed out + # by the BMC itself, acquired well before the real NICs finish negotiating carrier. The + # result is a boot with no default route, no DNS and no option 42. + # + # Simply re-running the same call does not help, and actively hurts: `persistent` keeps + # the BMC lease configured on exit and the lease directory survives across invocations, + # so the next -1 call is satisfied from that cached lease sooner than the real NICs can + # finish negotiating. `waitip 4` does not help either, as the BMC lease already supplies + # an IPv4 address. Each retry therefore excludes the interfaces that hold a lease but no + # default route, forcing dhcpcd to race the interfaces that are still unconfigured. An + # interface that has not leased yet is never excluded, so a slow real NIC is still caught. + tries=1 + deny="" + while [ $tries -le 5 ]; do + if [ -n "${deny}" ]; then + /sbin/dhcpcd -f /dhcpcd.conf --allowinterfaces "${al}" --denyinterfaces "${deny}" -1 || true + else + /sbin/dhcpcd -f /dhcpcd.conf --allowinterfaces "${al}" -1 || true + fi + if [ -n "$(default_route_iface)" ]; then + break + fi + deny="$(leased_ifaces)" + echo "no default route after dhcpcd attempt #$tries; excluding '${deny}' and retrying" + sleep 2 + tries=$((tries + 1)) + done # use busybox's ntpd to set the time after getting an IP address; don't fail echo "sleep 1 second before calling ntpd; date: '$(date)'" && sleep 1 - if ! /usr/sbin/ntpd -n -q -dd -p pool.ntp.org; then - echo "ntpd call failed; setting time manually and retrying" - # set system time to the date of the dhcpd binary file - # this should recover from ntpd failures due to time being too far off - date -s "$(stat -c %y /sbin/dhcpcd | cut -d'.' -f1)" || true - tries=1 # retry up to 5 times - while [ $tries -le 5 ]; do - echo "waiting 1 second before retrying ntpd call; try #$tries ; date is now: '$(date)'" - sleep 1 - if /usr/sbin/ntpd -n -q -dd -p pool.ntp.org; then - echo "ntpd retry call succeeded on try #$tries; date is now: '$(date)'" - break - else - echo "ntpd retry call failed on try #$tries" - fi - tries=$((tries + 1)) - done - else - echo "ntpd call succeeded; date is now: '$(date)'" - fi + sync_time || true else /sbin/dhcpcd --nobackground -f /dhcpcd.conf --allowinterfaces "${al}" fi