From 00c1414865c61115abcb7c76640ec9f049cf2d41 Mon Sep 17 00:00:00 2001 From: Richard Oliver Date: Fri, 7 Aug 2026 12:21:29 +0000 Subject: [PATCH 1/3] rpi-make-boot-image: Choose a FAT width that suits an explicit IMAGE_SIZE Left to itself mkfs.fat picks the FAT width from the size of the device alone, so an explicit IMAGE_SIZE of 64MB or more failed outright: at the default one sector per cluster the resulting cluster count lies well beyond what the chosen width can address. Determine the width from the number of clusters the image will actually hold, and pass that to mkfs.fat. --- tools/rpi-make-boot-image | 92 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/tools/rpi-make-boot-image b/tools/rpi-make-boot-image index f875aa2..84aea83 100755 --- a/tools/rpi-make-boot-image +++ b/tools/rpi-make-boot-image @@ -18,6 +18,13 @@ SECTORS_PER_CLUSTER=${SECTORS_PER_CLUSTER:-1} DISK_GEOMETRY=${DISK_GEOMETRY:-1/1} ROOT_DIR_ENTRIES=${ROOT_DIR_ENTRIES:-256} +# mkfs.fat derives the FAT width from the number of clusters and rejects a -F +# option that disagrees with it. Cluster counts between MAX_CLUSTERS_FAT12 and +# MIN_CLUSTERS_FAT16 are accepted at neither width. +MAX_CLUSTERS_FAT12=4084 +MIN_CLUSTERS_FAT16=4087 +MAX_CLUSTERS_FAT16=65524 + # Add 16k to the size calculation to reserve some space for the FAT, # directory entries and rounding up files to cluster sizes. FAT_OVERHEAD=${FAT_OVERHEAD:-16} @@ -37,6 +44,88 @@ die() { exit 1 } +# FAT32 reserves room for the FSInfo sector and for the backup boot sector, +# which must be at sector 6. These match the mkfs.fat defaults. +reserved_sectors() { + if [ "$1" = "32" ]; then + echo 32 + else + echo 1 + fi +} + +# Sectors occupied by a FAT holding the given number of clusters +fat_sectors() { + fs_entries="$(($1 + 2))" + + case "$2" in + 12) fs_bytes="$(((fs_entries * 3 + 1) / 2))" ;; + 16) fs_bytes="$((fs_entries * 2))" ;; + 32) fs_bytes="$((fs_entries * 4))" ;; + *) die "Unsupported FAT size $2" ;; + esac + + echo "$(((fs_bytes + SECTOR_SIZE - 1) / SECTOR_SIZE))" +} + +# Total sectors needed for the reserved sectors, the FAT, the root directory +# and the given number of data clusters +layout_sectors() { + ls_clusters="$1" + ls_width="$2" + ls_root_entries="$3" + + ls_sectors="$(reserved_sectors "${ls_width}")" + ls_sectors="$((ls_sectors + $(fat_sectors "${ls_clusters}" "${ls_width}")))" + + # FAT32 holds the root directory in the data region + if [ "${ls_width}" != "32" ]; then + ls_sectors="$((ls_sectors + ls_root_entries * 32 / SECTOR_SIZE))" + fi + + echo "$((ls_sectors + ls_clusters * SECTORS_PER_CLUSTER))" +} + +# Largest number of data clusters whose layout fits in the given sectors. The +# size of the FAT depends on the number of clusters it describes, so search +# for the largest count that still fits rather than solving directly. +max_clusters() { + mc_lo=1 + mc_hi="$(($1 / SECTORS_PER_CLUSTER + 1))" + + while [ "${mc_lo}" -lt "${mc_hi}" ]; do + mc_mid="$(((mc_lo + mc_hi + 1) / 2))" + if [ "$(layout_sectors "${mc_mid}" "$2" "$3")" -le "$1" ]; then + mc_lo="${mc_mid}" + else + mc_hi="$((mc_mid - 1))" + fi + done + + echo "${mc_lo}" +} + +# Narrowest FAT that mkfs.fat will accept for a filesystem of the given number +# of sectors. Left to itself mkfs.fat picks the width from the size of the +# device alone, which fails whenever the cluster size in use puts the cluster +# count outside the range permitted for that width. +select_fat_width() { + sf_clusters="$(max_clusters "$1" 12 "$2")" + if [ "${sf_clusters}" -le "${MAX_CLUSTERS_FAT12}" ]; then + echo 12 + return + fi + + sf_clusters="$(max_clusters "$1" 16 "$2")" + if [ "${sf_clusters}" -ge "${MIN_CLUSTERS_FAT16}" ] && \ + [ "${sf_clusters}" -le "${MAX_CLUSTERS_FAT16}" ]; then + echo 16 + return + fi + + echo 32 +} + createfs() { size_kib="$1" image="$2" @@ -229,6 +318,9 @@ createstaging() { # Add a little padding for FAT etc IMAGE_SIZE=$((IMAGE_SIZE + FAT_OVERHEAD)) + elif [ -z "${FAT_SIZE}" ]; then + FAT_SIZE="$(select_fat_width "$((IMAGE_SIZE * 1024 / SECTOR_SIZE))" \ + "${ROOT_DIR_ENTRIES}")" fi echo "Using IMAGE_SIZE of ${IMAGE_SIZE}" From c40f6b870275dc12e125e734f6b0c9b247319036 Mon Sep 17 00:00:00 2001 From: Richard Oliver Date: Tue, 5 Aug 2025 13:49:00 +0100 Subject: [PATCH 2/3] rpi-make-boot-image: Fix long file name entry estimation The 8.3 name pattern required an extension, and was matched against the full path rather than the file name. Both caused the number of directory entries needed for long file names to be over-estimated. --- tools/rpi-make-boot-image | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/rpi-make-boot-image b/tools/rpi-make-boot-image index 84aea83..42d20e0 100755 --- a/tools/rpi-make-boot-image +++ b/tools/rpi-make-boot-image @@ -236,7 +236,7 @@ estimate_fat_clusters() { # ! # $ % & ' ( ) - @ ^ _ ` { } ~ # Values 128-255 VALID_MSDOS_FILENAME_CHAR='[A-Z0-9 \!#\$%\&'"'"'\(\)\-@\^_`\{\}~\x80-\xff]' - VALID_MSDOS_FILENAME="^${VALID_MSDOS_FILENAME_CHAR}{1,8}(?:\.${VALID_MSDOS_FILENAME_CHAR}{1,3})$" + VALID_MSDOS_FILENAME="^${VALID_MSDOS_FILENAME_CHAR}{1,8}(?:\.${VALID_MSDOS_FILENAME_CHAR}{1,3})?$" local msdos_compatible_filename msdos_compatible_filename="0" @@ -245,7 +245,7 @@ estimate_fat_clusters() { if [ "$msdos_compatible_filename" -ne "0" ]; then local ucs2_bytes - ucs2_bytes="$(printf "%s" "${file_name}" | iconv --to-code=UCS2 | wc --bytes)" + ucs2_bytes="$(printf "%s" "${file_base_name}" | iconv --to-code=UCS2 | wc --bytes)" dir_entries="$((dir_entries + ((ucs2_bytes + 25)/26)))" fi done From 741f53e314a31dbac9a46a1d1a62370a735e89b8 Mon Sep 17 00:00:00 2001 From: Richard Oliver Date: Tue, 5 Aug 2025 15:14:45 +0100 Subject: [PATCH 3/3] rpi-make-boot-image: Size FAT12 and FAT16 images from their real layout The image size was derived from a cluster count that counted the root directory entries as data but never accounted for the root directory area itself, which on FAT12 and FAT16 sits outside the data region, and the number of root entries was fixed at 256 regardless of the source files. Size the image instead from the reserved sectors, the FAT, the root directory and the data clusters in turn, allocate it with truncate and drop the -C option so that mkfs.fat is given an exact sector count. Take the number of root directory entries from the source files, rounded up to a whole sector so that the Linux vfat driver will mount the result, and enforce a minimum data region so that very small payloads still produce a filesystem. FAT_OVERHEAD now specifies spare space to leave free in the filesystem and defaults to none, rather than being an allowance for metadata that is now accounted for exactly. --- tools/rpi-make-boot-image | 220 ++++++++++++++++++++++++++++---------- 1 file changed, 161 insertions(+), 59 deletions(-) diff --git a/tools/rpi-make-boot-image b/tools/rpi-make-boot-image index 42d20e0..bb76378 100755 --- a/tools/rpi-make-boot-image +++ b/tools/rpi-make-boot-image @@ -16,7 +16,8 @@ NAME="$(basename "$0")" SECTOR_SIZE=${SECTOR_SIZE:-512} SECTORS_PER_CLUSTER=${SECTORS_PER_CLUSTER:-1} DISK_GEOMETRY=${DISK_GEOMETRY:-1/1} -ROOT_DIR_ENTRIES=${ROOT_DIR_ENTRIES:-256} +# Calculated from the source files unless set +ROOT_DIR_ENTRIES=${ROOT_DIR_ENTRIES:-0} # mkfs.fat derives the FAT width from the number of clusters and rejects a -F # option that disagrees with it. Cluster counts between MAX_CLUSTERS_FAT12 and @@ -25,9 +26,12 @@ MAX_CLUSTERS_FAT12=4084 MIN_CLUSTERS_FAT16=4087 MAX_CLUSTERS_FAT16=65524 -# Add 16k to the size calculation to reserve some space for the FAT, -# directory entries and rounding up files to cluster sizes. -FAT_OVERHEAD=${FAT_OVERHEAD:-16} +# Spare space, in KiB, to leave free in the filesystem +FAT_OVERHEAD=${FAT_OVERHEAD:-0} + +# mkfs.fat refuses to create a filesystem whose data region is much smaller +# than this, whatever the sector and cluster sizes in use. +MIN_DATA_KIB=64 HAVE_MCOPY=false @@ -127,10 +131,16 @@ select_fat_width() { } createfs() { - size_kib="$1" + image_sectors="$1" image="$2" + root_dir_entries="$3" volume_label="BOOT" + + rm -f "${image}" + truncate -s "$((image_sectors * SECTOR_SIZE))" "${image}" || \ + die "Failed to allocate ${image}" + if [ -n "${SECTORS_PER_CLUSTER}" ]; then SECTORS_PER_CLUSTER="-s ${SECTORS_PER_CLUSTER}" fi @@ -143,10 +153,12 @@ createfs() { fat_size="-F ${FAT_SIZE}" fi - mkfs.fat -C -f 1 \ + # -a suppresses alignment of the FAT and root directory to cluster + # boundaries, which would otherwise consume sectors not budgeted for above. + mkfs.fat -a -f 1 \ ${SECTORS_PER_CLUSTER} -n "${volume_label}" \ ${fat_size} ${DISK_GEOMETRY} \ - -S "${SECTOR_SIZE}" -r "${ROOT_DIR_ENTRIES}" "${image}" ${size_kib} || \ + -S "${SECTOR_SIZE}" -r "${root_dir_entries}" "${image}" || \ die "Failed to create FAT filesystem" } @@ -195,35 +207,32 @@ copyfiles() { fi } -estimate_fat_clusters() { - local cluster_size - cluster_size="$1" +# Extract directory entry calculation into a separate function +estimate_directory_entries() { local directory - directory="$2" - - local clusters - clusters="0" - local subdir - - for subdir in $(find "$directory" -mindepth 1 -maxdepth 1 -type d); do - local sd_clusters - sd_clusters="$(estimate_fat_clusters "$cluster_size" "$subdir")" - clusters="$((clusters + sd_clusters))" - done - - # Determine number of clusters required for file contents - local file_clusters - file_clusters="$(find "$directory" -maxdepth 1 -type f -exec du --apparent-size --block-size="${cluster_size}" {} + | awk '{clust=clust+$1} END {print clust}')" - clusters="$((clusters + file_clusters))" + directory="$1" + local is_root_dir + is_root_dir="${2:-0}" # Default to 0 (not root directory) - # Determine number of clusters required for directory entries - # Two additional entries are required for "." and ".." + # Determine number of directory entries required local dir_entries - dir_entries="2" + if [ "$is_root_dir" = "1" ]; then + # The root directory has no "." or ".." entries but does hold the + # volume label + dir_entries="1" + else + # Two additional entries are required for "." and ".." in subdirectories + dir_entries="2" + fi + local file_name local file_base_name for file_name in "${directory}"/*; do + # An unmatched glob expands to the pattern itself. A dangling symbolic + # link still occupies a directory entry. + [ -e "${file_name}" ] || [ -L "${file_name}" ] || continue + file_base_name="$(basename "${file_name}")" # Always at least one entry @@ -250,11 +259,74 @@ estimate_fat_clusters() { fi done - # 32-bytes are required for each entry - clusters="$((clusters + ((dir_entries * 32) + cluster_size - 1)/cluster_size))" + echo "$dir_entries" +} + +# Specific function for root directory entries in FAT12/16 +estimate_root_directory_entries() { + local directory + directory="$1" + estimate_directory_entries "$directory" 1 +} + +estimate_fat_clusters() { + local cluster_size + cluster_size="$1" + local directory + directory="$2" + local include_root_dir_entries + include_root_dir_entries="${3:-1}" # Default to 1 (include) for backwards compatibility + + local clusters + clusters="0" + local subdir + + for subdir in $(find "$directory" -mindepth 1 -maxdepth 1 -type d); do + local sd_clusters + # Subdirectories always include directory entries regardless of FAT type + sd_clusters="$(estimate_fat_clusters "$cluster_size" "$subdir" 1)" + clusters="$((clusters + sd_clusters))" + done + + # Determine number of clusters required for file contents + local file_clusters + file_clusters="$(find "$directory" -maxdepth 1 -type f -exec du --apparent-size --block-size="${cluster_size}" {} + | awk '{clust=clust+$1} END {print clust}')" + clusters="$((clusters + file_clusters))" + + # Only include directory entry clusters if requested (FAT32 or non-root directories) + if [ "$include_root_dir_entries" = "1" ]; then + local dir_entries + dir_entries="$(estimate_directory_entries "$directory")" + + # 32-bytes are required for each entry + local dir_clusters + dir_clusters="$(((dir_entries * 32 + cluster_size - 1)/cluster_size))" + clusters="$((clusters + dir_clusters))" + fi + echo "$clusters" } +# Entry point for FAT32 (includes all directory entries as clusters) +estimate_fat_clusters_fat32() { + local cluster_size + cluster_size="$1" + local directory + directory="$2" + + estimate_fat_clusters "$cluster_size" "$directory" 1 +} + +# Entry point for FAT12/FAT16 (excludes root directory entries from cluster calculation) +estimate_fat_clusters_fat12_16() { + local cluster_size + cluster_size="$1" + local directory + directory="$2" + + estimate_fat_clusters "$cluster_size" "$directory" 0 +} + createstaging() { source_dir="$1" staging="$2" @@ -289,41 +361,67 @@ createstaging() { rm -f "${staging}/fixup4x.dat" fi + cluster_size="$((SECTOR_SIZE * SECTORS_PER_CLUSTER))" + + # Calculate actual root directory entries needed for FAT12/16. The Linux + # vfat driver refuses to mount a filesystem whose root directory does not + # span a whole number of sectors. + required_root_entries="$(estimate_root_directory_entries "${staging}")" + estimated_root_entries="${required_root_entries}" + if [ "${ROOT_DIR_ENTRIES}" != 0 ]; then + estimated_root_entries="${ROOT_DIR_ENTRIES}" + if [ "${ROOT_DIR_ENTRIES}" -lt "${required_root_entries}" ]; then + echo "Warning: ROOT_DIR_ENTRIES=${ROOT_DIR_ENTRIES} is fewer than the" \ + "${required_root_entries} entries the source files require" + fi + fi + entries_per_sector="$((SECTOR_SIZE / 32))" + ROOT_ENTRIES="$(((estimated_root_entries + entries_per_sector - 1) \ + / entries_per_sector * entries_per_sector))" + if [ "${IMAGE_SIZE}" = 0 ]; then # Estimate the size of the image in clusters - cluster_size="$((SECTOR_SIZE * SECTORS_PER_CLUSTER))" - clusters="$(estimate_fat_clusters "${cluster_size}" "${staging}")" + overhead_clusters="$(((FAT_OVERHEAD * 1024 + cluster_size - 1)/cluster_size))" - IMAGE_SIZE="$((clusters * cluster_size))" + # First, estimate assuming FAT32 to determine potential FAT type + clusters="$(($(estimate_fat_clusters_fat32 "${cluster_size}" "${staging}") \ + + overhead_clusters))" - root_dir_sectors="$((((ROOT_DIR_ENTRIES * 32) + cluster_size - 1)/cluster_size))" # FAT32/FAT16 determined by number of clusters - if [ "$clusters" -gt "65526" ]; then + if [ "$clusters" -gt "${MAX_CLUSTERS_FAT16}" ]; then FAT_SIZE="32" - fat_table_sectors="$(((((clusters + 2)*4) + SECTOR_SIZE - 1)/SECTOR_SIZE))" - elif [ "$clusters" -gt "4085" ]; then - FAT_SIZE="16" - fat_table_sectors="$(((((clusters + 2)*2) + SECTOR_SIZE - 1)/SECTOR_SIZE))" - # Add some sectors based on ROOT_DIR_ENTRIES - fat_table_sectors="$((fat_table_sectors + root_dir_sectors))" else - FAT_SIZE="12" - #12 bits per cluster = 3 bytes for 2 clusters - fat_table_sectors=$(((((clusters + 2)*3+1) / 2 + SECTOR_SIZE - 1)/SECTOR_SIZE)) - # Add some sectors based on ROOT_DIR_ENTRIES - fat_table_sectors="$((fat_table_sectors + root_dir_sectors))" + # Will be FAT12 or FAT16, so recalculate without the root directory + # entries, which live outside the data region at those widths + clusters="$(($(estimate_fat_clusters_fat12_16 "${cluster_size}" "${staging}") \ + + overhead_clusters))" + + if [ "$clusters" -gt "${MAX_CLUSTERS_FAT12}" ]; then + FAT_SIZE="16" + if [ "$clusters" -lt "${MIN_CLUSTERS_FAT16}" ]; then + clusters="${MIN_CLUSTERS_FAT16}" + fi + else + FAT_SIZE="12" + fi + fi + + # mkfs.fat refuses to create a filesystem with a very small data region + min_clusters="$(((MIN_DATA_KIB * 1024 + cluster_size - 1)/cluster_size))" + if [ "$clusters" -lt "$min_clusters" ]; then + clusters="$min_clusters" + fi + + IMAGE_SECTORS="$(layout_sectors "${clusters}" "${FAT_SIZE}" "${ROOT_ENTRIES}")" + IMAGE_SIZE="$(((IMAGE_SECTORS * SECTOR_SIZE + 1023)/1024))" + else + IMAGE_SECTORS="$((IMAGE_SIZE * 1024 / SECTOR_SIZE))" + if [ -z "${FAT_SIZE}" ]; then + FAT_SIZE="$(select_fat_width "${IMAGE_SECTORS}" "${ROOT_ENTRIES}")" fi - IMAGE_SIZE="$((IMAGE_SIZE + fat_table_sectors * SECTOR_SIZE))" - IMAGE_SIZE="$(((IMAGE_SIZE + 1023)/1024))" - - # Add a little padding for FAT etc - IMAGE_SIZE=$((IMAGE_SIZE + FAT_OVERHEAD)) - elif [ -z "${FAT_SIZE}" ]; then - FAT_SIZE="$(select_fat_width "$((IMAGE_SIZE * 1024 / SECTOR_SIZE))" \ - "${ROOT_DIR_ENTRIES}")" fi - echo "Using IMAGE_SIZE of ${IMAGE_SIZE}" + echo "Using IMAGE_SIZE of ${IMAGE_SIZE} with ${ROOT_ENTRIES} root directory entries" if [ "${IMAGE_SIZE}" -gt "$((20 * 1024))" ]; then echo "Warning: Large image size detected. Try removing unused files." @@ -334,6 +432,9 @@ checkDependencies() { if ! mkfs.fat --help > /dev/null 2> /dev/null ; then die "mkfs.fat is required. Run this script on Linux" fi + if ! truncate --help > /dev/null 2> /dev/null ; then + die "truncate is required. Run this script on Linux" + fi if mcopy --help > /dev/null 2> /dev/null ; then HAVE_MCOPY=true fi @@ -361,10 +462,11 @@ Environment variables: The following environment variables may be specified to optionally override mkfs.vfat arguments to help minimise the size of the boot image: -Name mkfs.vfat parameter +Name mkfs.vfat parameter / Description SECTOR_SIZE -S -ROOT_DIR_ENTRIES -r +ROOT_DIR_ENTRIES -r (calculated from the source files if unset) FAT_SIZE -F +FAT_OVERHEAD Spare space to leave free in the filesystem, in KiB EOF exit 0 @@ -406,7 +508,7 @@ createstaging "${SOURCE_DIR}" "${STAGING}" "${BOARD}" echo "Creating FAT file system" TMP_IMAGE="${TMP_DIR}/boot.img" -createfs ${IMAGE_SIZE} "${TMP_IMAGE}" +createfs "${IMAGE_SECTORS}" "${TMP_IMAGE}" "${ROOT_ENTRIES}" echo "Copying files to file system image ${TMP_IMAGE}" copyfiles "${TMP_IMAGE}" "${staging}"/*