diff --git a/README.md b/README.md index ffc16ec4..c119f434 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,40 @@ $ ./jmsctl.sh install ## 管理命令 +新安装默认使用 `postgres:18.6-bookworm`。统一的 `compose/postgresql.yml` 显式设置 +`PGDATA=/var/lib/postgresql/data`,继续把 `${VOLUME_DIR}/postgresql/data` +挂载到该目录,以兼容 JDMC 的主备重建、回切和启动写入隔离。同时显式挂载 +`${VOLUME_DIR}/postgresql:/var/lib/postgresql`,避免 PostgreSQL 18 镜像自动创建 +额外匿名卷,导致 JDMC 的 Compose 挂载一致性校验失败。 +主 Compose 的镜像固定为 `postgres:18.6-bookworm`,不需要数据库镜像配置项。 +旧版本按实际数据版本选择镜像,仅在迁移前自动生成 `${CONFIG_DIR}/postgresql/image-.yml` +运行时镜像覆盖文件,使用相同的 `PGDATA`;迁移完成后使用主 Compose 中的 18 镜像。 +运行时文件会保留,避免影响 JDMC 已记录的 Compose 绑定。JDMC 升级已有主备时继续使用 +已绑定的旧 Compose 文件,在协调迁移切换时更新数据库配置和绑定,不改写旧文件。 +单机生成数据库 Compose 命令时会补齐父目录的可穿越权限,避免旧目录为 `700` 时容器 +无法访问;数据目录 `postgresql/data` 的权限保持不变。 + +从 JumpServer 5.0 等旧应用版本升级时,内置 PostgreSQL 10–17 会在同一次维护中 +迁移到 `postgres:18.6-bookworm`。安装器先加载旧镜像和 18 镜像,停止应用写入并禁用 +自动重启,使用 18 客户端导出全部数据库和角色,在隔离目录恢复并校验后再切换数据目录。 +迁移前仍按 `PG_VERSION` 选择旧镜像;不能让 18 直接打开旧数据。 +迁移完成后更新业务表结构并恢复原先运行的应用。外置数据库不会被升级。 + +旧目录、逻辑备份和进度保存在 `${VOLUME_DIR}/postgresql/data.upgrade18/`; +`original/` 是原数据,`cluster.sql` 包含全库及角色凭据,目录和文件必须保持私有。 +迁移备份是必需步骤,不受 `SKIP_BACKUP_DB` 影响。中断时应用保持停止,重试同一升级 +继续处理;完成迁移前普通启动会被拒绝。恢复业务后新产生的数据不会写回旧目录, +不能直接把旧目录切回去。空间预检按旧数据三倍加 1 GiB 估算;大对象或高压缩数据可能需要更多空间。 +自动迁移要求可验证的管理员账号、标准数据路径和配置;自定义表空间、外置配置或 SSL +文件、逻辑复制、非 libc 模板区域设置等定制部署在预检时中止,需要单独迁移。原镜像及旧数据不会自动清理。 + +JDMC 主备由两台协调维护:停止业务和选举,迁移主库、重新建立 18 备库,校验流复制、 +同步提交和启动写入隔离,再更新应用并退出维护。两台 JDMC 必须先具备此迁移能力; +新的全量升级包声明 `maintenance-pg18-v1`,旧 JDMC 会拒绝该包,避免跳过数据库迁移。 +`pg_rewind` 不承担数据库大版本迁移。 +参考 [PostgreSQL 升级说明](https://www.postgresql.org/docs/18/upgrading.html) +和 [主备版本要求](https://www.postgresql.org/docs/18/warm-standby.html)。 + ``` # 启动 $ ./jmsctl.sh start diff --git a/compose/postgresql.yml b/compose/postgresql.yml index 917a25aa..0ce7b29b 100644 --- a/compose/postgresql.yml +++ b/compose/postgresql.yml @@ -1,6 +1,6 @@ services: postgresql: - image: postgres:16.15-bookworm + image: postgres:18.6-bookworm container_name: jms_postgresql restart: always environment: @@ -8,7 +8,11 @@ services: POSTGRES_DB: $DB_NAME POSTGRES_USER: $DB_USER POSTGRES_PASSWORD: $DB_PASSWORD + # Keep the data path used by existing installations and JDMC on every major. + PGDATA: /var/lib/postgresql/data volumes: + # PostgreSQL 18 declares this parent volume; make it explicit for JDMC. + - ${VOLUME_DIR}/postgresql:/var/lib/postgresql - ${VOLUME_DIR}/postgresql/data:/var/lib/postgresql/data healthcheck: test: "pg_isready -q -U $$POSTGRES_USER" diff --git a/config-example.txt b/config-example.txt index 019a087c..2a791534 100644 --- a/config-example.txt +++ b/config-example.txt @@ -83,6 +83,8 @@ DB_HOST=postgresql # For a new built-in DB_ENGINE=mysql installation, choose mysql:8.0 or # mariadb:10.6. Existing mysql/data or mariadb/data takes precedence. DB_IMAGE=mariadb:10.6 +# New PostgreSQL installations use 18.6-bookworm. Upgrades select the old image +# from the existing data version until migration to 18 has completed. DB_PORT=5432 DB_USER=postgres DB_PASSWORD= diff --git a/scripts/3_load_images.sh b/scripts/3_load_images.sh index 4bd0b1d4..892087b2 100644 --- a/scripts/3_load_images.sh +++ b/scripts/3_load_images.sh @@ -60,6 +60,15 @@ function load_image_files() { function main() { # Panda belongs to remote publishers, even if packaging flags are inherited. local INCLUDE_PANDA_IMAGE=0 + if [[ "$(get_config HA_MODE)" == 1 && -n "${JDMC_HA_MAINTENANCE_ID:-}" && + "${JDMC_HA_PG18_COORDINATED:-0}" != 1 && "${POSTGRESQL_MAJOR_UPGRADE:-0}" != 1 && "$(get_config DB_ENGINE)" == postgresql ]]; then + local pg_major + pg_major=$(get_internal_postgresql_data_version) || return 1 + if [[ -n "${pg_major}" && "${pg_major}" -lt 18 ]]; then + log_error 'This installer requires coordinated PostgreSQL 18 migration. Update JDMC on both nodes before applying this package.' + return 1 + fi + fi if [[ -d "${IMAGE_DIR}" && $(find "${IMAGE_DIR}" -type f -name "*.zst" -print -quit 2>/dev/null) ]]; then load_image_files || return 1 else diff --git a/scripts/7_upgrade.sh b/scripts/7_upgrade.sh index c2ae8e35..f23a0e00 100644 --- a/scripts/7_upgrade.sh +++ b/scripts/7_upgrade.sh @@ -239,7 +239,9 @@ function migrate_internal_mysql_business_account() { function remove_obsolete_containers() { local container existing_containers - local containers=("jms_guacamole" "jms_lina" "jms_luna" "jms_nginx" "jms_xpack" "jms_lb" "jms_omnidb" "jms_kael" "jms_magnus" "jms_video" "jms_lion" "jms_facelive" "jms_panda") + # Kael and Magnus are current services. Keep their containers until the normal + # stop/recreate flow so PostgreSQL migration can record their restart intent. + local containers=("jms_guacamole" "jms_lina" "jms_luna" "jms_nginx" "jms_xpack" "jms_lb" "jms_omnidb" "jms_video" "jms_lion" "jms_facelive" "jms_panda") # XRDP is still supported when explicitly enabled in the enterprise edition. if [[ "$(get_config_or_env USE_XPACK)" != "1" || "$(get_config_or_env XRDP_ENABLED)" != "1" ]]; then @@ -455,6 +457,7 @@ function cleanup_failed_upgrade() { local status=$? safe_to_restore=1 if [[ "${status}" != "0" && "${UPGRADE_COMMITTED}" != "1" ]]; then + fence_internal_postgresql_upgrade || log_error "Could not stop all application containers after the interrupted PostgreSQL migration" if [[ "${UPGRADE_CONFIG_CHANGED}" == "1" && \ "${UPGRADE_DATABASE_MIGRATION_STARTED}" == "0" && \ -f "${backup_config_file:-}" ]]; then @@ -484,13 +487,21 @@ function cleanup_failed_upgrade() { } function backup_db() { - local db_image backup_failed=0 confirm="y" + local db_image migration_backup backup_failed=0 confirm="y" if [[ "${SKIP_BACKUP_DB:-0}" == "1" ]]; then echo "SKIP_BACKUP_DB=${SKIP_BACKUP_DB}, $(gettext 'Skip database backup')" return 0 fi + migration_backup="$(get_config VOLUME_DIR)/postgresql/data.upgrade18/cluster.sql" + if postgresql_upgrade_pending && [[ -s "${migration_backup}" ]]; then + # A retry must not start the temporary Core container while migration is + # fenced. The atomic full-cluster export still contains the original data. + log_success "Resuming PostgreSQL migration; retained full-cluster backup: ${migration_backup}" + return 0 + fi + if ! db_image=$(get_db_images) || [[ -z "${db_image}" ]]; then log_warn 'Could not determine the database backup image.' backup_failed=1 @@ -536,6 +547,10 @@ function db_migrations() { fi if ! perform_db_migrations; then log_error "$(gettext 'Failed to change the table structure')!" + if [[ "${PG18_UPGRADE_ACTIVE:-0}" == 1 ]]; then + log_error 'PostgreSQL migrated, but application schema migration failed; keep business stopped and retry the same upgrade' + exit 1 + fi confirm="n" read_from_input confirm "$(gettext 'Failed to change the table structure. Continue to upgrade')?" "y/n" "${confirm}" if [[ "${confirm}" != "y" ]]; then @@ -660,7 +675,7 @@ function main() { # Database backup starts a Core container using the target version. # Load the offline images before that container is created. echo_yellow "\n3. $(gettext 'Loading Docker Image')" - if ! ALLOW_MISSING_DB_IMAGE=1 bash "${BASE_DIR}/3_load_images.sh"; then + if ! POSTGRESQL_MAJOR_UPGRADE=1 ALLOW_MISSING_DB_IMAGE=1 bash "${BASE_DIR}/3_load_images.sh"; then log_error "$(gettext 'Failed to load Docker images')" exit 1 fi @@ -681,6 +696,14 @@ function main() { echo "$(gettext 'Changing database schema may take a while, please wait patiently')" write_upgrade_recovery_state recovery_required || exit 1 UPGRADE_DATABASE_MIGRATION_STARTED=1 + upgrade_internal_postgresql || { + log_error "PostgreSQL migration did not complete; business remains stopped. Retry the same upgrade." + exit 1 + } + # Remove the former image setting only after cutover; an old HA Compose + # binding may still require it while its database is being migrated. + remove_config POSTGRESQL_IMAGE || exit 1 + gen_safe_config >/dev/null || exit 1 db_migrations echo_yellow "\n7. $(gettext 'Upgrade Docker')" @@ -708,6 +731,10 @@ function main() { log_error "Failed to update CURRENT_VERSION" exit 1 fi + complete_internal_postgresql_upgrade || { + log_error "PostgreSQL migrated, but application recovery is incomplete; retry the same upgrade" + exit 1 + } UPGRADE_COMMITTED=1 trap - EXIT installation_log "upgrade" diff --git a/scripts/gists/image.sh b/scripts/gists/image.sh index 3196f734..fdebd42f 100644 --- a/scripts/gists/image.sh +++ b/scripts/gists/image.sh @@ -38,11 +38,15 @@ function report_image_failure() { function reuse_local_db_image() { local image=$1 - # New offline packages only bundle PostgreSQL. Existing MySQL/MariaDB - # installations retain their database image, including legacy tags. + # New offline packages bundle the default PostgreSQL image. Existing + # databases may reuse their local image when no matching archive is bundled. case "${image}" in mysql:8.0|mariadb:10.6) ;; - postgres:*) is_external_db_image "${image}" || return 1 ;; + postgres:*) + if ! is_external_db_image "${image}"; then + [[ -n "$(get_internal_postgresql_data_version)" && ( "${image}" == "$(get_db_images)" || ( "${POSTGRESQL_MAJOR_UPGRADE:-0}" == 1 && "${image}" == postgres:18.6-bookworm ) ) ]] || return 1 + fi + ;; *) return 1 ;; esac if docker image inspect "${image}" &>/dev/null; then @@ -97,6 +101,10 @@ function get_image_mappings() { if [[ -n "${db_image}" ]] && ! should_skip_external_db_image "${db_image}"; then emit_image_mapping "${db_image}" "${db_image}" fi + if [[ "${POSTGRESQL_MAJOR_UPGRADE:-0}" == 1 && "$(get_config DB_ENGINE)" == postgresql && + ( "$(get_config DB_HOST)" == postgresql || "$(get_config HA_MODE)" == 1 ) && "${db_image}" != postgres:18.6-bookworm ]]; then + emit_image_mapping postgres:18.6-bookworm postgres:18.6-bookworm + fi enabled_services=${OFFLINE_IMAGE_SERVICES:-$(get_enabled_services)} enabled_services=${enabled_services//,/ } diff --git a/scripts/gists/postgresql-upgrade-standalone.sh b/scripts/gists/postgresql-upgrade-standalone.sh new file mode 100644 index 00000000..51faba36 --- /dev/null +++ b/scripts/gists/postgresql-upgrade-standalone.sh @@ -0,0 +1,130 @@ +#!/usr/bin/env bash +# PG18_* variables are inputs to the separately sourced migration library. +# shellcheck disable=SC2034 + +postgresql_upgrade_is_internal() { + [[ "$(get_config DB_ENGINE)" == postgresql && "$(get_config DB_HOST)" == postgresql && "$(get_config HA_MODE)" != 1 ]] +} + +postgresql_upgrade_pending() { + local directory phase + postgresql_upgrade_is_internal || return 1 + directory="$(get_config VOLUME_DIR)/postgresql/data.upgrade18" + [[ -e "${directory}/phase" ]] || return 1 + phase=$(cat "${directory}/phase") || return 0 + [[ "${phase}" != complete ]] +} + +postgresql_upgrade_start_guard() { + if [[ "${PG18_UPGRADE_ACTIVE:-0}" != 1 ]] && postgresql_upgrade_pending; then + log_error 'PostgreSQL migration is pending; retry the same upgrade before starting services' + return 1 + fi +} + +upgrade_internal_postgresql() { + local version container policy running retries service cmd + local -a names=() services=() + postgresql_upgrade_is_internal || return 0 + PG18_DATA="$(get_config VOLUME_DIR)/postgresql/data" + version=$(cat "${PG18_DATA}/PG_VERSION" 2>/dev/null || true) + if [[ ! -e "${PG18_DATA}.upgrade18/phase" ]]; then + [[ -n "${version}" && "${version}" -lt 18 ]] || return 0 + elif [[ "$(cat "${PG18_DATA}.upgrade18/phase")" == complete ]]; then + return 0 + fi + . "${SCRIPT_DIR}/gists/postgresql-upgrade.sh" + PG18_CONTAINER=jms_postgresql PG18_USER=$(get_config DB_USER) PG18_PASSWORD=$(get_config DB_PASSWORD) PG18_PORT=$(get_config DB_PORT 5432) + pg18_paths || return 1 + exec 201>"${PG18_WORK}/upgrade.lock" + flock -n 201 || { log_error "Another PostgreSQL migration is running"; return 1; } + export PG18_UPGRADE_ACTIVE=1 + if [[ ! -f "${PG18_WORK}/phase" ]]; then + docker start "${PG18_CONTAINER}" >/dev/null && wait_container_healthy "${PG18_CONTAINER}" || return 1 + pg18_preflight || return 1 + fi + # Save restart intent before stopping anything. A host restart during the + # migration must not restart an old application against a partially upgraded DB. + if [[ ! -f "${PG18_WORK}/writers" ]]; then + for service in $(get_enabled_services); do + container="jms_${service}" + [[ "${service}" != ansible-executor ]] || container=jms_ansible_executor + names+=("${container}") + done + : >"${PG18_WORK}/writers.partial" + for container in "${names[@]}"; do + if docker inspect "${container}" >/dev/null 2>&1; then + docker inspect -f '{{.Name}} {{.State.Running}} {{.HostConfig.RestartPolicy.Name}} {{.HostConfig.RestartPolicy.MaximumRetryCount}} {{index .Config.Labels "com.docker.compose.service"}}' "${container}" >>"${PG18_WORK}/writers.partial" || return 1 + fi + done + chmod 600 "${PG18_WORK}/writers.partial" && sync "${PG18_WORK}/writers.partial" && + mv "${PG18_WORK}/writers.partial" "${PG18_WORK}/writers" || return 1 + fi + [[ -f "${PG18_WORK}/phase" ]] || pg18_save phase reserved || return 1 + while read -r container running policy retries service; do + [[ "${container}" == /jms_* && "${service}" =~ ^[A-Za-z0-9_.-]+$ ]] || return 1 + if docker inspect "${container}" >/dev/null 2>&1; then + docker update --restart=no "${container}" >/dev/null && docker stop "${container}" >/dev/null || return 1 + [[ "$(docker inspect -f '{{.State.Running}}' "${container}")" == false ]] || return 1 + fi + done <"${PG18_WORK}/writers" + # The data container's old image remains valid until the staged cluster is ready. + if [[ ! -f "${PG18_WORK}/phase" || "$(cat "${PG18_WORK}/phase")" == preparing || "$(cat "${PG18_WORK}/phase")" == reserved ]]; then + docker start "${PG18_CONTAINER}" >/dev/null && wait_container_healthy "${PG18_CONTAINER}" || return 1 + fi + pg18_prepare && pg18_cutover || return 1 + gen_safe_config >/dev/null || return 1 + cmd=$(get_db_compose_cmd db) || return 1 + ${cmd} up -d --no-deps --force-recreate postgresql || return 1 + wait_container_healthy "${PG18_CONTAINER}" || return 1 + [[ "$(pg18_sql "select current_setting('server_version_num')::int/10000;")" == 18 ]] || return 1 + pg18_save phase switched +} + +complete_internal_postgresql_upgrade() { + local container running policy retries service cmd state deadline timeout + local -a services=() + [[ "${PG18_UPGRADE_ACTIVE:-0}" == 1 && -f "${PG18_WORK:-}/writers" ]] || return 0 + # Called only after application schema and JDMC upgrades have succeeded. + while read -r container running policy retries service; do + [[ "${running}" != true ]] || services+=("${service}") + done <"${PG18_WORK}/writers" + if (( ${#services[@]} )); then + cmd=$(get_docker_compose_cmd_line) || return 1 + ${cmd} up -d "${services[@]}" || return 1 + fi + timeout=$(get_config CONTAINER_HEALTH_TIMEOUT 300) + [[ "${timeout}" =~ ^[0-9]+$ ]] || timeout=300 + deadline=$((SECONDS + timeout)) + while read -r container running policy retries service; do + if [[ "${running}" == true ]]; then + while :; do + state=$(docker inspect -f '{{if not .State.Running}}{{.State.Status}}{{else if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "${container}") || return 1 + [[ "${state}" != healthy && "${state}" != running ]] || break + if [[ "${state}" == exited || "${state}" == dead ]] || (( SECONDS >= deadline )); then + log_error "Failed to recover ${container}: ${state}" + return 1 + fi + sleep 2 + done + fi + [[ "${policy}" =~ ^(no|always|unless-stopped|on-failure)$ && "${retries}" =~ ^[0-9]+$ ]] || return 1 + if [[ "${policy}" == on-failure && "${retries}" != 0 ]]; then policy+="${policy:+:}${retries}"; fi + if docker inspect "${container}" >/dev/null 2>&1; then docker update --restart="${policy}" "${container}" >/dev/null || return 1; fi + done <"${PG18_WORK}/writers" + pg18_save phase complete || return 1 + log_success "PostgreSQL is now ${PG18_IMAGE}; original data and migration backup retained in ${PG18_WORK}" +} + +# EXIT cleanup must re-fence even a partly started application on recovery failure. +fence_internal_postgresql_upgrade() { + local service container + [[ "${PG18_UPGRADE_ACTIVE:-0}" == 1 && -f "${PG18_WORK:-}/phase" ]] || return 0 + for service in $(get_enabled_services); do + container="jms_${service}" + [[ "${service}" != ansible-executor ]] || container=jms_ansible_executor + if docker inspect "${container}" >/dev/null 2>&1; then + docker update --restart=no "${container}" >/dev/null && docker stop "${container}" >/dev/null || return 1 + fi + done +} diff --git a/scripts/gists/postgresql-upgrade.sh b/scripts/gists/postgresql-upgrade.sh new file mode 100644 index 00000000..31ff5941 --- /dev/null +++ b/scripts/gists/postgresql-upgrade.sh @@ -0,0 +1,168 @@ +#!/usr/bin/env bash +# Shared by the standalone installer and JDMC's coordinated maintenance worker. +# Callers must fence every application writer before prepare/cutover, and keep +# those fences until the replacement database and application are verified. +PG18_IMAGE=postgres:18.6-bookworm + +pg18_error() { printf '[postgresql-upgrade] %s\n' "$*" >&2; return 1; } +pg18_save() { + local name=$1 value=$2 tmp + tmp=$(mktemp "${PG18_WORK}/.${name}.XXXXXX") || return 1 + printf '%s\n' "${value}" >"${tmp}" && chmod 600 "${tmp}" && sync "${tmp}" && + mv -f -- "${tmp}" "${PG18_WORK}/${name}" && sync -f "${PG18_WORK}" +} + +pg18_paths() { + [[ "${PG18_DATA}" == /* && "${PG18_DATA}" != / && ! -L "${PG18_DATA}" ]] || return 1 + [[ "$(readlink -f "$(dirname "${PG18_DATA}")")/$(basename "${PG18_DATA}")" == "${PG18_DATA}" ]] || + pg18_error 'PostgreSQL migration requires a canonical data directory' || return 1 + PG18_WORK="${PG18_DATA}.upgrade18" + [[ ! -L "${PG18_WORK}" ]] || return 1 + if [[ ! -e "${PG18_WORK}" ]]; then mkdir -m 700 "${PG18_WORK}" || return 1; fi + [[ "$(stat -Lc '%u:%a' "${PG18_WORK}")" == 0:700 ]] || return 1 + PG18_HELPER="jms-pg18-$(printf %s "${PG18_DATA}" | sha256sum | cut -c1-20)" +} + +pg18_sql() { + # Password travels through stdin, never through an argument or a journal. + printf '%s\n' "${PG18_PASSWORD}" | docker exec -i -e PGCONNECT_TIMEOUT=10 "${PG18_CONTAINER}" sh -ceu ' + IFS= read -r PGPASSWORD; export PGPASSWORD + exec psql -X -v ON_ERROR_STOP=1 -h 127.0.0.1 -p "$1" -U "$2" -d postgres -tAqc "$3" + ' sh "${PG18_PORT}" "${PG18_USER}" "$1" +} + +pg18_remove_stale_helper() { + if docker inspect "${PG18_HELPER}" >/dev/null 2>&1; then + [[ "$(docker inspect -f '{{index .Config.Labels "jms.pg18.work"}}' "${PG18_HELPER}")" == "${PG18_WORK}" ]] || return 1 + docker rm -f -v "${PG18_HELPER}" >/dev/null || return 1 + fi +} + +pg18_preflight() { + local version tablespaces settings bytes free + docker image inspect "${PG18_IMAGE}" >/dev/null || return 1 + [[ -f "${PG18_DATA}/PG_VERSION" && ! -L "${PG18_DATA}/PG_VERSION" ]] || return 1 + version=$(cat "${PG18_DATA}/PG_VERSION") || return 1 + [[ "${version}" =~ ^1[0-7]$ ]] || pg18_error "Unsupported migration source: ${version}" || return 1 + [[ "$(pg18_sql "select current_setting('server_version_num')::int/10000;")" == "${version}" ]] || return 1 + [[ "$(pg18_sql 'select rolsuper from pg_roles where rolname=current_user;')" == t ]] || + pg18_error 'A verified local PostgreSQL administrator is required for a full-cluster migration' || return 1 + [[ "${PG18_ALLOW_STANDBY:-0}" == 1 || "$(pg18_sql 'select pg_is_in_recovery();')" == f ]] || return 1 + tablespaces=$(pg18_sql "select count(*) from pg_tablespace where spcname not in ('pg_default','pg_global');") || return 1 + [[ "${tablespaces}" == 0 ]] || pg18_error 'Custom tablespaces require a dedicated migration plan' || return 1 + settings=$(pg18_sql "select current_setting('data_directory')='/var/lib/postgresql/data' and current_setting('config_file')='/var/lib/postgresql/data/postgresql.conf' and current_setting('hba_file')='/var/lib/postgresql/data/pg_hba.conf' and current_setting('ident_file')='/var/lib/postgresql/data/pg_ident.conf' and current_setting('ssl')='off' and not exists (select 1 from pg_file_settings where sourcefile not in ('/var/lib/postgresql/data/postgresql.conf','/var/lib/postgresql/data/postgresql.auto.conf'));") || return 1 + [[ "${settings}" == t ]] || pg18_error 'External PostgreSQL configuration/SSL files require a dedicated migration plan' || return 1 + [[ "$(pg18_sql "select (select count(*) from pg_subscription)=0 and (select count(*) from pg_prepared_xacts)=0 and not exists (select 1 from pg_replication_slots where slot_type='logical' or slot_name<>'jdmc_standby');")" == t ]] || + pg18_error 'Logical replication, custom replication slots or prepared transactions require a dedicated migration plan' || return 1 + if (( version >= 15 )); then + [[ "$(pg18_sql "select datlocprovider='c' from pg_database where datname='template0';")" == t ]] || + pg18_error 'A non-libc template cluster requires a dedicated locale migration plan' || return 1 + fi + # Reject symlinked WAL/configuration and user-defined storage outside PGDATA. + [[ -z "$(find "${PG18_DATA}" -type l -print -quit)" ]] || pg18_error 'Symlinked PostgreSQL storage is not supported by the automatic migration' || return 1 + [[ "$(pg18_sql "select count(*) from pg_database where datname='jms_pg18_migration';")" == 0 ]] || return 1 + bytes=$(du -sk "${PG18_DATA}" | awk '{print $1}') || return 1 + free=$(df -Pk "$(dirname "${PG18_DATA}")" | awk 'END {print $4}') || return 1 + [[ "${bytes}" =~ ^[0-9]+$ && "${free}" =~ ^[0-9]+$ ]] || return 1 + (( free > bytes * 3 + 1048576 )) || pg18_error 'Insufficient free space for the original cluster, logical backup and new cluster' || return 1 +} + +pg18_prepare() { + local phase locale + pg18_paths || return 1 + phase=$(cat "${PG18_WORK}/phase" 2>/dev/null || true) + case "${phase}" in ready|switching|switched|complete) return 0 ;; ''|reserved|preparing) ;; *) return 1 ;; esac + pg18_preflight || return 1 + # A previous crashed helper must never keep writing the staging directory. + pg18_remove_stale_helper || return 1 + pg18_save source-version "$(cat "${PG18_DATA}/PG_VERSION")" || return 1 + pg18_save source-image "$(docker inspect -f '{{.Config.Image}}' "${PG18_CONTAINER}")" || return 1 + pg18_save source-identity "$(stat -Lc '%d:%i' "${PG18_DATA}")" || return 1 + pg18_save target-image-id "$(docker image inspect -f '{{.Id}}' "${PG18_IMAGE}")" || return 1 + locale=$(pg18_sql "select pg_encoding_to_char(encoding)||E'\\t'||datcollate||E'\\t'||datctype from pg_database where datname='template0';") || return 1 + pg18_save locale "${locale}" || return 1 + pg18_save bootstrap-role "$(pg18_sql 'select rolname from pg_authid where oid=10;')" || return 1 + pg18_save bootstrap-quoted "$(pg18_sql 'select quote_ident(rolname) from pg_authid where oid=10;')" || return 1 + pg18_save phase preparing || return 1 + printf '[postgresql-upgrade] Exporting all databases and roles; original data remains at %s\n' "${PG18_DATA}" >&2 + if ! printf '%s\n' "${PG18_PASSWORD}" | docker run --rm -i --name "${PG18_HELPER}" --label "jms.pg18.work=${PG18_WORK}" --network "container:${PG18_CONTAINER}" \ + --entrypoint sh "${PG18_IMAGE}" -ceu ' + IFS= read -r PGPASSWORD; export PGPASSWORD PGCONNECT_TIMEOUT=10 + exec pg_dumpall -h 127.0.0.1 -p "$1" -U "$2" --clean --if-exists + ' sh "${PG18_PORT}" "${PG18_USER}" >"${PG18_WORK}/cluster.partial.sql"; then return 1; fi + [[ -s "${PG18_WORK}/cluster.partial.sql" ]] || return 1 + chmod 600 "${PG18_WORK}/cluster.partial.sql" && sync "${PG18_WORK}/cluster.partial.sql" && + mv -f "${PG18_WORK}/cluster.partial.sql" "${PG18_WORK}/cluster.sql" || return 1 + # Only the isolated, transaction-owned target is disposable on a retry. + [[ ! -L "${PG18_WORK}/target" ]] || return 1 + rm -rf -- "${PG18_WORK}/target" || return 1 + mkdir -m 700 "${PG18_WORK}/target" || return 1 + if ! docker run --rm -i --name "${PG18_HELPER}" --label "jms.pg18.work=${PG18_WORK}" --network none \ + -e PGDATA=/var/lib/postgresql/data -v "${PG18_WORK}:/migration" -v "${PG18_WORK}/target:/var/lib/postgresql/data" -v "${PG18_DATA}:/source:ro" --entrypoint bash "${PG18_IMAGE}" -s -- "${PG18_USER}" \ + >"${PG18_WORK}/restore.log" 2>&1 <<'PG18_RESTORE' +set -Eeuo pipefail +IFS=$'\t' read -r encoding collate ctype /dev/null 2>&1 || true' EXIT +gosu postgres createdb --template=template0 jms_pg18_migration +export PGDATABASE=jms_pg18_migration +# OID 10 owns PostgreSQL's system objects and cannot be dropped/reassigned. +# Retain that exact source identity; apply its final privileges only after restore. +awk -v role="$(cat /migration/bootstrap-quoted)" ' + $0 == "DROP ROLE " role ";" || $0 == "DROP ROLE IF EXISTS " role ";" || $0 == "CREATE ROLE " role ";" {next} + index($0,"ALTER ROLE " role " WITH ")==1 {print > "/migration/bootstrap-final.sql"; next} + {print} +' /migration/cluster.sql > /migration/restore.sql +gosu postgres psql -X -v ON_ERROR_STOP=1 < /migration/restore.sql +export PGDATABASE=postgres +PGUSER="$1" gosu postgres dropdb jms_pg18_migration +PGUSER="$1" gosu postgres vacuumdb --all --analyze-in-stages +gosu postgres psql -X -v ON_ERROR_STOP=1 < /migration/bootstrap-final.sql +gosu postgres pg_ctl -D /var/lib/postgresql/data -m fast -w stop +trap - EXIT +for file in postgresql.conf postgresql.auto.conf pg_hba.conf pg_ident.conf; do + [[ -f /source/$file && ! -L /source/$file ]] + cp -p "/source/$file" "/var/lib/postgresql/data/$file" +done +chown -R postgres:postgres /var/lib/postgresql/data +# PostgreSQL validates inherited settings before any production cutover. +gosu postgres postgres -D /var/lib/postgresql/data -C data_directory >/dev/null +printf 'local all all trust\n' >/tmp/migration-hba.conf +chmod 644 /tmp/migration-hba.conf +gosu postgres pg_ctl -D /var/lib/postgresql/data -l /var/lib/postgresql/data/validation-server.log -o "-c listen_addresses='' -c unix_socket_directories='/tmp' -c hba_file=/tmp/migration-hba.conf -c synchronous_standby_names=''" -w start +trap 'gosu postgres pg_ctl -D /var/lib/postgresql/data -m immediate -w stop >/dev/null 2>&1 || true' EXIT +PGUSER="$1" gosu postgres psql -X -v ON_ERROR_STOP=1 -c 'SELECT 1;' >/dev/null +gosu postgres pg_ctl -D /var/lib/postgresql/data -m fast -w stop +trap - EXIT +sync -f /migration +PG18_RESTORE + then pg18_error "Restore failed; original data is intact. Private log: ${PG18_WORK}/restore.log"; return 1; fi + [[ "$(cat "${PG18_WORK}/target/PG_VERSION")" == 18 ]] || return 1 + pg18_save phase ready +} + +pg18_cutover() { + local phase + pg18_paths || return 1 + phase=$(cat "${PG18_WORK}/phase") || return 1 + case "${phase}" in complete|switched) [[ "$(cat "${PG18_DATA}/PG_VERSION")" == 18 ]]; return $? ;; ready|switching) ;; *) return 1 ;; esac + [[ "$(docker image inspect -f '{{.Id}}' "${PG18_IMAGE}")" == "$(cat "${PG18_WORK}/target-image-id")" ]] || return 1 + docker stop "${PG18_CONTAINER}" >/dev/null || return 1 + [[ "$(docker inspect -f '{{.State.Running}}' "${PG18_CONTAINER}")" == false ]] || return 1 + # The 18 image also traverses its declared parent volume after dropping UID. + # Preserve existing read/write permissions; grant only directory traversal. + pg18_save parent-mode "$(stat -Lc '%a' "$(dirname "${PG18_DATA}")")" || return 1 + chmod a+x "$(dirname "${PG18_DATA}")" || return 1 + pg18_save phase switching || return 1 + if [[ ! -e "${PG18_WORK}/original" ]]; then + [[ "$(stat -Lc '%d:%i' "${PG18_DATA}")" == "$(cat "${PG18_WORK}/source-identity")" ]] || return 1 + sync -f "${PG18_DATA}" && mv -- "${PG18_DATA}" "${PG18_WORK}/original" && sync -f "${PG18_WORK}" || return 1 + fi + [[ ! -L "${PG18_WORK}/original" && "$(stat -Lc '%d:%i' "${PG18_WORK}/original")" == "$(cat "${PG18_WORK}/source-identity")" ]] || return 1 + if [[ ! -e "${PG18_DATA}" ]]; then mv -- "${PG18_WORK}/target" "${PG18_DATA}" && sync -f "${PG18_WORK}" || return 1; fi + [[ ! -L "${PG18_DATA}" && "$(cat "${PG18_DATA}/PG_VERSION")" == 18 ]] || return 1 + pg18_save phase switched +} diff --git a/scripts/gists/service.sh b/scripts/gists/service.sh index de0f89ee..30aff64f 100644 --- a/scripts/gists/service.sh +++ b/scripts/gists/service.sh @@ -1,3 +1,4 @@ +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/postgresql-upgrade-standalone.sh" common_services=(core kael celery ansible-executor koko chen web) xpack_services=(magnus razor xrdp video-worker nec) @@ -51,6 +52,98 @@ function get_db_images_file() { get_db_info "file" } +function get_internal_postgresql_data_version() { + local volume_dir version_file data_version + [[ "$(get_config DB_ENGINE)" == "postgresql" ]] || return 0 + [[ "$(get_config DB_HOST)" == "postgresql" || "$(get_config HA_MODE)" == "1" ]] || return 0 + volume_dir=$(get_config VOLUME_DIR) + [[ -n "${volume_dir}" ]] || return 0 + version_file="${volume_dir}/postgresql/data/PG_VERSION" + [[ -e "${version_file}" ]] || return 0 + data_version=$(cat "${version_file}") || return 1 + [[ "${data_version}" =~ ^[1-9][0-9]$ ]] || { + log_error "Cannot determine the existing PostgreSQL major version from ${version_file}" + return 1 + } + printf '%s\n' "${data_version}" +} + +function get_postgresql_image() { + local image data_version container_image candidate + image=postgres:18.6-bookworm + data_version=$(get_internal_postgresql_data_version) || return 1 + if [[ -n "${data_version}" && ! "${image}" =~ ^postgres:${data_version}([.-]|$) ]]; then + # Keep the installed patch/OS variant when the old container still exists. + container_image=$(docker inspect -f '{{.Config.Image}}' jms_postgresql 2>/dev/null || true) + if [[ ! "${container_image}" =~ ^postgres:${data_version}([.-]|$) ]]; then + # A stopped/removed container must not force an offline upgrade to fetch + # a different patch when a compatible local PostgreSQL image remains. + while IFS= read -r candidate; do + if [[ "${candidate}" =~ ^postgres:${data_version}([.-]|$) ]]; then + container_image=${candidate} + break + fi + done < <(docker image ls --filter "reference=postgres:${data_version}*" --format '{{.Repository}}:{{.Tag}}' 2>/dev/null || true) + fi + if [[ "${container_image}" =~ ^postgres:${data_version}([.-]|$) ]]; then + image=${container_image} + else + case "${data_version}" in + 16) image=postgres:16.15-bookworm ;; + 17) image=postgres:17.11-bookworm ;; + 18) image=postgres:18.6-bookworm ;; + *) image="postgres:${data_version}-bookworm" ;; + esac + fi + fi + printf '%s\n' "${image}" +} + +function get_postgresql_compose_files() { + local image directory file content tmp + image=$(get_postgresql_image) || return 1 + if [[ "${image}" == postgres:18.6-bookworm ]]; then + printf 'compose/postgresql.yml -f compose/postgresql.port.yml\n' + return 0 + fi + # Old data still needs its matching image until migration finishes. Keep the + # release Compose fixed at 18 and record this runtime override in its labels, + # so JDMC can render the same files without any image configuration variable. + [[ "${image}" =~ ^postgres:[A-Za-z0-9_][A-Za-z0-9_.-]*$ ]] || return 1 + directory="${CONFIG_DIR}/postgresql" + file="${directory}/image-${image#postgres:}.yml" + content=$(printf 'services:\n postgresql:\n image: %s\n' "${image}") + [[ ! -L "${directory}" && ! -L "${file}" ]] || return 1 + mkdir -p -m 700 "${directory}" || return 1 + # A captured JDMC Compose binding may still reference this file. Never + # rewrite or delete it when the selected database version changes. + if [[ -e "${file}" ]]; then + [[ -f "${file}" && "$(cat "${file}")" == "${content}" ]] || return 1 + else + tmp=$(mktemp "${directory}/.image.XXXXXX") || return 1 + if ! { printf '%s\n' "${content}" >"${tmp}" && chmod 600 "${tmp}" && mv "${tmp}" "${file}"; }; then + rm -f "${tmp}" + return 1 + fi + fi + printf 'compose/postgresql.yml -f compose/postgresql.port.yml -f %s\n' "${file}" +} + +function prepare_postgresql_parent_directory() { + # The shared Compose now binds PGDATA's parent even while using an old image. + # Allow the container's postgres UID to traverse it without opening PGDATA. + # Existing HA mounts and permissions are changed only by coordinated cutover. + [[ "$(get_config DB_HOST)" == postgresql && "$(get_config HA_MODE)" != 1 ]] || return 0 + local volume_dir parent mode + volume_dir=$(get_config VOLUME_DIR) + [[ -n "${volume_dir}" ]] || return 0 + parent="${volume_dir}/postgresql" + [[ -d "${parent}" ]] || return 0 + [[ ! -L "${parent}" ]] || return 1 + mode=$(stat -Lc '%a' -- "${parent}") || return 1 + (( (8#${mode} & 0111) == 0111 )) || chmod a+x -- "${parent}" +} + function get_db_info() { local db_image info_type=$1 @@ -86,6 +179,7 @@ function get_db_info() { ;; "postgresql") postgres_data_exists="1" + db_image=$(get_postgresql_image) || return 1 ;; esac @@ -96,7 +190,7 @@ function get_db_info() { elif [[ "${mariadb_data_exists}" == "1" ]]; then echo "mariadb:10.6" elif [[ "${postgres_data_exists}" == "1" ]]; then - echo "postgres:16.15-bookworm" + echo "${db_image}" fi ;; "file") @@ -105,7 +199,8 @@ function get_db_info() { elif [[ "${mariadb_data_exists}" == "1" ]]; then echo "compose/mariadb.yml -f compose/mysql.port.yml" elif [[ "${postgres_data_exists}" == "1" ]]; then - echo "compose/postgresql.yml -f compose/postgresql.port.yml" + prepare_postgresql_parent_directory || return 1 + get_postgresql_compose_files || return 1 fi ;; *) @@ -151,6 +246,7 @@ function get_docker_compose_services() { } function get_docker_compose_cmd_line() { + postgresql_upgrade_start_guard || return 1 ignore_db=${1-} use_ipv6=$(get_config USE_IPV6) https_port=$(get_config HTTPS_PORT) @@ -175,7 +271,7 @@ function get_docker_compose_cmd_line() { done if [[ "${ignore_db}" != "ignore_db" ]]; then - db_yml=$(get_db_compose_yml) + db_yml=$(get_db_compose_yml) || return 1 if [[ -n "${db_yml}" ]]; then cmd+=" ${db_yml}" fi @@ -319,7 +415,7 @@ function get_db_compose_yml() { db_host=$(get_config DB_HOST) redis_host=$(get_config REDIS_HOST) ha_mode=$(get_config HA_MODE) - db_images_file=$(get_db_info "file") + db_images_file=$(get_db_info "file") || return 1 redis_expose_port=$(get_config REDIS_EXPOSE_PORT) if [[ -z "${target}" ]]; then @@ -329,7 +425,7 @@ function get_db_compose_yml() { cmd="" if [[ "${target}" == "all" || "${target}" == "db" ]]; then if [[ "${db_host}" == "mysql" || "${db_host}" == "postgresql" || "${ha_mode}" == "1" ]]; then - db_images_file=$(get_db_images_file) + db_images_file=$(get_db_images_file) || return 1 cmd+=" -f ${db_images_file}" fi fi @@ -346,12 +442,13 @@ function get_db_compose_yml() { function get_db_compose_cmd() { + postgresql_upgrade_start_guard || return 1 target=${1:-"all"} use_ipv6=$(get_config USE_IPV6) use_xpack=$(get_config_or_env USE_XPACK) cmd="docker compose " - yml=$(get_db_compose_yml "${target}") + yml=$(get_db_compose_yml "${target}") || return 1 if [[ -n "${yml}" ]]; then cmd+=" ${yml}" @@ -366,13 +463,13 @@ function get_db_compose_cmd() { } function get_db_migrate_compose_cmd() { - cmd=$(get_db_compose_cmd "all") + cmd=$(get_db_compose_cmd "all") || return 1 cmd+=" -f compose/init-db.yml" echo "$cmd" } function create_db_ops_env() { - cmd=$(get_db_migrate_compose_cmd) + cmd=$(get_db_migrate_compose_cmd) || return 1 ${cmd} up -d || { return 1 } @@ -385,7 +482,7 @@ function down_db_ops_env() { function db_redis_start() { target=$1 - cmd=$(get_db_compose_cmd "${target}") + cmd=$(get_db_compose_cmd "${target}") || return 1 ${cmd} up -d || { return 1 } diff --git a/tests/test_external_mysql_image.sh b/tests/test_external_mysql_image.sh index 0595d77b..f668fbe5 100644 --- a/tests/test_external_mysql_image.sh +++ b/tests/test_external_mysql_image.sh @@ -182,10 +182,10 @@ assert_db_image_required mariadb:10.6 set_config DB_HOST postgresql assert_db_image_required mariadb:10.6 set_config DB_ENGINE postgresql -assert_db_image_required postgres:16.15-bookworm +assert_db_image_required postgres:18.6-bookworm set_config DB_HOST db.example.test set_config HA_MODE 1 -assert_db_image_required postgres:16.15-bookworm +assert_db_image_required postgres:18.6-bookworm set_config HA_MODE 0 printf 'PASS: internal MySQL/PostgreSQL databases, HA, and missing hosts retain required image checks during upgrade\n' @@ -197,16 +197,16 @@ unset SKIP_BACKUP_DB main >"${test_dir}/load.log" assert_contains "$(cat "${test_dir}/load.log")" 'WARNING: Docker image not found' assert_contains "$(cat "${test_dir}/load.log")" 'continuing to the backup step' -installed_db_image=postgres:16.15-bookworm +installed_db_image=postgres:18.6-bookworm main >"${test_dir}/load.log" if [[ "$(cat "${test_dir}/load.log")" == *WARNING* ]]; then fail 'an existing external PostgreSQL client must be reused without an archive' fi installed_db_image='' IMAGE_DIR="${test_dir}/no-offline-images" -fail_pull_image=postgres:16.15-bookworm +fail_pull_image=postgres:18.6-bookworm main >"${test_dir}/pull.log" 2>&1 -assert_contains "$(cat "${test_dir}/pull.log")" 'WARNING: Failed to pull Docker image: postgres:16.15-bookworm' +assert_contains "$(cat "${test_dir}/pull.log")" 'WARNING: Failed to pull Docker image: postgres:18.6-bookworm' export SKIP_BACKUP_DB=1 : >"${docker_calls}" main >"${test_dir}/pull.log" 2>&1 diff --git a/tests/test_image.sh b/tests/test_image.sh index a029b8b7..814cade1 100644 --- a/tests/test_image.sh +++ b/tests/test_image.sh @@ -13,7 +13,7 @@ cp "${TEST_ROOT}/config-example.txt" "${test_dir}/config.txt" IMAGE_DIR="${test_dir}/images" mkdir -p "${IMAGE_DIR}" set_config VOLUME_DIR "${test_dir}/volume" -assert_eq 'postgres:16.15-bookworm' "$(get_db_images)" 'new installations must default to PostgreSQL' +assert_eq 'postgres:18.6-bookworm' "$(get_db_images)" 'new installations must default to PostgreSQL' set_config DB_ENGINE mysql set_config DB_HOST mysql @@ -84,7 +84,7 @@ for test_image in mysql:8.0 mariadb:10.6; do done printf 'PASS: offline upgrades reuse local MySQL/MariaDB images and legacy tags\n' -for test_image in postgres:16.15-bookworm jumpserver/core:test; do +for test_image in postgres:18.6-bookworm jumpserver/core:test; do installed_image=${test_image} if load_image_files >"${test_dir}/load.log"; then fail 'non-legacy images must still require their packaged archives' diff --git a/tests/test_postgresql_migration_containers.sh b/tests/test_postgresql_migration_containers.sh new file mode 100644 index 00000000..84d8481c --- /dev/null +++ b/tests/test_postgresql_migration_containers.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# Run inside a Linux Bash/Docker CLI test runner with its Docker socket mounted. +# PG18_* inputs are consumed by the sourced migration library. +# shellcheck disable=SC2034 +set -Eeuo pipefail +umask 077 +docker() { + if [[ "${PG18_FAIL_RESTORE:-0}" == 1 && "$*" == *"--label jms.pg18.work="* && "$*" == *"--entrypoint bash"* ]]; then return 99; fi + command docker "$@" +} + +if [[ "${RUN_POSTGRESQL_MIGRATION_INTEGRATION:-0}" != 1 && "${PG18_TEST_CHILD:-0}" != 1 ]]; then echo "SKIP: set RUN_POSTGRESQL_MIGRATION_INTEGRATION=1 in a Linux Docker runner"; exit 0; fi +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +if [[ "${PG18_TEST_CHILD:-0}" != 1 ]]; then + volume="jms-pg18-test-$$-${RANDOM}" + docker volume create "$volume" >/dev/null + trap 'docker volume rm "$volume" >/dev/null 2>&1 || true' EXIT + directory=$(docker volume inspect -f '{{.Mountpoint}}' "$volume") + runner=$(docker inspect -f '{{.Image}}' "$HOSTNAME") + docker run --rm -i -e PG18_TEST_CHILD=1 -e "PG18_TEST_ROOT=$directory" -e "PG18_SOURCE_IMAGE=${PG18_SOURCE_IMAGE:-postgres:16.15-bookworm}" \ + -v /var/run/docker.sock:/var/run/docker.sock -v "$volume:$directory" \ + --entrypoint bash "$runner" -c 'mkdir -p "$PG18_TEST_ROOT/src"; tar -C "$PG18_TEST_ROOT/src" -xf -; exec bash "$PG18_TEST_ROOT/src/tests/test_postgresql_migration_containers.sh"' \ + < <(tar -C "$root" -cf - scripts/gists/postgresql-upgrade.sh tests/test_postgresql_migration_containers.sh) + exit +fi +source "$root/scripts/gists/postgresql-upgrade.sh" +PG18_DATA="$PG18_TEST_ROOT/data" +PG18_CONTAINER="jms-pg18-source-$$-${RANDOM}" +PG18_USER=old_admin PG18_PASSWORD=test-only PG18_PORT=5432 +cleanup() { local status=$?; if [[ "$status" != 0 && -f "${PG18_WORK:-}/restore.log" ]]; then tail -n 35 "$PG18_WORK/restore.log" >&2; fi; docker rm -fv "$PG18_CONTAINER" "${PG18_HELPER:-unused}" >/dev/null 2>&1 || true; } +trap cleanup EXIT +mkdir -m 700 "$PG18_DATA" +docker run -d --name "$PG18_CONTAINER" --network none -e POSTGRES_USER=old_admin -e POSTGRES_PASSWORD=test-only \ + -e PGDATA=/var/lib/postgresql/data -v "$PG18_DATA:/var/lib/postgresql/data" "${PG18_SOURCE_IMAGE:-postgres:16.15-bookworm}" >/dev/null +ready=false +for _ in $(seq 1 90); do if pg18_sql 'select 1' >/dev/null 2>&1; then ready=true; break; fi; sleep 1; done +[[ "$ready" == true ]] +source_major=$(cat "$PG18_DATA/PG_VERSION") +pg18_sql "CREATE ROLE application LOGIN PASSWORD 'test-application'; CREATE TABLE public.survivor(id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, value text); INSERT INTO survivor(value) VALUES ('中文迁移'),('second'); GRANT SELECT ON survivor TO application; CREATE EXTENSION pgcrypto;" >/dev/null +pg18_sql 'CREATE DATABASE extra;' >/dev/null +before_hash=$(pg18_sql "select rolpassword from pg_authid where rolname='application';") +# Inject a helper failure after a real dump, then retry without touching source data. +PG18_FAIL_RESTORE=1 +if pg18_prepare; then echo 'FAIL failed restore was accepted'; exit 1; fi +[[ "$(cat "$PG18_DATA/PG_VERSION")" == "$source_major" && "$(pg18_sql 'select count(*) from survivor;')" == 2 ]] +PG18_FAIL_RESTORE=0 +pg18_prepare +[[ + "$(cat "$PG18_WORK/phase")" == ready && "$(cat "$PG18_DATA/PG_VERSION")" == "$source_major" ]] +pg18_prepare # Idempotent staging must not replace its verified snapshot. +# Emulate a crash between the two directory renames during cutover. +docker stop "$PG18_CONTAINER" >/dev/null +pg18_save phase switching +mv "$PG18_DATA" "$PG18_WORK/original" +pg18_cutover +pg18_cutover +[[ "$(cat "$PG18_DATA/PG_VERSION")" == 18 && "$(cat "$PG18_WORK/original/PG_VERSION")" == "$source_major" ]] +docker rm -v "$PG18_CONTAINER" >/dev/null +docker run -d --name "$PG18_CONTAINER" --network none -e PGDATA=/var/lib/postgresql/data \ + -v "$PG18_DATA:/var/lib/postgresql/data" "$PG18_IMAGE" >/dev/null +ready=false +for _ in $(seq 1 90); do if pg18_sql 'select 1' >/dev/null 2>&1; then ready=true; break; fi; sleep 1; done +[[ "$ready" == true ]] +[[ "$(pg18_sql 'select count(*) from survivor;')" == 2 ]] +[[ "$(pg18_sql 'select value from survivor where id=1;')" == 中文迁移 ]] +[[ "$(pg18_sql "insert into survivor(value) values ('after') returning id;")" == 3 ]] +[[ "$(pg18_sql "select rolpassword from pg_authid where rolname='application';")" == "$before_hash" ]] +[[ "$(pg18_sql "select count(*) from pg_database where datname='extra';")" == 1 ]] +[[ "$(pg18_sql "select count(*) from pg_extension where extname='pgcrypto';")" == 1 ]] +[[ "$(pg18_sql "select has_table_privilege('application','survivor','SELECT');")" == t ]] +[[ "$(pg18_sql "select count(*) from pg_roles where rolname='jms_pg18_migration';")" == 0 ]] +printf 'PASS: real PostgreSQL -> 18 preserves all databases, roles/passwords, grants, extension, identity sequence and Unicode; interrupted cutover resumes with original cluster retained\n' diff --git a/tests/test_upgrade_backup.sh b/tests/test_upgrade_backup.sh index 99d48bc5..65939dd7 100644 --- a/tests/test_upgrade_backup.sh +++ b/tests/test_upgrade_backup.sh @@ -66,7 +66,7 @@ assert_backup_choices() { fi } -for selected_db_image in mariadb:10.6 mysql:8.0 postgres:16.15-bookworm; do +for selected_db_image in mariadb:10.6 mysql:8.0 postgres:18.6-bookworm; do select_database "${selected_db_image}" installed_db_image='' assert_backup_choices '' @@ -103,7 +103,7 @@ unset SKIP_BACKUP_DB printf 'PASS: legacy MySQL configurations and explicit backup skipping remain supported\n' # The same confirmation applies to built-in databases and HA backup failures. -for selected_db_image in mariadb:10.6 postgres:16.15-bookworm; do +for selected_db_image in mariadb:10.6 postgres:18.6-bookworm; do select_database "${selected_db_image}" for deployment in internal ha; do set_config HA_MODE 0 @@ -154,7 +154,7 @@ persist_installer_version() { :; } installation_log() { printf 'success\n' >>"${BACKUP_EVENTS}"; } clean_images() { :; } -for selected_db_image in mariadb:10.6 postgres:16.15-bookworm; do +for selected_db_image in mariadb:10.6 postgres:18.6-bookworm; do select_database "${selected_db_image}" for installed_db_image in '' "${selected_db_image}"; do for response in y n; do @@ -217,7 +217,7 @@ assert_contains "$(cat "${test_dir}/backup.log")" 'default: y): y' printf 'mock archive\n' >"${UPGRADE_TEST_IMAGES}/jdmc:${VERSION}.zst" printf 'sha256:installed\n' >"${UPGRADE_TEST_IMAGES}/jdmc:${VERSION}.sha256" - for selected_db_image in mariadb:10.6 postgres:16.15-bookworm; do + for selected_db_image in mariadb:10.6 postgres:18.6-bookworm; do select_database "${selected_db_image}" for installed_db_image in '' "${selected_db_image}"; do set_config CURRENT_VERSION v3.10.21