fix: stabilize external-db deployment and updates
Release / release (push) Canceled after 0s

This commit is contained in:
Qiufeng
2026-08-18 22:51:35 +08:00
parent 227c4b0129
commit 8c6c10eb85
22 changed files with 767 additions and 279 deletions
+181 -40
View File
@@ -30,6 +30,7 @@ HEALTH_URL=${KAIDI_HEALTH_URL:-http://127.0.0.1:18080/actuator/health}
APP_INDEX_URL=${KAIDI_APP_INDEX_URL:-http://127.0.0.1:18080/}
LOCK_FILE=$STATE_ROOT/update.lock
BACKUP_ROOT=${KAIDI_BACKUP_ROOT:-$STATE_ROOT/backups}
DB_BACKUP_MODE=${KAIDI_DB_BACKUP_MODE:-skip}
UPDATER_PATH=${KAIDI_UPDATER_PATH:-$APP_ROOT/bin/update.sh}
SYSTEMD_ROOT=${KAIDI_SYSTEMD_ROOT:-/etc/systemd/system}
LOG_ROOT=${KAIDI_LOG_ROOT:-/var/log/kaidi}
@@ -45,33 +46,167 @@ DOWNLOAD_PID=
DOWNLOAD_PGID=
LAST_DOWNLOAD_SPEED=0
SURFACE_FAILURE=
LOG_LOCALE=${KAIDI_LOG_LOCALE:-zh-CN}
localize_message() {
message=$1
[ "$LOG_LOCALE" = en ] && { printf '%s' "$message"; return; }
# A rollback failure must keep the manual-recovery signal visible even when
# the original reason also contains a more specific error prefix.
case "$message" in
*"rollback is incomplete and manual recovery is required"*)
printf '新版本应用验证失败,回滚未完成,需要人工恢复'
return
;;
esac
case "$message" in
"Update health-check settings must be non-negative integers") printf '更新健康检查参数必须是非负整数' ;;
"Update health-check attempts must be at least 1") printf '更新健康检查次数至少为 1' ;;
"KAIDI_PROCESS_MANAGER must be systemd or baota") printf 'KAIDI_PROCESS_MANAGER 只能是 systemd 或 baota' ;;
"KAIDI_UPDATE_SHUTDOWN_ATTEMPTS must be a positive integer") printf 'KAIDI_UPDATE_SHUTDOWN_ATTEMPTS 必须是正整数' ;;
"KAIDI_UPDATE_SHUTDOWN_ATTEMPTS must be at least 1") printf 'KAIDI_UPDATE_SHUTDOWN_ATTEMPTS 至少为 1' ;;
"KAIDI_PID_FILE must be an absolute path") printf 'KAIDI_PID_FILE 必须是绝对路径' ;;
"KAIDI_RUNTIME_ENV_FILE must be an absolute path") printf 'KAIDI_RUNTIME_ENV_FILE 必须是绝对路径' ;;
"KAIDI_DB_BACKUP_MODE must be skip or mysqldump") printf 'KAIDI_DB_BACKUP_MODE 只能是 skip 或 mysqldump' ;;
"Setup runtime environment must be a regular file") printf '安装向导运行时配置必须是普通文件' ;;
"Setup runtime environment is not readable") printf '安装向导运行时配置不可读' ;;
"Setup runtime environment is too large") printf '安装向导运行时配置过大' ;;
"Setup runtime environment has an unexpected owner") printf '安装向导运行时配置属主不正确' ;;
"Setup runtime environment must not be writable"*) printf '安装向导运行时配置不能被组或其他用户写入' ;;
"Setup runtime environment contains an invalid line") printf '安装向导运行时配置包含无效行' ;;
"Setup runtime environment "*) printf '安装向导运行时配置错误,请检查 application.env' ;;
"Setup runtime database port is invalid") printf '安装向导数据库端口无效' ;;
"Database configuration is invalid: DB_URL is missing"*) printf '数据库配置无效:缺少 DB_URL,未重启应用' ;;
"Database configuration is invalid: database host and name are missing"*) printf '数据库配置无效:缺少数据库主机或库名,未重启应用' ;;
"Database configuration is invalid: DB_URL does not match"*) printf '数据库配置无效:DB_URL 与主机、端口、库名不一致,未重启应用' ;;
"UPDATE_RELEASE_TOKEN is invalid") printf 'UPDATE_RELEASE_TOKEN 无效' ;;
"UPDATE_RELEASE_BASE_URL or UPDATE_RELEASE_API_URL is not configured") printf '未配置更新源地址' ;;
"Service user "*" is missing") printf '缺少服务用户:%s' "$(printf '%s' "${message#Service user }" | sed 's/ is missing$//')" ;;
"Service user "*" is not a member"*) printf '服务用户不属于指定服务组' ;;
"runuser is required") printf '需要 runuser' ;;
"setsid is required") printf '需要 setsid' ;;
"Managed application or update directories could not be prepared") printf '应用或更新目录准备失败' ;;
"Writable update directories could not be prepared") printf '可写更新目录准备失败' ;;
"Private update directories could not be prepared") printf '私有更新目录准备失败' ;;
"Private update queue directories could not be prepared") printf '私有更新队列目录准备失败' ;;
"Downloading signed release "*) printf '正在下载已签名版本:%s' "${message#Downloading signed release }" ;;
"Release "*" download completed") printf '版本下载完成:%s' "${message#Release }" ;;
"Validating signed release "*) printf '正在校验签名版本:%s' "${message#Validating signed release }" ;;
"Revalidating cached release "*) printf '正在重新校验已缓存版本:%s' "${message#Revalidating cached release }" ;;
"Release "*" is downloaded and verified; confirm installation") printf '版本已下载并校验,请确认安装:%s' "${message#Release }" ;;
"Release "*" is running") printf '版本运行中:%s' "${message#Release }" ;;
*"automatic update watcher could not be started"*) printf '自动更新监听器未能启动' ;;
"Version "*" is already installed") printf '版本已安装:%s' "${message#Version }" ;;
"Starting and verifying release "*) printf '正在启动并验证版本:%s' "${message#Starting and verifying release }" ;;
"Validating current release before switching to "*) printf '切换前正在验证当前版本(目标:%s)' "${message#Validating current release before switching to }" ;;
"Backing up database before installing "*) printf '安装前正在备份数据库(目标:%s)' "${message#Backing up database before installing }" ;;
"Database backup skipped; updater does not access MySQL") printf '已跳过数据库备份;更新器不会访问 MySQL' ;;
"Installing release "*) printf '正在安装版本:%s' "${message#Installing release }" ;;
"Update process was interrupted"*) printf '更新进程被中断,系统将执行自动恢复' ;;
"Update transaction evidence could not be quarantined"*) printf '更新事务证据无法隔离,已停止自动更新' ;;
"Update request could not be archived"*) printf '更新请求无法归档,已停止自动更新' ;;
"Update service is locked for manual recovery"*) printf '更新服务已锁定,需要人工恢复;新请求已拒绝' ;;
"Processing update request must be a regular file") printf '处理中更新请求必须是普通文件' ;;
"Update request must be a regular file") printf '更新请求必须是普通文件' ;;
"Claimed update request must be a regular file") printf '已领取的更新请求必须是普通文件' ;;
"Verified release cache is missing") printf '缺少已校验的发布缓存' ;;
"Cached release manifest is missing") printf '缺少缓存的发布清单' ;;
"Cached release signature is missing") printf '缺少缓存的发布签名' ;;
"Cached release artifact is missing") printf '缺少缓存的发布包' ;;
"Update request version is invalid") printf '更新请求版本无效' ;;
"Update request action is invalid") printf '更新请求动作无效' ;;
"Release manifest "*" is missing") printf '缺少发布清单资产' ;;
"Release manifest download failed") printf '发布清单下载失败' ;;
"Release manifest signature download failed") printf '发布清单签名下载失败' ;;
"Gitea latest Release lookup failed") printf 'Gitea 最新 Release 查询失败' ;;
"Release manifest signature verification failed") printf '发布清单签名校验失败' ;;
"Requested version is no longer the latest signed release") printf '请求版本已不是最新签名版本,请重新获取版本' ;;
"Release artifact download failed") printf '发布包下载失败' ;;
"Release artifact asset is missing") printf '缺少发布包资产' ;;
"Release artifact SHA-256 verification failed") printf '发布包 SHA-256 校验失败' ;;
"Release artifact size verification failed") printf '发布包大小校验失败' ;;
"Release archive contains an unsafe path") printf '发布归档包含不安全路径,已拒绝' ;;
"Release archive must not contain symbolic links") printf '发布归档不能包含符号链接,已拒绝' ;;
"Release application restart failed") printf '新版本应用重启失败' ;;
"Release application verification failed"*) printf '新版本应用验证失败:%s' "${message#Release application verification failed }" ;;
"Current release preflight failed"*) printf '当前版本预检查失败,未重启应用:%s' "${message#Current release preflight failed }" ;;
"Current release path is invalid"*) printf '当前版本路径无效,未重启应用' ;;
"Service user cannot access the release or selected Java runtime") printf '服务用户无法访问发布目录或 Java 运行时' ;;
"Release operations files could not be backed up") printf '旧版本运维文件备份失败' ;;
"Existing operations files could not be backed up") printf '已有运维文件备份失败' ;;
"Release directory is already active") printf '目标版本目录已处于 active 状态' ;;
"Failed release staging directory could not be cleaned") printf '失败版本暂存目录清理失败' ;;
"Release directory could not be activated") printf '目标版本目录无法启用' ;;
"Release ownership or permissions could not be secured") printf '目标版本目录权限保护失败' ;;
"Release operations validation failed") printf '发布运维文件校验失败' ;;
"Release public key is missing") printf '发布公钥缺失' ;;
"Release verification public key is missing") printf '缺少发布校验公钥' ;;
"Release manifest version is invalid") printf '发布清单版本无效' ;;
"Release artifact name is invalid") printf '发布包文件名无效' ;;
"Release SHA-256 is invalid") printf '发布包 SHA-256 无效' ;;
"Release artifact size is invalid") printf '发布包大小无效' ;;
"Release archive could not be listed") printf '发布归档无法读取' ;;
"Release archive could not be extracted") printf '发布归档无法解压' ;;
"Release app.jar is missing") printf '发布包缺少 app.jar' ;;
"Release frontend is missing") printf '发布包缺少前端入口' ;;
"Release version file mismatch") printf '发布包 VERSION 与目标版本不一致' ;;
"Release updater is missing") printf '发布包缺少更新脚本' ;;
"Release Baota launcher is missing") printf '发布包缺少宝塔启动脚本' ;;
"Release Baota initializer is missing") printf '发布包缺少宝塔初始化脚本' ;;
"Release application unit is missing") printf '发布包缺少应用 systemd 单元' ;;
"Release updater unit is missing") printf '发布包缺少更新 systemd 单元' ;;
"Release updater path unit is missing") printf '发布包缺少更新监听单元' ;;
"Verified release cache path is unsafe") printf '已校验发布缓存路径不安全' ;;
"Verified release cache could not be activated") printf '已校验发布缓存无法启用' ;;
"Database backup failed") printf '数据库备份失败' ;;
"Database backup is empty") printf '数据库备份为空' ;;
"Database backup compression failed") printf '数据库备份压缩失败' ;;
"mysqldump is required"*) printf '更新前需要 mysqldump;请配置 KAIDI_MYSQLDUMP_BIN 或安装客户端' ;;
"Release operations files could not be activated") printf '发布运维文件无法切换' ;;
"Release application link could not be activated") printf '应用 current 链接无法切换' ;;
*"previous application release was restored and verified; database backup was retained"*) printf '已恢复并验证旧版本;数据库备份已保留' ;;
*"previous application release was restored and verified; no database backup was created"*) printf '已恢复并验证旧版本;本次未创建数据库备份' ;;
*"previous application release is running"*) printf '旧版本正在运行,但运维文件需要人工检查' ;;
*) printf '%s' "$message" ;;
esac
}
case "$HEALTH_ATTEMPTS:$HEALTH_INTERVAL_SECONDS" in
*[!0-9:]* | :* | *:) printf '%s\n' "Update health-check settings must be non-negative integers" >&2; exit 1 ;;
*[!0-9:]* | :* | *:) printf '%s\n' "$(localize_message 'Update health-check settings must be non-negative integers')" >&2; exit 1 ;;
esac
[ "$HEALTH_ATTEMPTS" -ge 1 ] || { printf '%s\n' "Update health-check attempts must be at least 1" >&2; exit 1; }
[ "$HEALTH_ATTEMPTS" -ge 1 ] || { printf '%s\n' "$(localize_message 'Update health-check attempts must be at least 1')" >&2; exit 1; }
case "$PROCESS_MANAGER" in
systemd|baota) ;;
*) printf '%s\n' "KAIDI_PROCESS_MANAGER must be systemd or baota" >&2; exit 1 ;;
*) printf '%s\n' "$(localize_message 'KAIDI_PROCESS_MANAGER must be systemd or baota')" >&2; exit 1 ;;
esac
case "$SHUTDOWN_ATTEMPTS" in
''|*[!0-9]*) printf '%s\n' "KAIDI_UPDATE_SHUTDOWN_ATTEMPTS must be a positive integer" >&2; exit 1 ;;
''|*[!0-9]*) printf '%s\n' "$(localize_message 'KAIDI_UPDATE_SHUTDOWN_ATTEMPTS must be a positive integer')" >&2; exit 1 ;;
esac
[ "$SHUTDOWN_ATTEMPTS" -ge 1 ] || { printf '%s\n' "KAIDI_UPDATE_SHUTDOWN_ATTEMPTS must be at least 1" >&2; exit 1; }
[ "$SHUTDOWN_ATTEMPTS" -ge 1 ] || { printf '%s\n' "$(localize_message 'KAIDI_UPDATE_SHUTDOWN_ATTEMPTS must be at least 1')" >&2; exit 1; }
case "$PID_FILE" in
/*) ;;
*) printf '%s\n' "KAIDI_PID_FILE must be an absolute path" >&2; exit 1 ;;
*) printf '%s\n' "$(localize_message 'KAIDI_PID_FILE must be an absolute path')" >&2; exit 1 ;;
esac
case "$RUNTIME_ENV_FILE" in
/*) ;;
*) printf '%s\n' "KAIDI_RUNTIME_ENV_FILE must be an absolute path" >&2; exit 1 ;;
*) printf '%s\n' "$(localize_message 'KAIDI_RUNTIME_ENV_FILE must be an absolute path')" >&2; exit 1 ;;
esac
case "${KAIDI_SKIP_DB_BACKUP:-}" in
true) DB_BACKUP_MODE=skip ;;
false)
[ -n "${KAIDI_DB_BACKUP_MODE:-}" ] || DB_BACKUP_MODE=mysqldump
;;
esac
case "$DB_BACKUP_MODE" in
skip|mysqldump) ;;
*) printf '%s\n' "$(localize_message 'KAIDI_DB_BACKUP_MODE must be skip or mysqldump')" >&2; exit 1 ;;
esac
bootstrap_die() {
if [ -f "$PROCESSING_FILE" ] && [ ! -L "$PROCESSING_FILE" ]; then
fail "$1"
fi
printf '%s\n' "$1" >&2
printf '%s\n' "$(localize_message "$1")" >&2
exit 1
}
@@ -149,7 +284,7 @@ validate_runtime_database_config() {
status() {
state=$1
message=$2
message=$(localize_message "$2")
version=${3:-}
downloaded_bytes=${4:-}
total_bytes=${5:-}
@@ -227,9 +362,10 @@ complete_request() {
fail() {
TERMINAL_STATUS_WRITTEN=true
status FAILED "$1" "${TARGET_VERSION:-}"
message=$(localize_message "$1")
status FAILED "$message" "${TARGET_VERSION:-}"
archive_processing_request || true
printf '%s\n' "$1" >&2
printf '%s\n' "$message" >&2
exit 1
}
@@ -601,7 +737,10 @@ mysqldump_bin() {
backup_database() {
DATABASE_BACKUP=
[ "${KAIDI_SKIP_DB_BACKUP:-false}" = "true" ] && return
if [ "$DB_BACKUP_MODE" = skip ]; then
event INFO BACKING_UP '数据库备份已跳过;更新器不会访问 MySQL'
return 0
fi
mkdir -p "$BACKUP_ROOT"
chmod 0700 "$BACKUP_ROOT"
dump_file=$(mktemp "$BACKUP_ROOT/.mysql-dump.XXXXXX.sql")
@@ -609,26 +748,16 @@ backup_database() {
backup_tmp="$backup.tmp.$$"
chmod 0600 "$dump_file"
if [ -n "${KAIDI_DB_CONTAINER:-}" ] && command -v docker >/dev/null 2>&1 \
&& docker inspect "$KAIDI_DB_CONTAINER" >/dev/null 2>&1; then
if ! docker exec -e MYSQL_PWD="${KAIDI_DB_PASSWORD:-}" "$KAIDI_DB_CONTAINER" \
mysqldump --single-transaction --routines --triggers \
-u "${KAIDI_DB_USERNAME:-kaidi}" "${KAIDI_DB_NAME:-kaidi_finance}" > "$dump_file"; then
rm -f "$dump_file" "$backup_tmp"
fail "Database backup failed"
fi
else
dump_bin=$(mysqldump_bin || true)
[ -n "$dump_bin" ] || {
rm -f "$dump_file" "$backup_tmp"
fail "mysqldump is required before installing an update; set KAIDI_MYSQLDUMP_BIN or install it in a standard MySQL bin directory"
}
if ! MYSQL_PWD=${KAIDI_DB_PASSWORD:-} "$dump_bin" --single-transaction --routines --triggers \
-h "${KAIDI_DB_HOST:-127.0.0.1}" -P "${KAIDI_DB_PORT:-3306}" \
-u "${KAIDI_DB_USERNAME:-kaidi}" "${KAIDI_DB_NAME:-kaidi_finance}" > "$dump_file"; then
rm -f "$dump_file" "$backup_tmp"
fail "Database backup failed"
fi
dump_bin=$(mysqldump_bin || true)
[ -n "$dump_bin" ] || {
rm -f "$dump_file" "$backup_tmp"
fail "mysqldump is required before installing an update; set KAIDI_MYSQLDUMP_BIN or install it in a standard MySQL bin directory"
}
if ! MYSQL_PWD=${KAIDI_DB_PASSWORD:-} "$dump_bin" --single-transaction --routines --triggers \
-h "${KAIDI_DB_HOST:-127.0.0.1}" -P "${KAIDI_DB_PORT:-3306}" \
-u "${KAIDI_DB_USERNAME:-kaidi}" "${KAIDI_DB_NAME:-kaidi_finance}" > "$dump_file"; then
rm -f "$dump_file" "$backup_tmp"
fail "Database backup failed"
fi
[ -s "$dump_file" ] || {
@@ -643,8 +772,12 @@ backup_database() {
mv -f "$backup_tmp" "$backup"
DATABASE_BACKUP=$backup
rm -f "$dump_file"
find "$BACKUP_ROOT" -type f -name 'mysql-*.sql.gz' -printf '%T@ %p\n' \
| sort -nr | awk 'NR > 5 {sub(/^[^ ]+ /, ""); print}' | xargs -r rm -f
# Keep the newest five backups without relying on GNU find's -printf; the
# updater is also exercised on BSD/macOS fixtures during release checks.
# shellcheck disable=SC2012 # Generated backup names contain no whitespace.
LC_ALL=C ls -1t "$BACKUP_ROOT"/mysql-*.sql.gz 2>/dev/null \
| awk 'NR > 5' \
| while IFS= read -r old_backup; do rm -f -- "$old_backup"; done
}
atomic_install() {
@@ -796,26 +929,30 @@ rollback_active_transaction() {
&& verify_app_surface "$previous_target"; then
remove_failed_release "$failed_release"
if [ "$operations_restored" = true ]; then
database_backup_record=$(transaction_value database-backup)
rm -rf "$ACTIVE_TRANSACTION"
fail "$reason; previous application release was restored and verified; database backup was retained"
if [ -n "$database_backup_record" ] || [ -n "${DATABASE_BACKUP:-}" ]; then
fail "$reason; previous application release was restored and verified; database backup was retained"
fi
fail "$reason; previous application release was restored and verified; no database backup was created"
fi
fail "$reason; previous application release is running, but operations restoration requires manual review; transaction evidence was retained"
fi
TERMINAL_STATUS_WRITTEN=true
recovery_guard_failed=false
if ! archive_processing_request; then
printf '%s\n' "Update request could not be archived after an incomplete rollback" >&2
printf '%s\n' "$(localize_message 'Update request could not be archived after an incomplete rollback')" >&2
recovery_guard_failed=true
fi
if ! quarantine_active_transaction; then
printf '%s\n' "Update transaction evidence could not be quarantined after an incomplete rollback" >&2
printf '%s\n' "$(localize_message 'Update transaction evidence could not be quarantined after an incomplete rollback')" >&2
recovery_guard_failed=true
fi
if [ "$recovery_guard_failed" = true ]; then
systemctl stop "$UPDATE_PATH_NAME" >/dev/null 2>&1 || true
fi
status RECOVERY_REQUIRED "$reason; rollback is incomplete and manual recovery is required" "${TARGET_VERSION:-}"
printf '%s\n' "$reason; rollback is incomplete and manual recovery is required" >&2
printf '%s\n' "$(localize_message "$reason; rollback is incomplete and manual recovery is required")" >&2
exit 1
}
@@ -885,7 +1022,7 @@ if [ -f "$STATUS_FILE" ] && [ ! -L "$STATUS_FILE" ] \
if ! archive_processing_request; then
systemctl stop "$UPDATE_PATH_NAME" >/dev/null 2>&1 || true
fi
printf '%s\n' "Update service is locked for manual recovery; new requests are rejected" >&2
printf '%s\n' "$(localize_message 'Update service is locked for manual recovery; new requests are rejected')" >&2
exit 1
fi
@@ -1053,7 +1190,11 @@ write_transaction_value previous-target "$PREVIOUS_TARGET"
write_transaction_value release-dir "$RELEASE_DIR"
write_transaction_value phase PREPARED
status BACKING_UP "Backing up database before installing $TARGET_VERSION" "$TARGET_VERSION"
if [ "$DB_BACKUP_MODE" = mysqldump ]; then
status BACKING_UP "Backing up database before installing $TARGET_VERSION" "$TARGET_VERSION"
else
status BACKING_UP "Database backup skipped; updater does not access MySQL" "$TARGET_VERSION"
fi
backup_database
[ -z "${DATABASE_BACKUP:-}" ] || write_transaction_value database-backup "$DATABASE_BACKUP"
if ! backup_operations; then