From cea46b494d2eafa0dbbede30ff90e818c8e3675c Mon Sep 17 00:00:00 2001 From: Qiufeng Date: Mon, 17 Aug 2026 23:36:37 +0800 Subject: [PATCH] fix: keep online update recovery available --- deploy/update.sh | 69 ++++++++++++++++++++++++---------- scripts/test-update-fixture.sh | 42 ++++++++++++++++++++- 2 files changed, 90 insertions(+), 21 deletions(-) diff --git a/deploy/update.sh b/deploy/update.sh index 8dcec5a..485636b 100755 --- a/deploy/update.sh +++ b/deploy/update.sh @@ -18,6 +18,7 @@ RELEASE_API_URL=${UPDATE_RELEASE_API_URL:-} RELEASE_TOKEN=${UPDATE_RELEASE_TOKEN:-} CACHE_ROOT=${KAIDI_UPDATE_CACHE_ROOT:-$STATE_ROOT/cache} SERVICE_NAME=${KAIDI_SERVICE_NAME:-kaidi-finance.service} +UPDATE_PATH_NAME=${KAIDI_UPDATE_PATH_NAME:-kaidi-update.path} SERVICE_USER=${KAIDI_SERVICE_USER:-kaidi} SERVICE_GROUP=${KAIDI_SERVICE_GROUP:-kaidi} HEALTH_URL=${KAIDI_HEALTH_URL:-http://127.0.0.1:18080/actuator/health} @@ -272,6 +273,22 @@ atomic_install() { mv -f "$install_temporary" "$install_destination" } +install_operation() { + operation_source=$1 + operation_destination=$2 + operation_mode=$3 + operation_is_systemd=$4 + if [ -f "$operation_destination" ] && [ ! -L "$operation_destination" ] \ + && cmp -s "$operation_source" "$operation_destination"; then + chmod "$operation_mode" "$operation_destination" + return + fi + atomic_install "$operation_source" "$operation_destination" "$operation_mode" || return 1 + if [ "$operation_is_systemd" = true ]; then + SYSTEMD_OPERATIONS_CHANGED=true + fi +} + backup_managed_file() { backup_source=$1 backup_name=$2 @@ -311,14 +328,16 @@ backup_operations() { } apply_operations() { - atomic_install "$RELEASE_DIR/ops/update.sh" "$UPDATER_PATH" 0755 \ - && atomic_install "$RELEASE_DIR/ops/kaidi-finance.service" \ - "$SYSTEMD_ROOT/kaidi-finance.service" 0644 \ - && atomic_install "$RELEASE_DIR/ops/kaidi-update.service" \ - "$SYSTEMD_ROOT/kaidi-update.service" 0644 \ - && atomic_install "$RELEASE_DIR/ops/kaidi-update.path" \ - "$SYSTEMD_ROOT/kaidi-update.path" 0644 \ - && systemctl daemon-reload + SYSTEMD_OPERATIONS_CHANGED=false + install_operation "$RELEASE_DIR/ops/update.sh" "$UPDATER_PATH" 0755 false \ + && install_operation "$RELEASE_DIR/ops/kaidi-finance.service" \ + "$SYSTEMD_ROOT/kaidi-finance.service" 0644 true \ + && install_operation "$RELEASE_DIR/ops/kaidi-update.service" \ + "$SYSTEMD_ROOT/kaidi-update.service" 0644 true \ + && install_operation "$RELEASE_DIR/ops/kaidi-update.path" \ + "$SYSTEMD_ROOT/kaidi-update.path" 0644 true \ + || return 1 + [ "$SYSTEMD_OPERATIONS_CHANGED" = false ] || systemctl daemon-reload } restore_operations() { @@ -357,24 +376,28 @@ rollback_active_transaction() { previous_target=$(transaction_value previous-target) failed_release=$(transaction_value release-dir) OPS_BACKUP="$ACTIVE_TRANSACTION/operations-backup" - rollback_ok=true + link_restored=true + operations_restored=true systemctl stop "$SERVICE_NAME" || true if [ -n "$previous_target" ] && [ -d "$previous_target" ]; then if ! ln -sfn "$previous_target" "$APP_ROOT/current.next" \ || ! mv -Tf "$APP_ROOT/current.next" "$APP_ROOT/current"; then - rollback_ok=false + link_restored=false fi else - rollback_ok=false + link_restored=false fi - restore_operations || rollback_ok=false + restore_operations || operations_restored=false systemctl reset-failed "$SERVICE_NAME" >/dev/null 2>&1 || true - systemctl start "$SERVICE_NAME" || rollback_ok=false - if [ "$rollback_ok" = true ] && verify_app_surface; then + systemctl start "$SERVICE_NAME" || true + if [ "$link_restored" = true ] && verify_app_surface; then remove_failed_release "$failed_release" rm -rf "$ACTIVE_TRANSACTION" - fail "$reason; previous release was restored and verified" + if [ "$operations_restored" = true ]; then + fail "$reason; previous release was restored and verified" + fi + fail "$reason; previous release is running, but operations restoration requires manual review" fi TERMINAL_STATUS_WRITTEN=true status FAILED "$reason; rollback is incomplete and will be retried" "${TARGET_VERSION:-}" @@ -600,15 +623,21 @@ write_transaction_value phase APP_SWITCHED status RUNNING "Starting and verifying release $TARGET_VERSION" "$TARGET_VERSION" write_transaction_value phase HEALTH_CHECKING systemctl reset-failed "$SERVICE_NAME" >/dev/null 2>&1 || true -if systemctl start "$SERVICE_NAME" \ - && systemctl is-active --quiet kaidi-update.path \ - && verify_app_surface; then +systemctl start "$SERVICE_NAME" || true +if verify_app_surface; then + success_message="Release $TARGET_VERSION is running" + if ! systemctl start "$UPDATE_PATH_NAME" >/dev/null 2>&1; then + success_message="$success_message; automatic update watcher could not be started" + fi write_transaction_value phase COMMITTED - status SUCCEEDED "Release $TARGET_VERSION is running" "$TARGET_VERSION" + status SUCCEEDED "$success_message" "$TARGET_VERSION" TERMINAL_STATUS_WRITTEN=true complete_request rm -rf "$ACTIVE_TRANSACTION" exit 0 fi -rollback_active_transaction "Release health or application-surface verification failed" +service_state=$(systemctl is-active "$SERVICE_NAME" 2>/dev/null || true) +[ -n "$service_state" ] || service_state=unknown +rollback_active_transaction \ + "Release health or application-surface verification failed (service state: $service_state)" diff --git a/scripts/test-update-fixture.sh b/scripts/test-update-fixture.sh index 58921f4..f39bd48 100755 --- a/scripts/test-update-fixture.sh +++ b/scripts/test-update-fixture.sh @@ -92,6 +92,7 @@ SH cat > "$mock_bin/systemctl" <<'SH' #!/bin/sh printf '%s\n' "$*" >> "$MOCK_SYSTEMCTL_LOG" +[ "${MOCK_UPDATE_PATH_START:-success}:$*" != 'fail:start kaidi-update.path' ] || exit 1 exit 0 SH @@ -170,6 +171,7 @@ SH build_release() { local fixture=$1 local version=$2 + local unit_prefix=${3:-new} local stage="$fixture/stage" mkdir -p "$fixture/release" "$stage/public" "$stage/ops" printf 'new application\n' > "$stage/app.jar" @@ -178,7 +180,7 @@ build_release() { printf '#!/bin/sh\nprintf "new updater\\n"\n' > "$stage/ops/update.sh" chmod 0755 "$stage/ops/update.sh" for name in kaidi-finance.service kaidi-update.service kaidi-update.path; do - printf 'new %s\n' "$name" > "$stage/ops/$name" + printf '%s %s\n' "$unit_prefix" "$name" > "$stage/ops/$name" done local artifact="kaidi-finance-$version.tar.gz" @@ -272,6 +274,7 @@ run_update() { MOCK_RELEASE_ORIGIN="${FIXTURE_RELEASE_ORIGIN:-https://release.fixture.invalid}" \ MOCK_EXPECT_RELEASE_TOKEN="${FIXTURE_RELEASE_TOKEN-}" \ MOCK_SYSTEMCTL_LOG="$fixture/systemctl.log" \ + MOCK_UPDATE_PATH_START="${MOCK_UPDATE_PATH_START:-success}" \ KAIDI_SERVICE_USER="$(id -un)" \ KAIDI_SERVICE_GROUP="$(id -gn)" \ KAIDI_APP_ROOT="$fixture/app" \ @@ -368,6 +371,41 @@ assert_success_case() { [ "$(mode_of "$fixture/app/current/ops/update.sh")" = 750 ] || fail 'release updater mode is not 0750' } +assert_identical_systemd_operations_case() { + local fixture="$WORK/identical-systemd" + local version='1.0.0-preview.2' + mkdir -p "$fixture" + write_mock_commands "$fixture/mock-bin" + build_release "$fixture" "$version" old + prepare_installation "$fixture" "$version" + + download_and_prepare_install "$fixture" "$version" + : > "$fixture/systemctl.log" + run_update "$fixture" success + [ "$(readlink "$fixture/app/current")" = "$fixture/app/releases/$version" ] \ + || fail 'identical systemd case did not activate the new application' + ! grep -qx 'daemon-reload' "$fixture/systemctl.log" \ + || fail 'identical systemd units triggered an unnecessary daemon reload' +} + +assert_update_path_failure_keeps_application_case() { + local fixture="$WORK/path-watcher-failure" + local version='1.0.0-preview.2' + mkdir -p "$fixture" + write_mock_commands "$fixture/mock-bin" + build_release "$fixture" "$version" + prepare_installation "$fixture" "$version" + + download_and_prepare_install "$fixture" "$version" + MOCK_UPDATE_PATH_START=fail run_update "$fixture" success + [ "$(readlink "$fixture/app/current")" = "$fixture/app/releases/$version" ] \ + || fail 'path watcher failure rolled back a healthy application' + [ "$(jq -r '.state' "$fixture/state/status.json")" = SUCCEEDED ] \ + || fail 'path watcher failure did not preserve successful application state' + grep -Fq 'automatic update watcher could not be started' "$fixture/state/status.json" \ + || fail 'path watcher failure did not preserve its diagnostic' +} + assert_rollback_case() { local fixture="$WORK/rollback" local version='1.0.0-preview.2' @@ -480,6 +518,8 @@ assert_install_without_verified_cache_rejected() { } assert_success_case +assert_identical_systemd_operations_case +assert_update_path_failure_keeps_application_case assert_rollback_case assert_download_failure_case assert_database_failure_case