fix: keep online update recovery available

deploy/update.sh

@@ -18,6 +18,7 @@ RELEASE_API_URL=${UPDATE_RELEASE_API_URL:-}
 RELEASE_TOKEN=${UPDATE_RELEASE_TOKEN:-}
 CACHE_ROOT=${KAIDI_UPDATE_CACHE_ROOT:-$STATE_ROOT/cache}
 SERVICE_NAME=${KAIDI_SERVICE_NAME:-kaidi-finance.service}
+UPDATE_PATH_NAME=${KAIDI_UPDATE_PATH_NAME:-kaidi-update.path}
 SERVICE_USER=${KAIDI_SERVICE_USER:-kaidi}
 SERVICE_GROUP=${KAIDI_SERVICE_GROUP:-kaidi}
 HEALTH_URL=${KAIDI_HEALTH_URL:-http://127.0.0.1:18080/actuator/health}
@@ -272,6 +273,22 @@ atomic_install() {
   mv -f "$install_temporary" "$install_destination"
 }
 
+install_operation() {
+  operation_source=$1
+  operation_destination=$2
+  operation_mode=$3
+  operation_is_systemd=$4
+  if [ -f "$operation_destination" ] && [ ! -L "$operation_destination" ] \
+    && cmp -s "$operation_source" "$operation_destination"; then
+    chmod "$operation_mode" "$operation_destination"
+    return
+  fi
+  atomic_install "$operation_source" "$operation_destination" "$operation_mode" || return 1
+  if [ "$operation_is_systemd" = true ]; then
+    SYSTEMD_OPERATIONS_CHANGED=true
+  fi
+}
+
 backup_managed_file() {
   backup_source=$1
   backup_name=$2
@@ -311,14 +328,16 @@ backup_operations() {
 }
 
 apply_operations() {
-  atomic_install "$RELEASE_DIR/ops/update.sh" "$UPDATER_PATH" 0755 \
+  SYSTEMD_OPERATIONS_CHANGED=false
-    && atomic_install "$RELEASE_DIR/ops/kaidi-finance.service" \
+  install_operation "$RELEASE_DIR/ops/update.sh" "$UPDATER_PATH" 0755 false \
-      "$SYSTEMD_ROOT/kaidi-finance.service" 0644 \
+    && install_operation "$RELEASE_DIR/ops/kaidi-finance.service" \
-    && atomic_install "$RELEASE_DIR/ops/kaidi-update.service" \
+      "$SYSTEMD_ROOT/kaidi-finance.service" 0644 true \
-      "$SYSTEMD_ROOT/kaidi-update.service" 0644 \
+    && install_operation "$RELEASE_DIR/ops/kaidi-update.service" \
-    && atomic_install "$RELEASE_DIR/ops/kaidi-update.path" \
+      "$SYSTEMD_ROOT/kaidi-update.service" 0644 true \
-      "$SYSTEMD_ROOT/kaidi-update.path" 0644 \
+    && install_operation "$RELEASE_DIR/ops/kaidi-update.path" \
-    && systemctl daemon-reload
+      "$SYSTEMD_ROOT/kaidi-update.path" 0644 true \
+    || return 1
+  [ "$SYSTEMD_OPERATIONS_CHANGED" = false ] || systemctl daemon-reload
 }
 
 restore_operations() {
@@ -357,25 +376,29 @@ rollback_active_transaction() {
   previous_target=$(transaction_value previous-target)
   failed_release=$(transaction_value release-dir)
   OPS_BACKUP="$ACTIVE_TRANSACTION/operations-backup"
-  rollback_ok=true
+  link_restored=true
+  operations_restored=true
 
   systemctl stop "$SERVICE_NAME" || true
   if [ -n "$previous_target" ] && [ -d "$previous_target" ]; then
     if ! ln -sfn "$previous_target" "$APP_ROOT/current.next" \
       || ! mv -Tf "$APP_ROOT/current.next" "$APP_ROOT/current"; then
-      rollback_ok=false
+      link_restored=false
     fi
   else
-    rollback_ok=false
+    link_restored=false
   fi
-  restore_operations || rollback_ok=false
+  restore_operations || operations_restored=false
   systemctl reset-failed "$SERVICE_NAME" >/dev/null 2>&1 || true
-  systemctl start "$SERVICE_NAME" || rollback_ok=false
+  systemctl start "$SERVICE_NAME" || true
-  if [ "$rollback_ok" = true ] && verify_app_surface; then
+  if [ "$link_restored" = true ] && verify_app_surface; then
     remove_failed_release "$failed_release"
     rm -rf "$ACTIVE_TRANSACTION"
+    if [ "$operations_restored" = true ]; then
       fail "$reason; previous release was restored and verified"
     fi
+    fail "$reason; previous release is running, but operations restoration requires manual review"
+  fi
   TERMINAL_STATUS_WRITTEN=true
   status FAILED "$reason; rollback is incomplete and will be retried" "${TARGET_VERSION:-}"
   printf '%s\n' "$reason; rollback is incomplete and will be retried" >&2
@@ -600,15 +623,21 @@ write_transaction_value phase APP_SWITCHED
 status RUNNING "Starting and verifying release $TARGET_VERSION" "$TARGET_VERSION"
 write_transaction_value phase HEALTH_CHECKING
 systemctl reset-failed "$SERVICE_NAME" >/dev/null 2>&1 || true
-if systemctl start "$SERVICE_NAME" \
+systemctl start "$SERVICE_NAME" || true
-  && systemctl is-active --quiet kaidi-update.path \
+if verify_app_surface; then
-  && verify_app_surface; then
+  success_message="Release $TARGET_VERSION is running"
+  if ! systemctl start "$UPDATE_PATH_NAME" >/dev/null 2>&1; then
+    success_message="$success_message; automatic update watcher could not be started"
+  fi
   write_transaction_value phase COMMITTED
-  status SUCCEEDED "Release $TARGET_VERSION is running" "$TARGET_VERSION"
+  status SUCCEEDED "$success_message" "$TARGET_VERSION"
   TERMINAL_STATUS_WRITTEN=true
   complete_request
   rm -rf "$ACTIVE_TRANSACTION"
   exit 0
 fi
 
-rollback_active_transaction "Release health or application-surface verification failed"
+service_state=$(systemctl is-active "$SERVICE_NAME" 2>/dev/null || true)
+[ -n "$service_state" ] || service_state=unknown
+rollback_active_transaction \
+  "Release health or application-surface verification failed (service state: $service_state)"

scripts/test-update-fixture.sh

@@ -92,6 +92,7 @@ SH
   cat > "$mock_bin/systemctl" <<'SH'
 #!/bin/sh
 printf '%s\n' "$*" >> "$MOCK_SYSTEMCTL_LOG"
+[ "${MOCK_UPDATE_PATH_START:-success}:$*" != 'fail:start kaidi-update.path' ] || exit 1
 exit 0
 SH
 
@@ -170,6 +171,7 @@ SH
 build_release() {
   local fixture=$1
   local version=$2
+  local unit_prefix=${3:-new}
   local stage="$fixture/stage"
   mkdir -p "$fixture/release" "$stage/public" "$stage/ops"
   printf 'new application\n' > "$stage/app.jar"
@@ -178,7 +180,7 @@ build_release() {
   printf '#!/bin/sh\nprintf "new updater\\n"\n' > "$stage/ops/update.sh"
   chmod 0755 "$stage/ops/update.sh"
   for name in kaidi-finance.service kaidi-update.service kaidi-update.path; do
-    printf 'new %s\n' "$name" > "$stage/ops/$name"
+    printf '%s %s\n' "$unit_prefix" "$name" > "$stage/ops/$name"
   done
 
   local artifact="kaidi-finance-$version.tar.gz"
@@ -272,6 +274,7 @@ run_update() {
     MOCK_RELEASE_ORIGIN="${FIXTURE_RELEASE_ORIGIN:-https://release.fixture.invalid}" \
     MOCK_EXPECT_RELEASE_TOKEN="${FIXTURE_RELEASE_TOKEN-}" \
     MOCK_SYSTEMCTL_LOG="$fixture/systemctl.log" \
+    MOCK_UPDATE_PATH_START="${MOCK_UPDATE_PATH_START:-success}" \
     KAIDI_SERVICE_USER="$(id -un)" \
     KAIDI_SERVICE_GROUP="$(id -gn)" \
     KAIDI_APP_ROOT="$fixture/app" \
@@ -368,6 +371,41 @@ assert_success_case() {
   [ "$(mode_of "$fixture/app/current/ops/update.sh")" = 750 ] || fail 'release updater mode is not 0750'
 }
 
+assert_identical_systemd_operations_case() {
+  local fixture="$WORK/identical-systemd"
+  local version='1.0.0-preview.2'
+  mkdir -p "$fixture"
+  write_mock_commands "$fixture/mock-bin"
+  build_release "$fixture" "$version" old
+  prepare_installation "$fixture" "$version"
+
+  download_and_prepare_install "$fixture" "$version"
+  : > "$fixture/systemctl.log"
+  run_update "$fixture" success
+  [ "$(readlink "$fixture/app/current")" = "$fixture/app/releases/$version" ] \
+    || fail 'identical systemd case did not activate the new application'
+  ! grep -qx 'daemon-reload' "$fixture/systemctl.log" \
+    || fail 'identical systemd units triggered an unnecessary daemon reload'
+}
+
+assert_update_path_failure_keeps_application_case() {
+  local fixture="$WORK/path-watcher-failure"
+  local version='1.0.0-preview.2'
+  mkdir -p "$fixture"
+  write_mock_commands "$fixture/mock-bin"
+  build_release "$fixture" "$version"
+  prepare_installation "$fixture" "$version"
+
+  download_and_prepare_install "$fixture" "$version"
+  MOCK_UPDATE_PATH_START=fail run_update "$fixture" success
+  [ "$(readlink "$fixture/app/current")" = "$fixture/app/releases/$version" ] \
+    || fail 'path watcher failure rolled back a healthy application'
+  [ "$(jq -r '.state' "$fixture/state/status.json")" = SUCCEEDED ] \
+    || fail 'path watcher failure did not preserve successful application state'
+  grep -Fq 'automatic update watcher could not be started' "$fixture/state/status.json" \
+    || fail 'path watcher failure did not preserve its diagnostic'
+}
+
 assert_rollback_case() {
   local fixture="$WORK/rollback"
   local version='1.0.0-preview.2'
@@ -480,6 +518,8 @@ assert_install_without_verified_cache_rejected() {
 }
 
 assert_success_case
+assert_identical_systemd_operations_case
+assert_update_path_failure_keeps_application_case
 assert_rollback_case
 assert_download_failure_case
 assert_database_failure_case