fix: recover stuck background updates

scripts/tallynote-update-runner.sh

@@ -41,7 +41,25 @@ if [[ "$request_operation" == download ]]; then
   [[ -n "$node_bin" ]] || die 'node runtime not found'
   cli="$CURRENT_LINK/dist/server/cli/update.js"
   [[ -f "$cli" ]] || die 'update CLI not found in current release'
-  "$node_bin" "$cli" --request-file "$REQUEST_FILE" || exit $?
+  set +e
+  "$node_bin" "$cli" --request-file "$REQUEST_FILE"
+  download_result=$?
+  set -e
+  if (( download_result != 0 )); then
+    # The CLI normally records failed itself. Retry the explicit finalization
+    # for failures that happen before its catch handler can persist the row,
+    # then remove the one-shot request so a failed download cannot keep the
+    # path unit in a permanently triggered state.
+    download_job_id=$(sed -n 's/.*"jobId"[[:space:]]*:[[:space:]]*"\([0-9a-f-]*\)".*/\1/p' "$REQUEST_FILE" | head -n 1)
+    if [[ "$download_job_id" =~ ^[0-9a-f-]{36}$ ]]; then
+      for _ in 1 2 3; do
+        if "$node_bin" "$cli" --finalize-job "$download_job_id" --finalize-status failed --message '更新下载失败' >/dev/null 2>&1; then break; fi
+        sleep 1
+      done
+    fi
+    rm -f -- "$REQUEST_FILE"
+    exit "$download_result"
+  fi
   rm -f -- "$REQUEST_FILE"
   exit 0
 fi
@@ -112,6 +130,27 @@ recover_stale_state() {
     done
     return 1
   fi
+  if [[ "$current_target" == "$state_old" ]]; then
+    # The process may have restored the old release before it was killed. In
+    # that case the old link is already safe to serve, but the database row
+    # can still be `applying`; finish it as failed before clearing recovery
+    # markers so the UI does not poll forever.
+    recovery_node="$CURRENT_LINK/runtime/bin/node"
+    [[ -x "$recovery_node" ]] || recovery_node=$(command -v node || true)
+    if finalize_state_job "$recovery_node" failed "$state_job"; then
+      rm -f -- "$REQUEST_FILE" 2>/dev/null || true
+      clear_update_state || true
+      return 11
+    fi
+    # A crash before the CLI created its job row is safe to retry. Preserve
+    # the request while dropping only the stale state marker.
+    if [[ -f "$REQUEST_FILE" && ! -L "$REQUEST_FILE" ]]; then
+      clear_update_state || true
+      return 0
+    fi
+    clear_update_state || true
+    return 0
+  fi
   if [[ "$current_target" != "$state_old" ]]; then
     rollback_link="$PREFIX/.current-recovery-$$-${RANDOM}.tmp"
     [[ ! -e "$rollback_link" && ! -L "$rollback_link" ]] || return 1
@@ -159,7 +198,12 @@ fi
 rollback_current() {
   local current_target rollback_link
   current_target=$(readlink -f -- "$CURRENT_LINK" 2>/dev/null || true)
-  [[ "$current_target" == "$old_target" ]] && return 0
+  if [[ "$current_target" == "$old_target" ]]; then
+    # An earlier failure branch may already have restored the link. Keep the
+    # marker truthful so the EXIT trap can still finalize the job.
+    switched=0
+    return 0
+  fi
   rollback_link="$PREFIX/.current-rollback-$$-${RANDOM}.tmp"
   [[ ! -e "$rollback_link" && ! -L "$rollback_link" ]] || return 1
   ln -s -- "$old_target" "$rollback_link" || return 1
@@ -172,8 +216,16 @@ rollback_current() {
 
 finalize_failed_job() {
   [[ "$job_id" =~ ^[0-9a-f-]{36}$ ]] || return 0
-  [[ -n "$old_node" && -f "$CURRENT_LINK/dist/server/cli/update.js" ]] || return 0
-  "$old_node" "$CURRENT_LINK/dist/server/cli/update.js" --finalize-job "$job_id" --finalize-status failed --message '新版本健康检查失败，已恢复上一版本' >/dev/null 2>&1
+  [[ -n "$old_node" && -f "$CURRENT_LINK/dist/server/cli/update.js" ]] || return 1
+  # Give SQLite a moment to release a transient lock before declaring the
+  # recovery itself failed.
+  for _ in 1 2 3; do
+    if "$old_node" "$CURRENT_LINK/dist/server/cli/update.js" --finalize-job "$job_id" --finalize-status failed --message '新版本健康检查失败，已恢复上一版本' >/dev/null 2>&1; then
+      return 0
+    fi
+    sleep 1
+  done
+  return 1
 }
 
 finalize_completed_job() {
@@ -187,7 +239,10 @@ cleanup_after_update() {
   local result=$? rollback_ok=1
   if (( result != 0 && handled == 0 )); then
     if ! rollback_current; then rollback_ok=0; fi
-    if (( rollback_ok == 1 && switched == 0 )); then
+    # Once the old release is active again, always try to close the job. The
+    # previous marker could remain set when an earlier branch had already
+    # rolled back before entering this EXIT trap, leaving `applying` forever.
+    if (( rollback_ok == 1 )); then
       if finalize_failed_job; then
         rm -f -- "$REQUEST_FILE"
         clear_update_state || true

scripts/test-installer.sh

@@ -4,6 +4,13 @@ root=$(cd "$(dirname "$0")/.." && pwd)
 bash -n "$root/install.sh" "$root/scripts/tallynote-update.sh"
 grep -Eq '^RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 AF_NETLINK$' "$root/systemd/tallynote.service"
 grep -Eq '^RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 AF_NETLINK$' "$root/systemd/tallynote-update.service"
+grep -Eq '^PathExists=/opt/tallynote/\.update-state$' "$root/systemd/tallynote-update.path"
+grep -Eq '^PathChanged=/opt/tallynote/\.update-state$' "$root/systemd/tallynote-update.path"
+grep -Eq '^PathChanged=/opt/tallynote$' "$root/systemd/tallynote-update.path"
+if grep -Eq '^ConditionPathExists=' "$root/systemd/tallynote-update.service"; then
+  echo 'update service must not require only the request file' >&2
+  exit 1
+fi
 output=$(bash "$root/install.sh" --dry-run --release-base-url https://releases.example.test/releases)
 grep -q 'dry-run' <<<"$output"
 grep -q '\[阶段\] 检查运行环境' <<<"$output"
@@ -192,6 +199,17 @@ bash -c '
   set_env_key test value
 ' _ "$installer_lib" "$release_archive" "$tmp/install-release"
 
+# The installed path unit must watch both the data-directory request and the
+# release-prefix recovery marker after custom paths are substituted.
+rendered_path="$tmp/rendered-update.path"
+sed "s#/opt/tallynote#$tmp/custom-prefix#g; s#/var/lib/tallynote#$tmp/custom-data#g" \
+  "$root/systemd/tallynote-update.path" > "$rendered_path"
+grep -Fxq "PathExists=$tmp/custom-data/update-request.json" "$rendered_path"
+grep -Fxq "PathChanged=$tmp/custom-data/update-request.json" "$rendered_path"
+grep -Fxq "PathExists=$tmp/custom-prefix/.update-state" "$rendered_path"
+grep -Fxq "PathChanged=$tmp/custom-prefix/.update-state" "$rendered_path"
+grep -Fxq "PathChanged=$tmp/custom-prefix" "$rendered_path"
+
 # The production admin wrapper must load a release-relative runtime, change to
 # the release root, and forward CLI arguments without requiring pnpm.
 wrapper_prefix="$tmp/wrapper-prefix"