fix: make online updates observable and recoverable

scripts/test-update-fixture.sh

@@ -65,10 +65,14 @@ write_mock_commands() {
 output=
 url=
 header_file=
+write_out=
+http_status=200
 printf '%s\n' "$*" >> "${MOCK_CURL_LOG:-/dev/null}"
 while [ "$#" -gt 0 ]; do
   case "$1" in
-    -o) shift; output=$1 ;;
+    -o|--output) shift; output=$1 ;;
+    --write-out) shift; write_out=$1 ;;
+    --connect-timeout|--max-time) shift ;;
     --header|-H)
       shift
       case "${1:-}" in @*) header_file=${1#@} ;; esac
@@ -90,6 +94,26 @@ esac
 if [ -n "$output" ]; then
   if [ -n "${MOCK_RELEASE_API_URL:-}" ] && [ "$url" = "$MOCK_RELEASE_API_URL" ]; then
     cp "$FIXTURE_RELEASE_ROOT/release-api.json" "$output"
+  elif [ "$url" = "${KAIDI_HEALTH_URL:-http://127.0.0.1:18080/actuator/health}" ]; then
+    health_ok=false
+    case "${MOCK_HEALTH:-success}" in
+      success) health_ok=true ;;
+      fail-new)
+        [ "$(cat "$MOCK_APP_ROOT/current/VERSION" 2>/dev/null)" = '1.0.0-preview.1' ] && health_ok=true
+        ;;
+      fail-after-first)
+        health_count=$(cat "$MOCK_APP_ROOT/health-count" 2>/dev/null || printf 0)
+        health_count=$((health_count + 1))
+        printf '%s\n' "$health_count" > "$MOCK_APP_ROOT/health-count"
+        [ "$health_count" -eq 1 ] && health_ok=true
+        ;;
+    esac
+    if [ "$health_ok" = true ]; then
+      printf '{"status":"UP"}\n' > "$output"
+    else
+      printf '{"status":"DOWN"}\n' > "$output"
+      http_status=503
+    fi
   elif [ "$url" = "${KAIDI_APP_INDEX_URL:-http://127.0.0.1:18080/}" ]; then
     cp "$MOCK_APP_ROOT/current/public/index.html" "$output"
   else
@@ -104,6 +128,7 @@ elif [ "${MOCK_HEALTH:-success}" = fail-new ] \
 else
   exit 22
 fi
+[ -z "$write_out" ] || printf '%s' "$http_status"
 SH
 
   cat > "$mock_bin/systemctl" <<'SH'
@@ -116,6 +141,11 @@ SH
   cat > "$mock_bin/flock" <<'SH'
 #!/bin/sh
 exit 0
+SH
+
+  cat > "$mock_bin/setsid" <<'SH'
+#!/bin/sh
+exec "$@"
 SH
 
   cat > "$mock_bin/systemd-analyze" <<'SH'
@@ -211,10 +241,12 @@ build_release() {
 
   local artifact="kaidi-finance-$version.tar.gz"
   COPYFILE_DISABLE=1 tar -czf "$fixture/release/$artifact" -C "$stage" .
-  local sha
+  local sha size
   sha=$($REAL_OPENSSL dgst -sha256 "$fixture/release/$artifact" | awk '{print $NF}')
-  jq -n --arg version "$version" --arg artifact "$artifact" --arg sha "$sha" \
-    '{version:$version,artifact:$artifact,sha256:$sha,publishedAt:"2026-08-16T00:00:00Z",releaseNotes:"fixture"}' \
+  size=$(wc -c < "$fixture/release/$artifact" | tr -d '[:space:]')
+  jq -n --arg version "$version" --arg artifact "$artifact" --arg sha "$sha" --argjson size "$size" \
+    '{version:$version,artifact:$artifact,sha256:$sha,artifactSizeBytes:$size,
+      publishedAt:"2026-08-16T00:00:00Z",releaseNotes:"fixture"}' \
     > "$fixture/release/release-manifest.json"
   "$REAL_OPENSSL" dgst -sha256 -sign "$fixture/private.pem" \
     -out "$fixture/release/release-manifest.sig" "$fixture/release/release-manifest.json"
@@ -269,6 +301,7 @@ write_request() {
 download_and_prepare_install() {
   local fixture=$1
   local version=$2
+  truncate -s 0 "$fixture/systemctl.log"
   run_update "$fixture" success
   [ "$(jq -r '.state' "$fixture/state/status.json")" = READY ] \
     || fail 'download phase did not persist READY'
@@ -276,6 +309,16 @@ download_and_prepare_install() {
     || fail 'download phase changed the active application'
   [ -s "$fixture/state/cache/$version/release.tar.gz" ] \
     || fail 'download phase did not persist the verified artifact cache'
+  [ "$(jq -r '.downloadPercent' "$fixture/state/status.json")" -eq 100 ] \
+    || fail 'download phase did not persist 100 percent progress'
+  [ "$(jq -r '.totalBytes' "$fixture/state/status.json")" -gt 0 ] \
+    || fail 'download phase did not persist artifact bytes'
+  [ "$(jq -r '.bytesPerSecond' "$fixture/state/status.json")" -gt 0 ] \
+    || fail 'download phase did not persist a measured transfer speed'
+  grep -Fq '"stage":"DOWNLOADING"' "$fixture/state/events.jsonl" \
+    || fail 'download phase did not persist structured runtime events'
+  ! grep -Eq '^(start|restart|stop) kaidi-finance.service$' "$fixture/systemctl.log" \
+    || fail 'download phase changed the application service'
   write_request "$fixture" "$version" INSTALL
 }
 
@@ -407,7 +450,7 @@ assert_identical_systemd_operations_case() {
   prepare_installation "$fixture" "$version"
 
   download_and_prepare_install "$fixture" "$version"
-  : > "$fixture/systemctl.log"
+  truncate -s 0 "$fixture/systemctl.log"
   run_update "$fixture" success
   [ "$(readlink "$fixture/app/current")" = "$fixture/app/releases/$version" ] \
     || fail 'identical systemd case did not activate the new application'
@@ -474,17 +517,33 @@ assert_incomplete_rollback_requires_manual_recovery_case() {
   prepare_installation "$fixture" "$version"
   download_and_prepare_install "$fixture" "$version"
 
-  if run_update "$fixture" fail > "$fixture/update.log" 2>&1; then
+  if run_update "$fixture" fail-after-first > "$fixture/update.log" 2>&1; then
     fail 'incomplete rollback case unexpectedly succeeded'
   fi
   grep -Fq 'manual recovery is required' "$fixture/update.log" \
     || fail 'incomplete rollback case did not require explicit recovery'
   [ ! -e "$fixture/state/processing/request.json" ] \
     || fail 'incomplete rollback case left an automatically retriggered processing request'
-  [ -d "$fixture/state/transactions/active" ] \
-    || fail 'incomplete rollback case did not preserve transaction evidence'
+  [ ! -e "$fixture/state/transactions/active" ] \
+    || fail 'incomplete rollback case left transaction evidence on the automatic recovery path'
+  [ -d "$fixture/state/transactions/recovery-required" ] \
+    || fail 'incomplete rollback case did not quarantine transaction evidence'
+  [ "$(jq -r '.state' "$fixture/state/status.json")" = RECOVERY_REQUIRED ] \
+    || fail 'incomplete rollback case did not lock the updater for recovery'
   find "$fixture/state/failed" -type f -name 'request-*.json' -print -quit | grep -q . \
     || fail 'incomplete rollback case did not archive its claimed request'
+
+  truncate -s 0 "$fixture/systemctl.log"
+  write_request "$fixture" "$version" DOWNLOAD
+  if run_update "$fixture" success > "$fixture/retry.log" 2>&1; then
+    fail 'recovery-locked updater accepted a new download request'
+  fi
+  [ "$(jq -r '.state' "$fixture/state/status.json")" = RECOVERY_REQUIRED ] \
+    || fail 'recovery-locked updater replaced the manual recovery status'
+  ! grep -Eq '^(start|restart|stop) kaidi-finance.service$' "$fixture/systemctl.log" \
+    || fail 'recovery-locked download request changed the application service'
+  [ ! -e "$fixture/state/inbox/request.json" ] && [ ! -e "$fixture/state/processing/request.json" ] \
+    || fail 'recovery-locked updater left a request on an automatic trigger path'
 }
 
 assert_download_failure_case() {
@@ -526,6 +585,58 @@ assert_database_failure_case() {
     || fail 'database failure did not persist FAILED'
 }
 
+assert_unhealthy_baseline_blocks_restart_case() {
+  local fixture="$WORK/unhealthy-baseline"
+  local version='1.0.0-preview.2'
+  mkdir -p "$fixture"
+  write_mock_commands "$fixture/mock-bin"
+  build_release "$fixture" "$version"
+  prepare_installation "$fixture" "$version"
+  download_and_prepare_install "$fixture" "$version"
+  truncate -s 0 "$fixture/systemctl.log"
+
+  if run_update "$fixture" fail > "$fixture/update.log" 2>&1; then
+    fail 'unhealthy baseline unexpectedly reached installation'
+  fi
+  grep -Fq 'Current release preflight failed' "$fixture/update.log" \
+    || fail 'unhealthy baseline did not preserve its preflight diagnostic'
+  ! grep -Eq '^(start|restart|stop) kaidi-finance.service$' "$fixture/systemctl.log" \
+    || fail 'unhealthy baseline restarted the application'
+  [ "$(readlink "$fixture/app/current")" = "$fixture/app/releases/1.0.0-preview.1" ] \
+    || fail 'unhealthy baseline changed the active release'
+  [ ! -e "$fixture/state/transactions/active" ] \
+    || fail 'unhealthy baseline created a switching transaction'
+}
+
+assert_invalid_database_url_blocks_restart_case() {
+  local fixture="$WORK/invalid-database-url"
+  local version='1.0.0-preview.2'
+  mkdir -p "$fixture"
+  write_mock_commands "$fixture/mock-bin"
+  build_release "$fixture" "$version"
+  prepare_installation "$fixture" "$version"
+  download_and_prepare_install "$fixture" "$version"
+  cat > "$fixture/runtime.env" <<'EOF'
+DB_URL="jdbc:mysql://"
+KAIDI_DB_HOST="127.0.0.1"
+KAIDI_DB_PORT="3306"
+KAIDI_DB_NAME="kaidi_finance"
+KAIDI_DB_USERNAME="kaidi"
+KAIDI_DB_PASSWORD="fixture"
+EOF
+  truncate -s 0 "$fixture/systemctl.log"
+
+  if run_update "$fixture" success > "$fixture/update.log" 2>&1; then
+    fail 'invalid database URL unexpectedly reached installation'
+  fi
+  grep -Fq 'DB_URL does not match the configured MySQL host, port, and database' "$fixture/update.log" \
+    || fail 'structurally invalid JDBC URL did not preserve its diagnostic'
+  ! grep -q '^restart kaidi-finance.service$' "$fixture/systemctl.log" \
+    || fail 'invalid database URL restarted the application'
+  [ "$(readlink "$fixture/app/current")" = "$fixture/app/releases/1.0.0-preview.1" ] \
+    || fail 'invalid database URL changed the active release'
+}
+
 assert_symlink_request_rejected() {
   local fixture="$WORK/symlink-request"
   local version='1.0.0-preview.2'
@@ -577,6 +688,8 @@ assert_rollback_case
 assert_incomplete_rollback_requires_manual_recovery_case
 assert_download_failure_case
 assert_database_failure_case
+assert_unhealthy_baseline_blocks_restart_case
+assert_invalid_database_url_blocks_restart_case
 assert_symlink_request_rejected
 assert_install_without_verified_cache_rejected
 assert_private_gitea_release_case