fix: make online updates observable and recoverable

scripts/package-release.sh

@@ -105,6 +105,7 @@ chmod 0755 "$STAGE/ops/update.sh" "$STAGE/ops/baota-start.sh" "$STAGE/ops/baota-
 ARTIFACT="kaidi-finance-$VERSION.tar.gz"
 COPYFILE_DISABLE=1 tar --format=ustar -czf "$OUTPUT_DIR/$ARTIFACT" -C "$STAGE" .
 SHA256=$(sha256_file "$OUTPUT_DIR/$ARTIFACT")
+ARTIFACT_SIZE_BYTES=$(wc -c < "$OUTPUT_DIR/$ARTIFACT" | tr -d '[:space:]')
 cp "$ROOT/backend/target/backend-sbom.json" "$OUTPUT_DIR/backend-sbom.cdx.json"
 (cd "$ROOT/frontend" && npm sbom --omit=dev --package-lock-only \
   --sbom-format cyclonedx --sbom-type application) > "$OUTPUT_DIR/frontend-sbom.cdx.json"
@@ -167,7 +168,8 @@ EOF
 BOOTSTRAP_SHA256=$(sha256_file "$OUTPUT_DIR/bootstrap-checksums.txt")
 
 RELEASE_NOTES_VALUE=${KAIDI_RELEASE_NOTES:-"Kaidi Finance Preview $VERSION"}
-VERSION="$VERSION" ARTIFACT="$ARTIFACT" SHA256="$SHA256" RELEASE_NOTES="$RELEASE_NOTES_VALUE" \
+VERSION="$VERSION" ARTIFACT="$ARTIFACT" SHA256="$SHA256" ARTIFACT_SIZE_BYTES="$ARTIFACT_SIZE_BYTES" \
+  RELEASE_NOTES="$RELEASE_NOTES_VALUE" \
   BACKEND_SBOM_SHA256="$BACKEND_SBOM_SHA256" FRONTEND_SBOM_SHA256="$FRONTEND_SBOM_SHA256" \
   BACKEND_BUILD_VERSION="$BACKEND_BUILD_VERSION" FRONTEND_BUILD_VERSION="$FRONTEND_BUILD_VERSION" \
   INSTALLER_SHA256="$INSTALLER_SHA256" PURGER_SHA256="$PURGER_SHA256" \
@@ -179,6 +181,7 @@ const manifest = {
   version: process.env.VERSION,
   artifact: process.env.ARTIFACT,
   sha256: process.env.SHA256,
+  artifactSizeBytes: Number(process.env.ARTIFACT_SIZE_BYTES),
   publishedAt: new Date().toISOString(),
   minimumJava: 17,
   source: {

scripts/test-install-fixture.sh

@@ -321,8 +321,10 @@ grep -Fqx 'Restart=no' "$ROOT/deploy/systemd/kaidi-update.service" \
   || fail 'update service can automatically repeat a failed switching transaction'
 grep -Fq -- '-/www/wwwroot/kaidi' "$ROOT/deploy/systemd/kaidi-update.service" \
   || fail 'update service requires the Baota application path on a systemd-only installation'
-! grep -Fq '/var/lib/kaidi-update/processing' "$ROOT/deploy/systemd/kaidi-update.path" \
-  || fail 'update path can automatically repeat a claimed switching transaction'
+grep -Fqx 'PathExists=/var/lib/kaidi-update/processing/request.json' "$ROOT/deploy/systemd/kaidi-update.path" \
+  || fail 'update path does not resume an interrupted claimed request'
+grep -Fqx 'PathExists=/var/lib/kaidi-update/transactions/active' "$ROOT/deploy/systemd/kaidi-update.path" \
+  || fail 'update path does not resume an interrupted switching transaction'
 # shellcheck disable=SC2016 # Match literal installer source.
 grep -Fq '[ "$actual_sha256" = "$java_sha256" ]' "$ROOT/deploy/install.sh" \
   || fail 'installer no longer verifies the Java runtime SHA-256'

scripts/test-update-fixture.sh

@@ -65,10 +65,14 @@ write_mock_commands() {
 output=
 url=
 header_file=
+write_out=
+http_status=200
 printf '%s\n' "$*" >> "${MOCK_CURL_LOG:-/dev/null}"
 while [ "$#" -gt 0 ]; do
   case "$1" in
-    -o) shift; output=$1 ;;
+    -o|--output) shift; output=$1 ;;
+    --write-out) shift; write_out=$1 ;;
+    --connect-timeout|--max-time) shift ;;
     --header|-H)
       shift
       case "${1:-}" in @*) header_file=${1#@} ;; esac
@@ -90,6 +94,26 @@ esac
 if [ -n "$output" ]; then
   if [ -n "${MOCK_RELEASE_API_URL:-}" ] && [ "$url" = "$MOCK_RELEASE_API_URL" ]; then
     cp "$FIXTURE_RELEASE_ROOT/release-api.json" "$output"
+  elif [ "$url" = "${KAIDI_HEALTH_URL:-http://127.0.0.1:18080/actuator/health}" ]; then
+    health_ok=false
+    case "${MOCK_HEALTH:-success}" in
+      success) health_ok=true ;;
+      fail-new)
+        [ "$(cat "$MOCK_APP_ROOT/current/VERSION" 2>/dev/null)" = '1.0.0-preview.1' ] && health_ok=true
+        ;;
+      fail-after-first)
+        health_count=$(cat "$MOCK_APP_ROOT/health-count" 2>/dev/null || printf 0)
+        health_count=$((health_count + 1))
+        printf '%s\n' "$health_count" > "$MOCK_APP_ROOT/health-count"
+        [ "$health_count" -eq 1 ] && health_ok=true
+        ;;
+    esac
+    if [ "$health_ok" = true ]; then
+      printf '{"status":"UP"}\n' > "$output"
+    else
+      printf '{"status":"DOWN"}\n' > "$output"
+      http_status=503
+    fi
   elif [ "$url" = "${KAIDI_APP_INDEX_URL:-http://127.0.0.1:18080/}" ]; then
     cp "$MOCK_APP_ROOT/current/public/index.html" "$output"
   else
@@ -104,6 +128,7 @@ elif [ "${MOCK_HEALTH:-success}" = fail-new ] \
 else
   exit 22
 fi
+[ -z "$write_out" ] || printf '%s' "$http_status"
 SH
 
   cat > "$mock_bin/systemctl" <<'SH'
@@ -116,6 +141,11 @@ SH
   cat > "$mock_bin/flock" <<'SH'
 #!/bin/sh
 exit 0
+SH
+
+  cat > "$mock_bin/setsid" <<'SH'
+#!/bin/sh
+exec "$@"
 SH
 
   cat > "$mock_bin/systemd-analyze" <<'SH'
@@ -211,10 +241,12 @@ build_release() {
 
   local artifact="kaidi-finance-$version.tar.gz"
   COPYFILE_DISABLE=1 tar -czf "$fixture/release/$artifact" -C "$stage" .
-  local sha
+  local sha size
   sha=$($REAL_OPENSSL dgst -sha256 "$fixture/release/$artifact" | awk '{print $NF}')
-  jq -n --arg version "$version" --arg artifact "$artifact" --arg sha "$sha" \
-    '{version:$version,artifact:$artifact,sha256:$sha,publishedAt:"2026-08-16T00:00:00Z",releaseNotes:"fixture"}' \
+  size=$(wc -c < "$fixture/release/$artifact" | tr -d '[:space:]')
+  jq -n --arg version "$version" --arg artifact "$artifact" --arg sha "$sha" --argjson size "$size" \
+    '{version:$version,artifact:$artifact,sha256:$sha,artifactSizeBytes:$size,
+      publishedAt:"2026-08-16T00:00:00Z",releaseNotes:"fixture"}' \
     > "$fixture/release/release-manifest.json"
   "$REAL_OPENSSL" dgst -sha256 -sign "$fixture/private.pem" \
     -out "$fixture/release/release-manifest.sig" "$fixture/release/release-manifest.json"
@@ -269,6 +301,7 @@ write_request() {
 download_and_prepare_install() {
   local fixture=$1
   local version=$2
+  truncate -s 0 "$fixture/systemctl.log"
   run_update "$fixture" success
   [ "$(jq -r '.state' "$fixture/state/status.json")" = READY ] \
     || fail 'download phase did not persist READY'
@@ -276,6 +309,16 @@ download_and_prepare_install() {
     || fail 'download phase changed the active application'
   [ -s "$fixture/state/cache/$version/release.tar.gz" ] \
     || fail 'download phase did not persist the verified artifact cache'
+  [ "$(jq -r '.downloadPercent' "$fixture/state/status.json")" -eq 100 ] \
+    || fail 'download phase did not persist 100 percent progress'
+  [ "$(jq -r '.totalBytes' "$fixture/state/status.json")" -gt 0 ] \
+    || fail 'download phase did not persist artifact bytes'
+  [ "$(jq -r '.bytesPerSecond' "$fixture/state/status.json")" -gt 0 ] \
+    || fail 'download phase did not persist a measured transfer speed'
+  grep -Fq '"stage":"DOWNLOADING"' "$fixture/state/events.jsonl" \
+    || fail 'download phase did not persist structured runtime events'
+  ! grep -Eq '^(start|restart|stop) kaidi-finance.service$' "$fixture/systemctl.log" \
+    || fail 'download phase changed the application service'
   write_request "$fixture" "$version" INSTALL
 }
 
@@ -407,7 +450,7 @@ assert_identical_systemd_operations_case() {
   prepare_installation "$fixture" "$version"
 
   download_and_prepare_install "$fixture" "$version"
-  : > "$fixture/systemctl.log"
+  truncate -s 0 "$fixture/systemctl.log"
   run_update "$fixture" success
   [ "$(readlink "$fixture/app/current")" = "$fixture/app/releases/$version" ] \
     || fail 'identical systemd case did not activate the new application'
@@ -474,17 +517,33 @@ assert_incomplete_rollback_requires_manual_recovery_case() {
   prepare_installation "$fixture" "$version"
   download_and_prepare_install "$fixture" "$version"
 
-  if run_update "$fixture" fail > "$fixture/update.log" 2>&1; then
+  if run_update "$fixture" fail-after-first > "$fixture/update.log" 2>&1; then
     fail 'incomplete rollback case unexpectedly succeeded'
   fi
   grep -Fq 'manual recovery is required' "$fixture/update.log" \
     || fail 'incomplete rollback case did not require explicit recovery'
   [ ! -e "$fixture/state/processing/request.json" ] \
     || fail 'incomplete rollback case left an automatically retriggered processing request'
-  [ -d "$fixture/state/transactions/active" ] \
-    || fail 'incomplete rollback case did not preserve transaction evidence'
+  [ ! -e "$fixture/state/transactions/active" ] \
+    || fail 'incomplete rollback case left transaction evidence on the automatic recovery path'
+  [ -d "$fixture/state/transactions/recovery-required" ] \
+    || fail 'incomplete rollback case did not quarantine transaction evidence'
+  [ "$(jq -r '.state' "$fixture/state/status.json")" = RECOVERY_REQUIRED ] \
+    || fail 'incomplete rollback case did not lock the updater for recovery'
   find "$fixture/state/failed" -type f -name 'request-*.json' -print -quit | grep -q . \
     || fail 'incomplete rollback case did not archive its claimed request'
+
+  truncate -s 0 "$fixture/systemctl.log"
+  write_request "$fixture" "$version" DOWNLOAD
+  if run_update "$fixture" success > "$fixture/retry.log" 2>&1; then
+    fail 'recovery-locked updater accepted a new download request'
+  fi
+  [ "$(jq -r '.state' "$fixture/state/status.json")" = RECOVERY_REQUIRED ] \
+    || fail 'recovery-locked updater replaced the manual recovery status'
+  ! grep -Eq '^(start|restart|stop) kaidi-finance.service$' "$fixture/systemctl.log" \
+    || fail 'recovery-locked download request changed the application service'
+  [ ! -e "$fixture/state/inbox/request.json" ] && [ ! -e "$fixture/state/processing/request.json" ] \
+    || fail 'recovery-locked updater left a request on an automatic trigger path'
 }
 
 assert_download_failure_case() {
@@ -526,6 +585,58 @@ assert_database_failure_case() {
     || fail 'database failure did not persist FAILED'
 }
 
+assert_unhealthy_baseline_blocks_restart_case() {
+  local fixture="$WORK/unhealthy-baseline"
+  local version='1.0.0-preview.2'
+  mkdir -p "$fixture"
+  write_mock_commands "$fixture/mock-bin"
+  build_release "$fixture" "$version"
+  prepare_installation "$fixture" "$version"
+  download_and_prepare_install "$fixture" "$version"
+  truncate -s 0 "$fixture/systemctl.log"
+
+  if run_update "$fixture" fail > "$fixture/update.log" 2>&1; then
+    fail 'unhealthy baseline unexpectedly reached installation'
+  fi
+  grep -Fq 'Current release preflight failed' "$fixture/update.log" \
+    || fail 'unhealthy baseline did not preserve its preflight diagnostic'
+  ! grep -Eq '^(start|restart|stop) kaidi-finance.service$' "$fixture/systemctl.log" \
+    || fail 'unhealthy baseline restarted the application'
+  [ "$(readlink "$fixture/app/current")" = "$fixture/app/releases/1.0.0-preview.1" ] \
+    || fail 'unhealthy baseline changed the active release'
+  [ ! -e "$fixture/state/transactions/active" ] \
+    || fail 'unhealthy baseline created a switching transaction'
+}
+
+assert_invalid_database_url_blocks_restart_case() {
+  local fixture="$WORK/invalid-database-url"
+  local version='1.0.0-preview.2'
+  mkdir -p "$fixture"
+  write_mock_commands "$fixture/mock-bin"
+  build_release "$fixture" "$version"
+  prepare_installation "$fixture" "$version"
+  download_and_prepare_install "$fixture" "$version"
+  cat > "$fixture/runtime.env" <<'EOF'
+DB_URL="jdbc:mysql://"
+KAIDI_DB_HOST="127.0.0.1"
+KAIDI_DB_PORT="3306"
+KAIDI_DB_NAME="kaidi_finance"
+KAIDI_DB_USERNAME="kaidi"
+KAIDI_DB_PASSWORD="fixture"
+EOF
+  truncate -s 0 "$fixture/systemctl.log"
+
+  if run_update "$fixture" success > "$fixture/update.log" 2>&1; then
+    fail 'invalid database URL unexpectedly reached installation'
+  fi
+  grep -Fq 'DB_URL does not match the configured MySQL host, port, and database' "$fixture/update.log" \
+    || fail 'structurally invalid JDBC URL did not preserve its diagnostic'
+  ! grep -q '^restart kaidi-finance.service$' "$fixture/systemctl.log" \
+    || fail 'invalid database URL restarted the application'
+  [ "$(readlink "$fixture/app/current")" = "$fixture/app/releases/1.0.0-preview.1" ] \
+    || fail 'invalid database URL changed the active release'
+}
+
 assert_symlink_request_rejected() {
   local fixture="$WORK/symlink-request"
   local version='1.0.0-preview.2'
@@ -577,6 +688,8 @@ assert_rollback_case
 assert_incomplete_rollback_requires_manual_recovery_case
 assert_download_failure_case
 assert_database_failure_case
+assert_unhealthy_baseline_blocks_restart_case
+assert_invalid_database_url_blocks_restart_case
 assert_symlink_request_rejected
 assert_install_without_verified_cache_rejected
 assert_private_gitea_release_case

scripts/verify-release.sh

@@ -56,6 +56,10 @@ EXPECTED_ARTIFACT_SHA256=$(jq -er \
   release-manifest.json)
 [ "$(sha256_file "$ARTIFACT")" = "$EXPECTED_ARTIFACT_SHA256" ] \
   || { printf 'Release artifact digest does not match the signed manifest\n' >&2; exit 1; }
+EXPECTED_ARTIFACT_SIZE=$(jq -er '.artifactSizeBytes | numbers | floor | select(. > 0)' \
+  release-manifest.json)
+[ "$(wc -c < "$ARTIFACT" | tr -d '[:space:]')" -eq "$EXPECTED_ARTIFACT_SIZE" ] \
+  || { printf 'Release artifact size does not match the signed manifest\n' >&2; exit 1; }
 [ "$(jq '.sboms | length' release-manifest.json)" -eq 2 ] \
   || { printf 'Release manifest must bind two SBOMs\n' >&2; exit 1; }
 jq -e --arg version "$VERSION" \