From cea7048f6c293a342c6e4a9e10f45f0496b1ef73 Mon Sep 17 00:00:00 2001 From: Qiufeng Date: Tue, 18 Aug 2026 12:56:14 +0800 Subject: [PATCH] fix: make online updates observable and recoverable --- README.md | 13 +- .../update/api/SystemUpdateEventView.java | 11 + .../finance/update/api/SystemUpdateView.java | 6 + .../SystemUpdateApplicationService.java | 86 +++- .../SystemUpdateApplicationServiceTest.java | 75 +++ deploy/baota-init.sh | 2 +- deploy/systemd/kaidi-update.path | 2 + deploy/update.sh | 252 +++++++++- frontend/e2e/system-update.e2e.ts | 107 +++- frontend/src/api/system-update.ts | 14 + .../src/pages/governance/SystemUpdatePage.vue | 461 ++++++++++++++++-- frontend/src/utils/request/Axios.ts | 1 + openapi.yaml | 31 ++ scripts/package-release.sh | 5 +- scripts/test-install-fixture.sh | 6 +- scripts/test-update-fixture.sh | 129 ++++- scripts/verify-release.sh | 4 + 17 files changed, 1138 insertions(+), 67 deletions(-) create mode 100644 backend/src/main/java/com/kaidi/finance/update/api/SystemUpdateEventView.java diff --git a/README.md b/README.md index fa89ddc..e4693c5 100644 --- a/README.md +++ b/README.md @@ -264,19 +264,22 @@ sudo journalctl -u kaidi-update.service -n 100 --no-pager cat /var/lib/kaidi-update/status.json ``` -如果 `status.json` 显示 `FAILED` 且日志提示回滚未完成，失败请求会归档到 -`/var/lib/kaidi-update/failed`，活动事务保留在 `transactions/active`，系统不会自动重复安装。修复日志所示的 -磁盘、权限或旧版本健康问题后，才明确执行一次： +进程被中断时，`kaidi-update.path` 会根据 `processing/request.json` 或 `transactions/active` 自动恢复一次。 +如果 `status.json` 显示 `RECOVERY_REQUIRED`，失败请求会归档到 `/var/lib/kaidi-update/failed`，事务证据会移到 +`transactions/recovery-required*` 并退出自动触发路径，新下载和安装请求也会被拒绝。修复日志所示的磁盘、权限或 +旧版本健康问题后，将唯一一份待恢复事务移回 `active`，再明确执行一次： ```bash +sudo mv /var/lib/kaidi-update/transactions/recovery-required /var/lib/kaidi-update/transactions/active sudo systemctl reset-failed kaidi-update.service kaidi-update.path sudo systemctl start kaidi-update.service sudo journalctl -u kaidi-update.service -n 100 --no-pager cat /var/lib/kaidi-update/status.json ``` -只有状态恢复为 `SUCCEEDED`、`CURRENT` 或确定性的终态 `FAILED`，且 `transactions/active` 已处理完成后， -才算本次恢复结束。后台会保留真实失败状态，不会把待恢复的 processing 请求误显示成普通排队。 +如果目录带时间后缀，先通过 `ls -1d /var/lib/kaidi-update/transactions/recovery-required*` 确认唯一目录，再替换上面 +命令中的源路径。只有状态恢复为 `SUCCEEDED`、`CURRENT` 或确定性的终态 `FAILED`，且 `transactions/active` 已处理 +完成后，才算本次恢复结束。后台会保留真实失败状态，不会把待恢复请求误显示成普通排队。 ## 手工生成 Release diff --git a/backend/src/main/java/com/kaidi/finance/update/api/SystemUpdateEventView.java b/backend/src/main/java/com/kaidi/finance/update/api/SystemUpdateEventView.java new file mode 100644 index 0000000..16a20d3 --- /dev/null +++ b/backend/src/main/java/com/kaidi/finance/update/api/SystemUpdateEventView.java @@ -0,0 +1,11 @@ +package com.kaidi.finance.update.api; + +import java.time.Instant; + +public record SystemUpdateEventView( + Instant occurredAt, + String level, + String stage, + String message +) { +} diff --git a/backend/src/main/java/com/kaidi/finance/update/api/SystemUpdateView.java b/backend/src/main/java/com/kaidi/finance/update/api/SystemUpdateView.java index f5cc3d9..db9e23e 100644 --- a/backend/src/main/java/com/kaidi/finance/update/api/SystemUpdateView.java +++ b/backend/src/main/java/com/kaidi/finance/update/api/SystemUpdateView.java @@ -14,6 +14,12 @@ public record SystemUpdateView( Instant publishedAt, Instant checkedAt, Instant statusUpdatedAt, + Long downloadedBytes, + Long totalBytes, + Long bytesPerSecond, + Integer downloadPercent, + Integer restartExpectedSeconds, + List events, List allowedActions ) { } diff --git a/backend/src/main/java/com/kaidi/finance/update/application/SystemUpdateApplicationService.java b/backend/src/main/java/com/kaidi/finance/update/application/SystemUpdateApplicationService.java index f2a5d4a..8793403 100644 --- a/backend/src/main/java/com/kaidi/finance/update/application/SystemUpdateApplicationService.java +++ b/backend/src/main/java/com/kaidi/finance/update/application/SystemUpdateApplicationService.java @@ -11,6 +11,7 @@ import com.kaidi.finance.shared.security.AuthorizationService; import com.kaidi.finance.shared.security.IdentityContext; import com.kaidi.finance.update.api.SystemUpdateContracts.DownloadUpdateRequest; import com.kaidi.finance.update.api.SystemUpdateContracts.InstallUpdateRequest; +import com.kaidi.finance.update.api.SystemUpdateEventView; import com.kaidi.finance.update.api.SystemUpdateView; import java.io.IOException; import java.io.InputStream; @@ -44,6 +45,8 @@ public class SystemUpdateApplicationService { private static final int MAX_MANIFEST_BYTES = 64 * 1024; private static final int MAX_RELEASE_API_BYTES = 256 * 1024; private static final int MAX_STATUS_BYTES = 64 * 1024; + private static final int MAX_EVENT_LOG_BYTES = 256 * 1024; + private static final int MAX_EVENT_COUNT = 120; private static final int MAX_REDIRECTS = 3; private static final String PRERELEASE_IDENTIFIER = "(?:0|[1-9][0-9]*|[0-9]*[A-Za-z-][0-9A-Za-z-]*)"; @@ -57,8 +60,8 @@ public class SystemUpdateApplicationService { "^(.*/)?api/v1/repos/([A-Za-z0-9._-]+)/([A-Za-z0-9._-]+)/releases/(?:latest|tags/[^/?#]+)$"); private static final Pattern GITEA_RELEASE_TAG = Pattern.compile("^v[0-9A-Za-z][0-9A-Za-z._+-]{0,127}$"); private static final List BUSY_STATES = List.of( - "QUEUED", "DOWNLOAD_QUEUED", "INSTALL_QUEUED", "VERIFYING", "DOWNLOADING", "BACKING_UP", - "INSTALLING", "RUNNING"); + "QUEUED", "DOWNLOAD_QUEUED", "INSTALL_QUEUED", "PRECHECKING", "VERIFYING", "DOWNLOADING", + "BACKING_UP", "INSTALLING", "RUNNING"); private final SystemUpdateProperties properties; private final AuthorizationService authorizationService; @@ -353,6 +356,10 @@ public class SystemUpdateApplicationService { try (FileChannel channel = FileChannel.open(lockFile, StandardOpenOption.CREATE, StandardOpenOption.WRITE); FileLock ignored = channel.lock()) { UpdateStatus currentStatus = readStatus(); + if ("RECOVERY_REQUIRED".equals(currentStatus.state())) { + throw new BusinessException(HttpStatus.CONFLICT, ErrorCode.COMMAND_IN_PROGRESS, + "更新服务需要人工恢复，暂不接受新的更新请求"); + } if (BUSY_STATES.contains(currentStatus.state())) { throw new BusinessException(HttpStatus.CONFLICT, ErrorCode.COMMAND_IN_PROGRESS, "已有系统更新任务正在执行"); @@ -421,7 +428,10 @@ public class SystemUpdateApplicationService { String state = root.path("state").asText("UNKNOWN").toUpperCase(Locale.ROOT); if (!state.matches("^[A-Z_]{2,32}$")) state = "UNKNOWN"; return new UpdateStatus(state, root.path("message").asText(""), - blank(root.path("targetVersion").asText(null)), parseInstant(root.path("updatedAt").asText(null))); + blank(root.path("targetVersion").asText(null)), parseInstant(root.path("updatedAt").asText(null)), + nonNegativeLong(root, "downloadedBytes"), nonNegativeLong(root, "totalBytes"), + nonNegativeLong(root, "bytesPerSecond"), boundedInteger(root, "downloadPercent", 0, 100), + boundedInteger(root, "restartExpectedSeconds", 0, 300)); } catch (IOException exception) { return new UpdateStatus("UNKNOWN", "更新状态读取失败", null, null); } @@ -433,17 +443,60 @@ public class SystemUpdateApplicationService { String candidateVersion = manifest == null ? status.targetVersion() : manifest.version(); boolean available = candidateVersion != null && compareVersions(candidateVersion, current) > 0; boolean busy = BUSY_STATES.contains(status.state()); + boolean recoveryRequired = "RECOVERY_REQUIRED".equals(status.state()); boolean pending = hasPendingRequest(properties.requestFile().toAbsolutePath().normalize()); boolean ready = "READY".equals(status.state()) && candidateVersion != null && candidateVersion.equals(status.targetVersion()); List actions = new ArrayList<>(); - if (enabled && !busy && !pending) actions.add("CHECK"); - if (enabled && available && !busy && !pending && authorizationService.hasPermission("admin:update:execute")) { + if (enabled && !busy && !pending && !recoveryRequired) actions.add("CHECK"); + if (enabled && available && !busy && !pending && !recoveryRequired + && authorizationService.hasPermission("admin:update:execute")) { actions.add(ready ? "INSTALL" : "DOWNLOAD"); } return new SystemUpdateView(enabled, current, candidateVersion, available, status.state(), status.message(), manifest == null ? null : manifest.releaseNotes(), - manifest == null ? null : manifest.publishedAt(), lastCheckedAt, status.updatedAt(), List.copyOf(actions)); + manifest == null ? null : manifest.publishedAt(), lastCheckedAt, status.updatedAt(), + status.downloadedBytes(), status.totalBytes(), status.bytesPerSecond(), status.downloadPercent(), + status.restartExpectedSeconds(), readUpdateEvents(), List.copyOf(actions)); + } + + private List readUpdateEvents() { + Path statusFile = properties.statusFile().toAbsolutePath().normalize(); + Path parent = statusFile.getParent(); + if (parent == null) return List.of(); + Path eventFile = parent.resolve("events.jsonl").normalize(); + if (!eventFile.startsWith(parent) || !Files.isRegularFile(eventFile, LinkOption.NOFOLLOW_LINKS) + || Files.isSymbolicLink(eventFile)) { + return List.of(); + } + try { + if (Files.size(eventFile) > MAX_EVENT_LOG_BYTES) return List.of(); + List lines = Files.readAllLines(eventFile, StandardCharsets.UTF_8); + int first = Math.max(0, lines.size() - MAX_EVENT_COUNT); + List events = new ArrayList<>(); + for (int index = first; index < lines.size(); index++) { + String line = lines.get(index); + if (line.isBlank() || line.length() > 4096) continue; + try { + JsonNode event = objectMapper.readTree(line); + Instant occurredAt = parseInstant(event.path("occurredAt").asText(null)); + String level = event.path("level").asText("INFO").toUpperCase(Locale.ROOT); + String stage = event.path("stage").asText("UNKNOWN").toUpperCase(Locale.ROOT); + String message = event.path("message").asText(""); + if (occurredAt == null || !level.matches("^(INFO|WARN|ERROR)$") + || !stage.matches("^[A-Z_]{2,32}$") || message.isBlank()) { + continue; + } + events.add(new SystemUpdateEventView(occurredAt, level, stage, + message.length() > 1000 ? message.substring(0, 1000) : message)); + } catch (IOException ignored) { + // Ignore a partially written event line. + } + } + return List.copyOf(events); + } catch (IOException exception) { + return List.of(); + } } private boolean hasPendingRequest(Path requestFile) { @@ -564,6 +617,20 @@ public class SystemUpdateApplicationService { } } + private static Long nonNegativeLong(JsonNode root, String field) { + JsonNode value = root.path(field); + if (!value.canConvertToLong()) return null; + long parsed = value.asLong(); + return parsed < 0 ? null : parsed; + } + + private static Integer boundedInteger(JsonNode root, String field, int minimum, int maximum) { + JsonNode value = root.path(field); + if (!value.canConvertToInt()) return null; + int parsed = value.asInt(); + return parsed < minimum || parsed > maximum ? null : parsed; + } + private BusinessException validation(String message) { return new BusinessException(HttpStatus.UNPROCESSABLE_ENTITY, ErrorCode.VALIDATION_FAILED, message); } @@ -580,7 +647,12 @@ public class SystemUpdateApplicationService { String releaseNotes) { } - private record UpdateStatus(String state, String message, String targetVersion, Instant updatedAt) { + private record UpdateStatus(String state, String message, String targetVersion, Instant updatedAt, + Long downloadedBytes, Long totalBytes, Long bytesPerSecond, + Integer downloadPercent, Integer restartExpectedSeconds) { + private UpdateStatus(String state, String message, String targetVersion, Instant updatedAt) { + this(state, message, targetVersion, updatedAt, null, null, null, null, null); + } } private record QueueTransition(UpdateStatus previous, UpdateStatus queued) { diff --git a/backend/src/test/java/com/kaidi/finance/update/application/SystemUpdateApplicationServiceTest.java b/backend/src/test/java/com/kaidi/finance/update/application/SystemUpdateApplicationServiceTest.java index 7cfc300..1fadf2a 100644 --- a/backend/src/test/java/com/kaidi/finance/update/application/SystemUpdateApplicationServiceTest.java +++ b/backend/src/test/java/com/kaidi/finance/update/application/SystemUpdateApplicationServiceTest.java @@ -211,6 +211,81 @@ class SystemUpdateApplicationServiceTest { } } + @Test + void restoresDownloadMetricsAndRuntimeEventsFromUpdaterState() throws Exception { + HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + Path inbox = Files.createDirectory(tempDir.resolve("progress-inbox")); + Path statusFile = tempDir.resolve("progress-status.json"); + Files.writeString(statusFile, """ + {"state":"DOWNLOADING","message":"Downloading signed release","targetVersion":"1.0.0-preview.2", + "updatedAt":"2026-08-18T03:00:00Z","downloadedBytes":5242880,"totalBytes":10485760, + "bytesPerSecond":1048576,"downloadPercent":50,"restartExpectedSeconds":10} + """); + Files.writeString(tempDir.resolve("events.jsonl"), """ + {"occurredAt":"2026-08-18T02:59:59Z","level":"INFO","stage":"VERIFYING","message":"签名校验开始"} + {"occurredAt":"2026-08-18T03:00:00Z","level":"INFO","stage":"DOWNLOADING","message":"更新包下载中"} + """); + SystemUpdateApplicationService service = serviceForGitea(server, inbox, statusFile); + + var status = service.status(); + + assertEquals("DOWNLOADING", status.state()); + assertEquals(5_242_880L, status.downloadedBytes()); + assertEquals(10_485_760L, status.totalBytes()); + assertEquals(1_048_576L, status.bytesPerSecond()); + assertEquals(50, status.downloadPercent()); + assertEquals(10, status.restartExpectedSeconds()); + assertEquals(2, status.events().size()); + assertEquals("DOWNLOADING", status.events().get(1).stage()); + assertFalse(status.allowedActions().contains("DOWNLOAD")); + } + + @Test + void recoveryRequiredStateBlocksEveryUpdateAction() throws Exception { + HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext("/api/v1/repos/ERP-Team/kaidi/releases/latest", exchange -> { + String assetUrl = "http://127.0.0.1:" + server.getAddress().getPort() + + "/ERP-Team/kaidi/releases/download/v1.0.0-preview.2/release-manifest.json"; + byte[] body = """ + {"tag_name":"v1.0.0-preview.2","assets":[ + {"name":"release-manifest.json","browser_download_url":"%s"}]} + """.formatted(assetUrl).getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(200, body.length); + exchange.getResponseBody().write(body); + exchange.close(); + }); + server.createContext("/ERP-Team/kaidi/releases/download/v1.0.0-preview.2/release-manifest.json", + exchange -> { + byte[] body = """ + {"version":"1.0.0-preview.2","artifact":"kaidi-finance-1.0.0-preview.2.tar.gz", + "sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"} + """.getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(200, body.length); + exchange.getResponseBody().write(body); + exchange.close(); + }); + server.start(); + Path inbox = Files.createDirectory(tempDir.resolve("recovery-inbox")); + Path statusFile = tempDir.resolve("recovery-status.json"); + Files.writeString(statusFile, """ + {"state":"RECOVERY_REQUIRED","message":"自动回滚未完成","targetVersion":"1.0.0-preview.2", + "updatedAt":"2026-08-18T03:10:00Z"} + """); + SystemUpdateApplicationService service = serviceForGitea(server, inbox, statusFile); + + try { + var status = service.status(); + + assertEquals("RECOVERY_REQUIRED", status.state()); + assertTrue(status.allowedActions().isEmpty()); + assertThrows(BusinessException.class, + () -> service.download(new DownloadUpdateRequest("1.0.0-preview.2"))); + assertFalse(Files.exists(inbox.resolve("request.json"))); + } finally { + server.stop(0); + } + } + @Test void checksConfiguredManifestAndQueuesOnlyItsLatestVersion() throws Exception { HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); diff --git a/deploy/baota-init.sh b/deploy/baota-init.sh index faf2475..0dc342b 100755 --- a/deploy/baota-init.sh +++ b/deploy/baota-init.sh @@ -95,7 +95,7 @@ main() { [ "$(uname -s)" = Linux ] || die "Baota initialization only supports Linux" command -v systemctl >/dev/null 2>&1 && [ -d /run/systemd/system ] \ || die "systemd is required for the privileged background updater" - for command in find openssl sha256sum; do + for command in find openssl setsid sha256sum; do command -v "$command" >/dev/null 2>&1 || die "$command is required" done diff --git a/deploy/systemd/kaidi-update.path b/deploy/systemd/kaidi-update.path index de57bdf..963644f 100644 --- a/deploy/systemd/kaidi-update.path +++ b/deploy/systemd/kaidi-update.path @@ -4,6 +4,8 @@ Description=Watch for Kaidi Finance update requests [Path] PathChanged=/var/lib/kaidi-update/inbox PathExists=/var/lib/kaidi-update/inbox/request.json +PathExists=/var/lib/kaidi-update/processing/request.json +PathExists=/var/lib/kaidi-update/transactions/active Unit=kaidi-update.service [Install] diff --git a/deploy/update.sh b/deploy/update.sh index aa29063..6d7952b 100755 --- a/deploy/update.sh +++ b/deploy/update.sh @@ -12,6 +12,7 @@ FAILED_REQUEST_ROOT=${KAIDI_UPDATE_FAILED_ROOT:-$STATE_ROOT/failed} TRANSACTION_ROOT=${KAIDI_UPDATE_TRANSACTION_ROOT:-$STATE_ROOT/transactions} ACTIVE_TRANSACTION=$TRANSACTION_ROOT/active STATUS_FILE=${UPDATE_STATUS_FILE:-$STATE_ROOT/status.json} +EVENT_LOG_FILE=${UPDATE_EVENT_LOG_FILE:-$STATE_ROOT/events.jsonl} PUBLIC_KEY=${UPDATE_PUBLIC_KEY:-/etc/kaidi/release-public.pem} RELEASE_BASE_URL=${UPDATE_RELEASE_BASE_URL:-} RELEASE_API_URL=${UPDATE_RELEASE_API_URL:-} @@ -39,6 +40,10 @@ CACHE_TEMP= RELEASE_AUTH_HEADER_FILE= TARGET_VERSION= TERMINAL_STATUS_WRITTEN=false +DOWNLOAD_PID= +DOWNLOAD_PGID= +LAST_DOWNLOAD_SPEED=0 +SURFACE_FAILURE= case "$HEALTH_ATTEMPTS:$HEALTH_INTERVAL_SECONDS" in *[!0-9:]* | :* | *:) printf '%s\n' "Update health-check settings must be non-negative integers" >&2; exit 1 ;; @@ -95,7 +100,8 @@ load_runtime_database_env() { runtime_size=$(wc -c < "$RUNTIME_ENV_FILE" | tr -d '[:space:]') [ "$runtime_size" -le 65536 ] \ || bootstrap_die "Setup runtime environment is too large" - runtime_owner=$(stat -c '%u' "$RUNTIME_ENV_FILE" 2>/dev/null || true) + runtime_owner=$(stat -c '%u' "$RUNTIME_ENV_FILE" 2>/dev/null \ + || stat -f '%u' "$RUNTIME_ENV_FILE" 2>/dev/null || true) service_uid=$(id -u "$SERVICE_USER" 2>/dev/null || true) [ "$runtime_owner" = 0 ] || [ "$runtime_owner" = "$service_uid" ] \ || bootstrap_die "Setup runtime environment has an unexpected owner" @@ -116,6 +122,7 @@ load_runtime_database_env() { KAIDI_DB_NAME) KAIDI_DB_NAME=$runtime_value; export KAIDI_DB_NAME ;; KAIDI_DB_USERNAME) KAIDI_DB_USERNAME=$runtime_value; export KAIDI_DB_USERNAME ;; KAIDI_DB_PASSWORD) KAIDI_DB_PASSWORD=$runtime_value; export KAIDI_DB_PASSWORD ;; + DB_URL) KAIDI_DB_URL=$runtime_value; export KAIDI_DB_URL ;; esac done < "$RUNTIME_ENV_FILE" @@ -126,20 +133,84 @@ load_runtime_database_env() { || bootstrap_die "Setup runtime database port is invalid" } +validate_runtime_database_config() { + [ -e "$RUNTIME_ENV_FILE" ] || return 0 + [ -n "${KAIDI_DB_URL:-}" ] \ + || fail "Database configuration is invalid: DB_URL is missing; application was not restarted" + [ -n "${KAIDI_DB_HOST:-}" ] && [ -n "${KAIDI_DB_NAME:-}" ] \ + || fail "Database configuration is invalid: database host and name are missing; application was not restarted" + expected_url_prefix="jdbc:mysql://${KAIDI_DB_HOST}:${KAIDI_DB_PORT}/${KAIDI_DB_NAME}" + case "$KAIDI_DB_URL" in + "$expected_url_prefix"|"$expected_url_prefix"\?*) ;; + *) fail "Database configuration is invalid: DB_URL does not match the configured MySQL host, port, and database; application was not restarted" ;; + esac +} + status() { state=$1 message=$2 version=${3:-} + downloaded_bytes=${4:-} + total_bytes=${5:-} + bytes_per_second=${6:-} + download_percent=${7:-} + restart_expected_seconds=${8:-} + previous_state= + previous_message= + if [ -f "$STATUS_FILE" ] && [ ! -L "$STATUS_FILE" ]; then + previous_state=$(jq -r '.state // empty' "$STATUS_FILE" 2>/dev/null || true) + previous_message=$(jq -r '.message // empty' "$STATUS_FILE" 2>/dev/null || true) + fi tmp="$STATUS_FILE.tmp.$$" jq -n \ --arg state "$state" \ --arg message "$message" \ --arg version "$version" \ + --arg downloadedBytes "$downloaded_bytes" \ + --arg totalBytes "$total_bytes" \ + --arg bytesPerSecond "$bytes_per_second" \ + --arg downloadPercent "$download_percent" \ + --arg restartExpectedSeconds "$restart_expected_seconds" \ --arg updatedAt "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ '{state:$state,message:$message,updatedAt:$updatedAt} - + (if ($version | length) > 0 then {targetVersion:$version} else {} end)' > "$tmp" + + (if ($version | length) > 0 then {targetVersion:$version} else {} end) + + (if ($downloadedBytes | test("^[0-9]+$")) then {downloadedBytes:($downloadedBytes | tonumber)} else {} end) + + (if ($totalBytes | test("^[0-9]+$")) then {totalBytes:($totalBytes | tonumber)} else {} end) + + (if ($bytesPerSecond | test("^[0-9]+$")) then {bytesPerSecond:($bytesPerSecond | tonumber)} else {} end) + + (if ($downloadPercent | test("^[0-9]+$")) then {downloadPercent:($downloadPercent | tonumber)} else {} end) + + (if ($restartExpectedSeconds | test("^[0-9]+$")) + then {restartExpectedSeconds:($restartExpectedSeconds | tonumber)} else {} end)' > "$tmp" chmod 0644 "$tmp" mv -f "$tmp" "$STATUS_FILE" + if [ "$state" != "$previous_state" ] || [ "$message" != "$previous_message" ]; then + case "$state" in + FAILED|RECOVERY_REQUIRED) event ERROR "$state" "$message" || true ;; + *) event INFO "$state" "$message" || true ;; + esac + fi +} + +event() { + level=$1 + stage=$2 + message=$3 + event_tmp="$EVENT_LOG_FILE.tmp.$$" + jq -cn --arg occurredAt "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + --arg level "$level" --arg stage "$stage" --arg message "$message" \ + '{occurredAt:$occurredAt,level:$level,stage:$stage,message:$message}' >> "$EVENT_LOG_FILE" + chmod 0644 "$EVENT_LOG_FILE" + event_size=$(wc -c < "$EVENT_LOG_FILE" | tr -d '[:space:]') + if [ "$event_size" -gt 262144 ]; then + tail -n 160 "$EVENT_LOG_FILE" > "$event_tmp" + chmod 0644 "$event_tmp" + mv -f "$event_tmp" "$EVENT_LOG_FILE" + fi +} + +reset_event_log() { + rm -f "$EVENT_LOG_FILE" + touch "$EVENT_LOG_FILE" + chmod 0644 "$EVENT_LOG_FILE" } archive_processing_request() { @@ -166,6 +237,7 @@ prepare_update_layout() { id -nG "$SERVICE_USER" | tr ' ' '\n' | grep -Fxq "$SERVICE_GROUP" \ || bootstrap_die "Service user $SERVICE_USER is not a member of group $SERVICE_GROUP" command -v runuser >/dev/null 2>&1 || bootstrap_die "runuser is required" + command -v setsid >/dev/null 2>&1 || bootstrap_die "setsid is required" install -d -o root -g "$SERVICE_GROUP" -m 0750 \ "$APP_ROOT" "$APP_ROOT/releases" "$APP_ROOT/runtime" "$APP_ROOT/bin" "$STATE_ROOT" \ || bootstrap_die "Managed application or update directories could not be prepared" @@ -213,6 +285,14 @@ verify_release_access() { cleanup() { rc=$? trap - EXIT HUP INT TERM + if [ -n "$DOWNLOAD_PID" ]; then + if [ -n "$DOWNLOAD_PGID" ]; then + kill -TERM -- "-$DOWNLOAD_PGID" 2>/dev/null || true + else + kill -TERM "$DOWNLOAD_PID" 2>/dev/null || true + fi + wait "$DOWNLOAD_PID" 2>/dev/null || true + fi [ -z "$WORK_DIR" ] || rm -rf "$WORK_DIR" [ -z "$CACHE_TEMP" ] || rm -rf "$CACHE_TEMP" if [ "$rc" -ne 0 ] && [ "$TERMINAL_STATUS_WRITTEN" != true ]; then @@ -253,6 +333,83 @@ download() { esac } +download_with_progress() { + progress_url=$1 + progress_output=$2 + progress_total=${3:-0} + progress_started=$(date +%s) + progress_previous_time=$progress_started + progress_previous_bytes=0 + + rm -f "$progress_output" + touch "$progress_output" + chmod 0600 "$progress_output" + case "$progress_url" in + https://*) + if [ -n "$RELEASE_TOKEN" ]; then + [ -s "$RELEASE_AUTH_HEADER_FILE" ] || return 2 + setsid curl --fail --silent --show-error --proto '=https' --tlsv1.2 \ + --header "@$RELEASE_AUTH_HEADER_FILE" "$progress_url" -o "$progress_output" & + else + setsid curl --fail --silent --show-error --location --proto '=https' --proto-redir '=https' \ + --tlsv1.2 "$progress_url" -o "$progress_output" & + fi + ;; + *) + [ "${KAIDI_ALLOW_INSECURE_UPDATE:-false}" = "true" ] || return 2 + if [ -n "$RELEASE_TOKEN" ]; then + [ -s "$RELEASE_AUTH_HEADER_FILE" ] || return 2 + setsid curl --fail --silent --show-error --proto '=http,https' \ + --header "@$RELEASE_AUTH_HEADER_FILE" "$progress_url" -o "$progress_output" & + else + setsid curl --fail --silent --show-error --location --proto '=http,https' \ + --proto-redir '=http,https' "$progress_url" -o "$progress_output" & + fi + ;; + esac + DOWNLOAD_PID=$! + DOWNLOAD_PGID=$DOWNLOAD_PID + while kill -0 "$DOWNLOAD_PID" 2>/dev/null; do + progress_now=$(date +%s) + progress_bytes=$(wc -c < "$progress_output" 2>/dev/null | tr -d '[:space:]' || printf 0) + case "$progress_bytes:$progress_total" in *[!0-9:]*) progress_bytes=0; progress_total=0 ;; esac + progress_delta_seconds=$((progress_now - progress_previous_time)) + [ "$progress_delta_seconds" -ge 1 ] || progress_delta_seconds=1 + progress_delta_bytes=$((progress_bytes - progress_previous_bytes)) + [ "$progress_delta_bytes" -ge 0 ] || progress_delta_bytes=0 + progress_speed=$((progress_delta_bytes / progress_delta_seconds)) + if [ "$progress_total" -gt 0 ]; then + progress_percent=$((progress_bytes * 100 / progress_total)) + [ "$progress_percent" -le 99 ] || progress_percent=99 + else + progress_percent=0 + fi + status DOWNLOADING "Downloading signed release $TARGET_VERSION" "$TARGET_VERSION" \ + "$progress_bytes" "$progress_total" "$progress_speed" "$progress_percent" + progress_previous_time=$progress_now + progress_previous_bytes=$progress_bytes + sleep 1 + done + if wait "$DOWNLOAD_PID"; then + DOWNLOAD_PID= + DOWNLOAD_PGID= + else + DOWNLOAD_PID= + DOWNLOAD_PGID= + return 1 + fi + + progress_bytes=$(wc -c < "$progress_output" | tr -d '[:space:]') + [ "$progress_total" -gt 0 ] || progress_total=$progress_bytes + progress_finished=$(date +%s) + progress_elapsed=$((progress_finished - progress_started)) + [ "$progress_elapsed" -ge 1 ] || progress_elapsed=1 + progress_speed=$((progress_bytes / progress_elapsed)) + LAST_DOWNLOAD_SPEED=$progress_speed + status DOWNLOADING "Release $TARGET_VERSION download completed" "$TARGET_VERSION" \ + "$progress_bytes" "$progress_total" "$progress_speed" 100 +} + release_asset_url() { asset_name=$1 if [ -n "$RELEASE_API_URL" ]; then @@ -284,30 +441,56 @@ EOF wait_for_health() { health_url=$1 attempts=0 + health_file=$(mktemp "$STATE_ROOT/work/health-response.XXXXXX") while [ "$attempts" -lt "$HEALTH_ATTEMPTS" ]; do - if curl -fsS "$health_url" | jq -e '.status == "UP"' >/dev/null 2>&1; then + health_http=$(curl --silent --show-error --connect-timeout 2 --max-time 4 \ + --output "$health_file" --write-out '%{http_code}' "$health_url" 2>/dev/null || true) + if [ "$health_http" = 200 ] && jq -e '.status == "UP"' "$health_file" >/dev/null 2>&1; then + rm -f "$health_file" return 0 fi + if [ "$health_http" = 200 ]; then + SURFACE_FAILURE=HEALTH_DOWN + elif [ -n "$health_http" ] && [ "$health_http" != 000 ]; then + SURFACE_FAILURE="HEALTH_HTTP_$health_http" + else + SURFACE_FAILURE=HEALTH_UNREACHABLE + fi attempts=$((attempts + 1)) sleep "$HEALTH_INTERVAL_SECONDS" done + rm -f "$health_file" return 1 } verify_app_surface() { expected_release=${1:-} + SURFACE_FAILURE= wait_for_health "$HEALTH_URL" || return 1 index_file=$(mktemp "$STATE_ROOT/work/public-index.XXXXXX") - if curl -fsS "$APP_INDEX_URL" -o "$index_file" \ + index_http=$(curl --silent --show-error --connect-timeout 2 --max-time 5 \ + --output "$index_file" --write-out '%{http_code}' "$APP_INDEX_URL" 2>/dev/null || true) + if [ "$index_http" = 200 ] \ && grep -Eiq '&2 + recovery_guard_failed=true fi + if ! quarantine_active_transaction; then + printf '%s\n' "Update transaction evidence could not be quarantined after an incomplete rollback" >&2 + recovery_guard_failed=true + fi + if [ "$recovery_guard_failed" = true ]; then + systemctl stop "$UPDATE_PATH_NAME" >/dev/null 2>&1 || true + fi + status RECOVERY_REQUIRED "$reason; rollback is incomplete and manual recovery is required" "${TARGET_VERSION:-}" printf '%s\n' "$reason; rollback is incomplete and manual recovery is required" >&2 exit 1 } @@ -651,6 +852,16 @@ else fi chmod 0600 "$PROCESSING_FILE" +if [ -f "$STATUS_FILE" ] && [ ! -L "$STATUS_FILE" ] \ + && jq -e '.state == "RECOVERY_REQUIRED"' "$STATUS_FILE" >/dev/null 2>&1; then + TERMINAL_STATUS_WRITTEN=true + if ! archive_processing_request; then + systemctl stop "$UPDATE_PATH_NAME" >/dev/null 2>&1 || true + fi + printf '%s\n' "Update service is locked for manual recovery; new requests are rejected" >&2 + exit 1 +fi + if ! REQUESTED_VERSION=$(jq -er '.version | strings | select(length > 0 and length <= 128)' \ "$PROCESSING_FILE"); then fail "Update request version is invalid" @@ -661,7 +872,11 @@ REQUEST_ACTION=$(jq -er '(.action // "INSTALL") | strings | ascii_upcase | select(. == "DOWNLOAD" or . == "INSTALL")' "$PROCESSING_FILE") \ || fail "Update request action is invalid" prepare_update_layout +reset_event_log load_runtime_database_env +if [ "$REQUEST_ACTION" = INSTALL ]; then + validate_runtime_database_config +fi [ -z "$RELEASE_TOKEN" ] || { [ "${#RELEASE_TOKEN}" -le 512 ] \ && ! printf '%s' "$RELEASE_TOKEN" | grep -q '[[:cntrl:]]'; } \ || fail "UPDATE_RELEASE_TOKEN is invalid" @@ -714,6 +929,8 @@ ARTIFACT=$(jq -er '.artifact | strings | select(test("^[A-Za-z0-9][A-Za-z0-9._+- "$WORK_DIR/release-manifest.json") || fail "Release artifact name is invalid" EXPECTED_SHA=$(jq -er '.sha256 | strings | ascii_downcase | select(test("^[0-9a-f]{64}$"))' \ "$WORK_DIR/release-manifest.json") || fail "Release SHA-256 is invalid" +EXPECTED_SIZE=$(jq -er '(.artifactSizeBytes // 0) | numbers | floor | select(. >= 0)' \ + "$WORK_DIR/release-manifest.json") || fail "Release artifact size is invalid" CURRENT_VERSION=$(cat "$APP_ROOT/current/VERSION" 2>/dev/null || true) if [ "$CURRENT_VERSION" = "$TARGET_VERSION" ]; then @@ -726,12 +943,16 @@ fi if [ "$REQUEST_ACTION" = DOWNLOAD ]; then status DOWNLOADING "Downloading signed release $TARGET_VERSION" "$TARGET_VERSION" ARTIFACT_URL=$(release_asset_url "$ARTIFACT") || fail "Release artifact asset is missing" - download "$ARTIFACT_URL" "$WORK_DIR/release.tar.gz" || fail "Release artifact download failed" + download_with_progress "$ARTIFACT_URL" "$WORK_DIR/release.tar.gz" "$EXPECTED_SIZE" \ + || fail "Release artifact download failed" else status VERIFYING "Revalidating cached release $TARGET_VERSION" "$TARGET_VERSION" fi ACTUAL_SHA=$(sha256sum "$WORK_DIR/release.tar.gz" | awk '{print $1}') [ "$ACTUAL_SHA" = "$EXPECTED_SHA" ] || fail "Release artifact SHA-256 verification failed" +ACTUAL_SIZE=$(wc -c < "$WORK_DIR/release.tar.gz" | tr -d '[:space:]') +[ "$EXPECTED_SIZE" -eq 0 ] || [ "$ACTUAL_SIZE" -eq "$EXPECTED_SIZE" ] \ + || fail "Release artifact size verification failed" tar -tzf "$WORK_DIR/release.tar.gz" > "$WORK_DIR/archive.list" || fail "Release archive could not be listed" if grep -Eq '(^/|(^|/)\.\.(/|$))' "$WORK_DIR/archive.list"; then @@ -768,7 +989,8 @@ if [ "$REQUEST_ACTION" = DOWNLOAD ]; then fi mv "$CACHE_TEMP" "$CACHE_DIR" || fail "Verified release cache could not be activated" CACHE_TEMP= - status READY "Release $TARGET_VERSION is downloaded and verified; confirm installation" "$TARGET_VERSION" + status READY "Release $TARGET_VERSION is downloaded and verified; confirm installation" "$TARGET_VERSION" \ + "$ACTUAL_SIZE" "$ACTUAL_SIZE" "$LAST_DOWNLOAD_SPEED" 100 TERMINAL_STATUS_WRITTEN=true complete_request exit 0 @@ -791,6 +1013,13 @@ if ! verify_release_access "$RELEASE_DIR"; then fi PREVIOUS_TARGET=$(readlink "$APP_ROOT/current" 2>/dev/null || true) +status PRECHECKING "Validating current release before switching to $TARGET_VERSION" "$TARGET_VERSION" +if [ -z "$PREVIOUS_TARGET" ] || [ ! -d "$PREVIOUS_TARGET" ]; then + fail "Current release path is invalid; application was not restarted" +fi +if ! verify_app_surface "$PREVIOUS_TARGET"; then + fail "Current release preflight failed (${SURFACE_FAILURE:-UNKNOWN}); application was not restarted" +fi mkdir "$ACTIVE_TRANSACTION" write_transaction_value target-version "$TARGET_VERSION" write_transaction_value previous-target "$PREVIOUS_TARGET" @@ -817,7 +1046,7 @@ if ! ln -sfn "$RELEASE_DIR" "$APP_ROOT/current.next" \ fi write_transaction_value phase APP_SWITCHED -status RUNNING "Starting and verifying release $TARGET_VERSION" "$TARGET_VERSION" +status RUNNING "Starting and verifying release $TARGET_VERSION" "$TARGET_VERSION" "" "" "" "" 10 write_transaction_value phase HEALTH_CHECKING if ! restart_application; then rollback_active_transaction "Release application restart failed" @@ -837,5 +1066,6 @@ fi service_state=$(application_state 2>/dev/null || true) [ -n "$service_state" ] || service_state=unknown +surface_failure=${SURFACE_FAILURE:-UNKNOWN} rollback_active_transaction \ - "Release health or application-surface verification failed (service state: $service_state)" + "Release application verification failed ($surface_failure; service state: $service_state)" diff --git a/frontend/e2e/system-update.e2e.ts b/frontend/e2e/system-update.e2e.ts index c6046b9..c519165 100644 --- a/frontend/e2e/system-update.e2e.ts +++ b/frontend/e2e/system-update.e2e.ts @@ -28,6 +28,7 @@ function updateView(state: string, checked: boolean, enabled = true, recoveryPen 'QUEUED', 'DOWNLOAD_QUEUED', 'INSTALL_QUEUED', + 'PRECHECKING', 'VERIFYING', 'DOWNLOADING', 'BACKING_UP', @@ -59,6 +60,14 @@ function updateView(state: string, checked: boolean, enabled = true, recoveryPen publishedAt: checked ? '2026-08-16T00:00:00Z' : null, checkedAt: checked ? '2026-08-16T00:01:00Z' : null, statusUpdatedAt: null, + downloadedBytes: state === 'READY' ? 10_485_760 : state === 'DOWNLOADING' ? 5_242_880 : 0, + totalBytes: ['DOWNLOAD_QUEUED', 'DOWNLOADING', 'VERIFYING', 'READY'].includes(state) ? 10_485_760 : null, + bytesPerSecond: state === 'DOWNLOADING' ? 1_048_576 : 0, + downloadPercent: state === 'READY' ? 100 : state === 'DOWNLOADING' ? 50 : 0, + restartExpectedSeconds: state === 'RUNNING' ? 10 : null, + events: checked + ? [{ occurredAt: '2026-08-16T00:01:00Z', level: 'INFO', stage: state, message: `状态进入 ${state}` }] + : [], allowedActions: !enabled || recoveryPending || busy ? [] @@ -72,8 +81,11 @@ function updateView(state: string, checked: boolean, enabled = true, recoveryPen interface FixtureOptions { canExecute?: boolean; + checkResponseState?: string; + downloadProgressReads?: number; failDownloadResponse?: boolean; failInstallResponse?: boolean; + restartOfflineReads?: number; } async function installFixture( @@ -87,9 +99,18 @@ async function installFixture( completeBusyAfterFirstRead = false, options: FixtureOptions = {}, ) { - const { canExecute = true, failDownloadResponse = false, failInstallResponse = false } = options; + const { + canExecute = true, + checkResponseState, + downloadProgressReads = 0, + failDownloadResponse = false, + failInstallResponse = false, + restartOfflineReads = 0, + } = options; let checked = recoveryPending || Boolean(initialState); - let state = recoveryPending ? 'FAILED' : initialState || 'IDLE'; + let state = recoveryPending ? 'RECOVERY_REQUIRED' : initialState || 'IDLE'; + let progressDeadline = 0; + let restartDeadline = 0; const history: Array> = []; const recordHistory = (actionCode: string, targetVersion = '1.0.0-preview.2') => { history.unshift({ @@ -131,13 +152,19 @@ async function installFixture( }); } if (pathname === '/api/v1/admin/system-update' && request.method() === 'GET') { + if (state === 'RUNNING' && Date.now() < restartDeadline) { + return route.abort('connectionrefused'); + } + if (state === 'RUNNING' && restartDeadline > 0) state = 'SUCCEEDED'; const response = updateView(state, checked, enabled, recoveryPending); + if (state === 'DOWNLOADING' && Date.now() >= progressDeadline) state = 'READY'; if (completeBusyAfterFirstRead && state === 'INSTALLING') state = 'SUCCEEDED'; return route.fulfill({ json: envelope(response) }); } if (pathname === '/api/v1/admin/system-update/check' && request.method() === 'POST') { checked = true; recordHistory('SYSTEM_UPDATE_CHECK'); + if (checkResponseState) state = checkResponseState; const response = updateView(state, checked, enabled, recoveryPending); if (!canExecute) response.allowedActions = ['CHECK']; return route.fulfill({ json: envelope(response) }); @@ -145,7 +172,8 @@ async function installFixture( if (pathname === '/api/v1/admin/system-update/download' && request.method() === 'POST') { downloadBodies.push(request.postDataJSON()); recordHistory('SYSTEM_UPDATE_DOWNLOAD_REQUEST'); - state = 'READY'; + state = downloadProgressReads > 0 ? 'DOWNLOADING' : 'READY'; + progressDeadline = Date.now() + downloadProgressReads * 1000; if (failDownloadResponse) return route.abort('connectionreset'); return route.fulfill({ json: envelope(updateView('DOWNLOAD_QUEUED', true, enabled)) }); } @@ -153,7 +181,13 @@ async function installFixture( installBodies.push(request.postDataJSON()); recordHistory('SYSTEM_UPDATE_REQUEST'); const queued = updateView('INSTALL_QUEUED', true, enabled); - state = failInstallResponse || completeAfterInstall ? 'SUCCEEDED' : 'INSTALL_QUEUED'; + restartDeadline = Date.now() + restartOfflineReads * 1000; + state = + restartOfflineReads > 0 + ? 'RUNNING' + : failInstallResponse || completeAfterInstall + ? 'SUCCEEDED' + : 'INSTALL_QUEUED'; if (failInstallResponse) return route.abort('connectionreset'); return route.fulfill({ json: envelope(queued) }); } @@ -189,6 +223,7 @@ test('system administrator checks, downloads, and explicitly installs a signed o expect(downloadBodies).toEqual([{ version: '1.0.0-preview.2' }]); const updateState = page.locator('.version-item').filter({ hasText: '更新状态' }); await expect(updateState.getByText('下载已排队', { exact: true })).toBeVisible(); + await expect(page.getByText('已发现新版本，但当前账号没有下载权限', { exact: true })).toHaveCount(0); await expect(updateState.getByText('下载完成', { exact: true })).toBeVisible({ timeout: 8_000 }); await expect(page.getByRole('button', { name: '立即更新并重启', exact: true })).toBeVisible(); @@ -198,6 +233,38 @@ test('system administrator checks, downloads, and explicitly installs a signed o expect(installBodies).toEqual([{ version: '1.0.0-preview.2', reason: '管理员确认立即更新并重启' }]); }); +test('download dialog shows real byte progress and transfer speed without a false permission warning', async ({ + page, +}) => { + await installFixture(page, [], [], true, false, false, undefined, false, { downloadProgressReads: 8 }); + await page.goto('/governance/update'); + + await page.getByRole('button', { name: '获取版本', exact: true }).click(); + await page.getByRole('button', { name: '立即更新', exact: true }).click(); + + const dialog = page.getByRole('dialog', { name: '系统更新' }); + await expect(dialog.getByText('50%', { exact: true })).toBeVisible({ timeout: 5_000 }); + await expect(dialog.getByText('1.0 MB/s', { exact: true })).toBeVisible(); + await expect(dialog.getByText('已发现新版本，但当前账号没有下载权限', { exact: true })).toHaveCount(0); + await expect(dialog.getByRole('button', { name: '立即更新并重启', exact: true })).toBeVisible({ timeout: 12_000 }); +}); + +test('restart dialog keeps counting down and reconnects while the backend is temporarily offline', async ({ page }) => { + await installFixture(page, [], [], true, false, false, 'READY', false, { restartOfflineReads: 8 }); + await page.goto('/governance/update'); + + await page.getByRole('button', { name: '立即更新并重启', exact: true }).click(); + + const dialog = page.getByRole('dialog', { name: '系统更新' }); + await expect(dialog.getByText('服务正在重启，页面会持续重连，不需要手动刷新。', { exact: false })).toBeVisible({ + timeout: 5_000, + }); + await expect(dialog.getByText('应用服务连接中断，正在等待进程重启', { exact: true })).toBeVisible(); + await expect(dialog.getByText(/新版本已通过健康检查，页面将在 \d+ 秒后自动刷新。/)).toBeVisible({ + timeout: 15_000, + }); +}); + test('rechecking a downloaded package prompts installation instead of downloading again', async ({ page }) => { await installFixture(page, [], [], true, false, false, 'READY'); await page.goto('/governance/update'); @@ -220,6 +287,19 @@ test('view-only administrators see a download permission message', async ({ page await expect(page.getByRole('button', { name: '立即更新并重启', exact: true })).toHaveCount(0); }); +test('a check response that is already queued is treated as busy instead of missing permission', async ({ page }) => { + await installFixture(page, [], [], true, false, false, undefined, false, { + checkResponseState: 'DOWNLOAD_QUEUED', + }); + await page.goto('/governance/update'); + + await page.getByRole('button', { name: '获取版本', exact: true }).click(); + + await expect(page.getByRole('dialog', { name: '系统更新' })).toBeVisible(); + await expect(page.getByText('发现新版本，但当前账号没有下载权限', { exact: true })).toHaveCount(0); + await expect(page.getByText('下载请求已排队', { exact: true }).first()).toBeVisible(); +}); + test('download and install continue polling after an accepted command loses its response', async ({ page }) => { await installFixture(page, [], [], true, false, false, undefined, false, { failDownloadResponse: true, @@ -252,6 +332,25 @@ test('accepted install continues to the refresh countdown after its response is }); }); +test('a lost install response stays pending while the application restarts', async ({ page }) => { + await installFixture(page, [], [], true, false, false, 'READY', false, { + failInstallResponse: true, + restartOfflineReads: 4, + }); + await page.goto('/governance/update'); + + await page.getByRole('button', { name: '立即更新并重启', exact: true }).click(); + + const dialog = page.getByRole('dialog', { name: '系统更新' }); + await expect(dialog.getByText('立即更新并重启失败', { exact: false })).toHaveCount(0); + await expect(dialog.getByText('服务正在重启，页面会持续重连，不需要手动刷新。', { exact: false })).toBeVisible({ + timeout: 5_000, + }); + await expect(dialog.getByText(/新版本已通过健康检查，页面将在 \d+ 秒后自动刷新。/)).toBeVisible({ + timeout: 10_000, + }); +}); + test('brand logo returns the active identity to its workbench', async ({ page }) => { await installFixture(page, [], []); await page.goto('/governance/update'); diff --git a/frontend/src/api/system-update.ts b/frontend/src/api/system-update.ts index e9cd202..0027427 100644 --- a/frontend/src/api/system-update.ts +++ b/frontend/src/api/system-update.ts @@ -2,6 +2,13 @@ import { request } from '@/utils/request'; import { commandHeaders } from './command'; +export interface SystemUpdateEvent { + occurredAt: string; + level: 'INFO' | 'WARN' | 'ERROR'; + stage: string; + message: string; +} + export interface SystemUpdateView { enabled: boolean; currentVersion: string; @@ -13,6 +20,12 @@ export interface SystemUpdateView { publishedAt?: string | null; checkedAt?: string | null; statusUpdatedAt?: string | null; + downloadedBytes?: number | null; + totalBytes?: number | null; + bytesPerSecond?: number | null; + downloadPercent?: number | null; + restartExpectedSeconds?: number | null; + events: SystemUpdateEvent[]; allowedActions: string[]; } @@ -21,6 +34,7 @@ function normalize(value: SystemUpdateView): SystemUpdateView { ...value, enabled: Boolean(value?.enabled), updateAvailable: Boolean(value?.updateAvailable), + events: Array.isArray(value?.events) ? value.events : [], allowedActions: Array.isArray(value?.allowedActions) ? value.allowedActions : [], }; } diff --git a/frontend/src/pages/governance/SystemUpdatePage.vue b/frontend/src/pages/governance/SystemUpdatePage.vue index 10ce14c..8708b5b 100644 --- a/frontend/src/pages/governance/SystemUpdatePage.vue +++ b/frontend/src/pages/governance/SystemUpdatePage.vue @@ -47,7 +47,7 @@ 立即更新 - -
- {{ statusError }} - 重新加载 +
+
+ 更新包下载进度 + {{ updateProgress }}%
- - - 新版本已通过健康检查，页面将在 {{ updateRefreshSeconds }} 秒后自动刷新。 - - -
- {{ updateStateLabel }} - {{ updateStatus?.message || '正在读取系统版本状态' }} - {{ updateActionHint }} + +
+ {{ formatBytes(updateStatus?.downloadedBytes) }} / {{ formatBytes(updateStatus?.totalBytes) }} + {{ downloadSpeedLabel }}
- +
+ +
版本说明 @@ -123,16 +137,99 @@ + + + +
@@ -613,6 +903,103 @@ onBeforeUnmount(() => { width: 100%; } +.download-progress { + display: flex; + flex-direction: column; + gap: 8px; + padding: 14px 16px; + background: var(--td-bg-color-secondarycontainer); + border: 1px solid var(--td-component-border); + border-radius: 6px; +} + +.download-progress__heading, +.download-progress__meta, +.runtime-log__heading, +.dialog-status-line, +.dialog-actions { + display: flex; + align-items: center; + justify-content: space-between; + gap: 12px; +} + +.download-progress__heading strong { + font-variant-numeric: tabular-nums; +} + +.download-progress__meta { + color: var(--td-text-color-secondary); + font-size: 13px; +} + +.update-dialog { + display: flex; + flex-direction: column; + gap: 18px; + min-width: 0; +} + +.dialog-status-line { + justify-content: flex-start; + min-width: 0; +} + +.dialog-status-line > span:last-child { + min-width: 0; + overflow-wrap: anywhere; +} + +.runtime-log { + min-width: 0; +} + +.runtime-log__heading { + margin-bottom: 8px; +} + +.runtime-log__heading small { + color: var(--td-text-color-secondary); +} + +.runtime-log__body { + display: flex; + flex-direction: column; + gap: 8px; + min-height: 120px; + max-height: 240px; + padding: 12px; + overflow: auto; + background: var(--td-bg-color-secondarycontainer); + border: 1px solid var(--td-component-border); + border-radius: 6px; +} + +.runtime-log__row { + display: grid; + grid-template-columns: 72px 58px minmax(0, 1fr); + align-items: start; + gap: 8px; + color: var(--td-text-color-primary); + font-size: 13px; + line-height: 22px; +} + +.runtime-log__row time { + color: var(--td-text-color-secondary); + font-variant-numeric: tabular-nums; +} + +.runtime-log__row > span:last-child { + min-width: 0; + overflow-wrap: anywhere; +} + +.dialog-actions { + justify-content: flex-end; + padding-top: 4px; +} + @media (width <= 900px) { .update-heading { align-items: stretch; @@ -650,5 +1037,23 @@ onBeforeUnmount(() => { flex-direction: column; gap: 8px; } + + .download-progress__meta, + .dialog-status-line, + .dialog-actions { + align-items: flex-start; + flex-direction: column; + } + + .dialog-actions, + .dialog-actions :deep(.t-button) { + width: 100%; + } + + .runtime-log__row { + grid-template-columns: 64px 52px minmax(0, 1fr); + gap: 6px; + font-size: 12px; + } } diff --git a/frontend/src/utils/request/Axios.ts b/frontend/src/utils/request/Axios.ts index 7cc0195..1475367 100644 --- a/frontend/src/utils/request/Axios.ts +++ b/frontend/src/utils/request/Axios.ts @@ -325,6 +325,7 @@ export class VAxios { status: e.response?.status, requestId, fieldErrors: problem?.fieldErrors, + transportFailure: !e.response, }); if (e.response?.status === 401 && window.location.pathname !== '/login') { window.dispatchEvent(new CustomEvent('kaidi:session-expired')); diff --git a/openapi.yaml b/openapi.yaml index 7603a32..9efdbdc 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -5967,6 +5967,18 @@ components: format: int64 type: integer type: object + SystemUpdateEventView: + properties: + level: + type: string + message: + type: string + occurredAt: + format: date-time + type: string + stage: + type: string + type: object SystemUpdateView: properties: allowedActions: @@ -5976,10 +5988,23 @@ components: checkedAt: format: date-time type: string + bytesPerSecond: + format: int64 + type: integer currentVersion: type: string enabled: type: boolean + downloadPercent: + format: int32 + type: integer + downloadedBytes: + format: int64 + type: integer + events: + items: + "$ref": "#/components/schemas/SystemUpdateEventView" + type: array latestVersion: type: string message: @@ -5989,11 +6014,17 @@ components: type: string releaseNotes: type: string + restartExpectedSeconds: + format: int32 + type: integer state: type: string statusUpdatedAt: format: date-time type: string + totalBytes: + format: int64 + type: integer updateAvailable: type: boolean type: object diff --git a/scripts/package-release.sh b/scripts/package-release.sh index 785e39b..11df0a8 100755 --- a/scripts/package-release.sh +++ b/scripts/package-release.sh @@ -105,6 +105,7 @@ chmod 0755 "$STAGE/ops/update.sh" "$STAGE/ops/baota-start.sh" "$STAGE/ops/baota- ARTIFACT="kaidi-finance-$VERSION.tar.gz" COPYFILE_DISABLE=1 tar --format=ustar -czf "$OUTPUT_DIR/$ARTIFACT" -C "$STAGE" . SHA256=$(sha256_file "$OUTPUT_DIR/$ARTIFACT") +ARTIFACT_SIZE_BYTES=$(wc -c < "$OUTPUT_DIR/$ARTIFACT" | tr -d '[:space:]') cp "$ROOT/backend/target/backend-sbom.json" "$OUTPUT_DIR/backend-sbom.cdx.json" (cd "$ROOT/frontend" && npm sbom --omit=dev --package-lock-only \ --sbom-format cyclonedx --sbom-type application) > "$OUTPUT_DIR/frontend-sbom.cdx.json" @@ -167,7 +168,8 @@ EOF BOOTSTRAP_SHA256=$(sha256_file "$OUTPUT_DIR/bootstrap-checksums.txt") RELEASE_NOTES_VALUE=${KAIDI_RELEASE_NOTES:-"Kaidi Finance Preview $VERSION"} -VERSION="$VERSION" ARTIFACT="$ARTIFACT" SHA256="$SHA256" RELEASE_NOTES="$RELEASE_NOTES_VALUE" \ +VERSION="$VERSION" ARTIFACT="$ARTIFACT" SHA256="$SHA256" ARTIFACT_SIZE_BYTES="$ARTIFACT_SIZE_BYTES" \ + RELEASE_NOTES="$RELEASE_NOTES_VALUE" \ BACKEND_SBOM_SHA256="$BACKEND_SBOM_SHA256" FRONTEND_SBOM_SHA256="$FRONTEND_SBOM_SHA256" \ BACKEND_BUILD_VERSION="$BACKEND_BUILD_VERSION" FRONTEND_BUILD_VERSION="$FRONTEND_BUILD_VERSION" \ INSTALLER_SHA256="$INSTALLER_SHA256" PURGER_SHA256="$PURGER_SHA256" \ @@ -179,6 +181,7 @@ const manifest = { version: process.env.VERSION, artifact: process.env.ARTIFACT, sha256: process.env.SHA256, + artifactSizeBytes: Number(process.env.ARTIFACT_SIZE_BYTES), publishedAt: new Date().toISOString(), minimumJava: 17, source: { diff --git a/scripts/test-install-fixture.sh b/scripts/test-install-fixture.sh index 47a6c45..c35e2d9 100755 --- a/scripts/test-install-fixture.sh +++ b/scripts/test-install-fixture.sh @@ -321,8 +321,10 @@ grep -Fqx 'Restart=no' "$ROOT/deploy/systemd/kaidi-update.service" \ || fail 'update service can automatically repeat a failed switching transaction' grep -Fq -- '-/www/wwwroot/kaidi' "$ROOT/deploy/systemd/kaidi-update.service" \ || fail 'update service requires the Baota application path on a systemd-only installation' -! grep -Fq '/var/lib/kaidi-update/processing' "$ROOT/deploy/systemd/kaidi-update.path" \ - || fail 'update path can automatically repeat a claimed switching transaction' +grep -Fqx 'PathExists=/var/lib/kaidi-update/processing/request.json' "$ROOT/deploy/systemd/kaidi-update.path" \ + || fail 'update path does not resume an interrupted claimed request' +grep -Fqx 'PathExists=/var/lib/kaidi-update/transactions/active' "$ROOT/deploy/systemd/kaidi-update.path" \ + || fail 'update path does not resume an interrupted switching transaction' # shellcheck disable=SC2016 # Match literal installer source. grep -Fq '[ "$actual_sha256" = "$java_sha256" ]' "$ROOT/deploy/install.sh" \ || fail 'installer no longer verifies the Java runtime SHA-256' diff --git a/scripts/test-update-fixture.sh b/scripts/test-update-fixture.sh index 1b47dc4..0cf326c 100755 --- a/scripts/test-update-fixture.sh +++ b/scripts/test-update-fixture.sh @@ -65,10 +65,14 @@ write_mock_commands() { output= url= header_file= +write_out= +http_status=200 printf '%s\n' "$*" >> "${MOCK_CURL_LOG:-/dev/null}" while [ "$#" -gt 0 ]; do case "$1" in - -o) shift; output=$1 ;; + -o|--output) shift; output=$1 ;; + --write-out) shift; write_out=$1 ;; + --connect-timeout|--max-time) shift ;; --header|-H) shift case "${1:-}" in @*) header_file=${1#@} ;; esac @@ -90,6 +94,26 @@ esac if [ -n "$output" ]; then if [ -n "${MOCK_RELEASE_API_URL:-}" ] && [ "$url" = "$MOCK_RELEASE_API_URL" ]; then cp "$FIXTURE_RELEASE_ROOT/release-api.json" "$output" + elif [ "$url" = "${KAIDI_HEALTH_URL:-http://127.0.0.1:18080/actuator/health}" ]; then + health_ok=false + case "${MOCK_HEALTH:-success}" in + success) health_ok=true ;; + fail-new) + [ "$(cat "$MOCK_APP_ROOT/current/VERSION" 2>/dev/null)" = '1.0.0-preview.1' ] && health_ok=true + ;; + fail-after-first) + health_count=$(cat "$MOCK_APP_ROOT/health-count" 2>/dev/null || printf 0) + health_count=$((health_count + 1)) + printf '%s\n' "$health_count" > "$MOCK_APP_ROOT/health-count" + [ "$health_count" -eq 1 ] && health_ok=true + ;; + esac + if [ "$health_ok" = true ]; then + printf '{"status":"UP"}\n' > "$output" + else + printf '{"status":"DOWN"}\n' > "$output" + http_status=503 + fi elif [ "$url" = "${KAIDI_APP_INDEX_URL:-http://127.0.0.1:18080/}" ]; then cp "$MOCK_APP_ROOT/current/public/index.html" "$output" else @@ -104,6 +128,7 @@ elif [ "${MOCK_HEALTH:-success}" = fail-new ] \ else exit 22 fi +[ -z "$write_out" ] || printf '%s' "$http_status" SH cat > "$mock_bin/systemctl" <<'SH' @@ -116,6 +141,11 @@ SH cat > "$mock_bin/flock" <<'SH' #!/bin/sh exit 0 +SH + + cat > "$mock_bin/setsid" <<'SH' +#!/bin/sh +exec "$@" SH cat > "$mock_bin/systemd-analyze" <<'SH' @@ -211,10 +241,12 @@ build_release() { local artifact="kaidi-finance-$version.tar.gz" COPYFILE_DISABLE=1 tar -czf "$fixture/release/$artifact" -C "$stage" . - local sha + local sha size sha=$($REAL_OPENSSL dgst -sha256 "$fixture/release/$artifact" | awk '{print $NF}') - jq -n --arg version "$version" --arg artifact "$artifact" --arg sha "$sha" \ - '{version:$version,artifact:$artifact,sha256:$sha,publishedAt:"2026-08-16T00:00:00Z",releaseNotes:"fixture"}' \ + size=$(wc -c < "$fixture/release/$artifact" | tr -d '[:space:]') + jq -n --arg version "$version" --arg artifact "$artifact" --arg sha "$sha" --argjson size "$size" \ + '{version:$version,artifact:$artifact,sha256:$sha,artifactSizeBytes:$size, + publishedAt:"2026-08-16T00:00:00Z",releaseNotes:"fixture"}' \ > "$fixture/release/release-manifest.json" "$REAL_OPENSSL" dgst -sha256 -sign "$fixture/private.pem" \ -out "$fixture/release/release-manifest.sig" "$fixture/release/release-manifest.json" @@ -269,6 +301,7 @@ write_request() { download_and_prepare_install() { local fixture=$1 local version=$2 + truncate -s 0 "$fixture/systemctl.log" run_update "$fixture" success [ "$(jq -r '.state' "$fixture/state/status.json")" = READY ] \ || fail 'download phase did not persist READY' @@ -276,6 +309,16 @@ download_and_prepare_install() { || fail 'download phase changed the active application' [ -s "$fixture/state/cache/$version/release.tar.gz" ] \ || fail 'download phase did not persist the verified artifact cache' + [ "$(jq -r '.downloadPercent' "$fixture/state/status.json")" -eq 100 ] \ + || fail 'download phase did not persist 100 percent progress' + [ "$(jq -r '.totalBytes' "$fixture/state/status.json")" -gt 0 ] \ + || fail 'download phase did not persist artifact bytes' + [ "$(jq -r '.bytesPerSecond' "$fixture/state/status.json")" -gt 0 ] \ + || fail 'download phase did not persist a measured transfer speed' + grep -Fq '"stage":"DOWNLOADING"' "$fixture/state/events.jsonl" \ + || fail 'download phase did not persist structured runtime events' + ! grep -Eq '^(start|restart|stop) kaidi-finance.service$' "$fixture/systemctl.log" \ + || fail 'download phase changed the application service' write_request "$fixture" "$version" INSTALL } @@ -407,7 +450,7 @@ assert_identical_systemd_operations_case() { prepare_installation "$fixture" "$version" download_and_prepare_install "$fixture" "$version" - : > "$fixture/systemctl.log" + truncate -s 0 "$fixture/systemctl.log" run_update "$fixture" success [ "$(readlink "$fixture/app/current")" = "$fixture/app/releases/$version" ] \ || fail 'identical systemd case did not activate the new application' @@ -474,17 +517,33 @@ assert_incomplete_rollback_requires_manual_recovery_case() { prepare_installation "$fixture" "$version" download_and_prepare_install "$fixture" "$version" - if run_update "$fixture" fail > "$fixture/update.log" 2>&1; then + if run_update "$fixture" fail-after-first > "$fixture/update.log" 2>&1; then fail 'incomplete rollback case unexpectedly succeeded' fi grep -Fq 'manual recovery is required' "$fixture/update.log" \ || fail 'incomplete rollback case did not require explicit recovery' [ ! -e "$fixture/state/processing/request.json" ] \ || fail 'incomplete rollback case left an automatically retriggered processing request' - [ -d "$fixture/state/transactions/active" ] \ - || fail 'incomplete rollback case did not preserve transaction evidence' + [ ! -e "$fixture/state/transactions/active" ] \ + || fail 'incomplete rollback case left transaction evidence on the automatic recovery path' + [ -d "$fixture/state/transactions/recovery-required" ] \ + || fail 'incomplete rollback case did not quarantine transaction evidence' + [ "$(jq -r '.state' "$fixture/state/status.json")" = RECOVERY_REQUIRED ] \ + || fail 'incomplete rollback case did not lock the updater for recovery' find "$fixture/state/failed" -type f -name 'request-*.json' -print -quit | grep -q . \ || fail 'incomplete rollback case did not archive its claimed request' + + truncate -s 0 "$fixture/systemctl.log" + write_request "$fixture" "$version" DOWNLOAD + if run_update "$fixture" success > "$fixture/retry.log" 2>&1; then + fail 'recovery-locked updater accepted a new download request' + fi + [ "$(jq -r '.state' "$fixture/state/status.json")" = RECOVERY_REQUIRED ] \ + || fail 'recovery-locked updater replaced the manual recovery status' + ! grep -Eq '^(start|restart|stop) kaidi-finance.service$' "$fixture/systemctl.log" \ + || fail 'recovery-locked download request changed the application service' + [ ! -e "$fixture/state/inbox/request.json" ] && [ ! -e "$fixture/state/processing/request.json" ] \ + || fail 'recovery-locked updater left a request on an automatic trigger path' } assert_download_failure_case() { @@ -526,6 +585,58 @@ assert_database_failure_case() { || fail 'database failure did not persist FAILED' } +assert_unhealthy_baseline_blocks_restart_case() { + local fixture="$WORK/unhealthy-baseline" + local version='1.0.0-preview.2' + mkdir -p "$fixture" + write_mock_commands "$fixture/mock-bin" + build_release "$fixture" "$version" + prepare_installation "$fixture" "$version" + download_and_prepare_install "$fixture" "$version" + truncate -s 0 "$fixture/systemctl.log" + + if run_update "$fixture" fail > "$fixture/update.log" 2>&1; then + fail 'unhealthy baseline unexpectedly reached installation' + fi + grep -Fq 'Current release preflight failed' "$fixture/update.log" \ + || fail 'unhealthy baseline did not preserve its preflight diagnostic' + ! grep -Eq '^(start|restart|stop) kaidi-finance.service$' "$fixture/systemctl.log" \ + || fail 'unhealthy baseline restarted the application' + [ "$(readlink "$fixture/app/current")" = "$fixture/app/releases/1.0.0-preview.1" ] \ + || fail 'unhealthy baseline changed the active release' + [ ! -e "$fixture/state/transactions/active" ] \ + || fail 'unhealthy baseline created a switching transaction' +} + +assert_invalid_database_url_blocks_restart_case() { + local fixture="$WORK/invalid-database-url" + local version='1.0.0-preview.2' + mkdir -p "$fixture" + write_mock_commands "$fixture/mock-bin" + build_release "$fixture" "$version" + prepare_installation "$fixture" "$version" + download_and_prepare_install "$fixture" "$version" + cat > "$fixture/runtime.env" <<'EOF' +DB_URL="jdbc:mysql://" +KAIDI_DB_HOST="127.0.0.1" +KAIDI_DB_PORT="3306" +KAIDI_DB_NAME="kaidi_finance" +KAIDI_DB_USERNAME="kaidi" +KAIDI_DB_PASSWORD="fixture" +EOF + truncate -s 0 "$fixture/systemctl.log" + + if run_update "$fixture" success > "$fixture/update.log" 2>&1; then + fail 'invalid database URL unexpectedly reached installation' + fi + grep -Fq 'DB_URL does not match the configured MySQL host, port, and database' "$fixture/update.log" \ + || fail 'structurally invalid JDBC URL did not preserve its diagnostic' + ! grep -q '^restart kaidi-finance.service$' "$fixture/systemctl.log" \ + || fail 'invalid database URL restarted the application' + [ "$(readlink "$fixture/app/current")" = "$fixture/app/releases/1.0.0-preview.1" ] \ + || fail 'invalid database URL changed the active release' +} + assert_symlink_request_rejected() { local fixture="$WORK/symlink-request" local version='1.0.0-preview.2' @@ -577,6 +688,8 @@ assert_rollback_case assert_incomplete_rollback_requires_manual_recovery_case assert_download_failure_case assert_database_failure_case +assert_unhealthy_baseline_blocks_restart_case +assert_invalid_database_url_blocks_restart_case assert_symlink_request_rejected assert_install_without_verified_cache_rejected assert_private_gitea_release_case diff --git a/scripts/verify-release.sh b/scripts/verify-release.sh index f6ceade..c01df77 100755 --- a/scripts/verify-release.sh +++ b/scripts/verify-release.sh @@ -56,6 +56,10 @@ EXPECTED_ARTIFACT_SHA256=$(jq -er \ release-manifest.json) [ "$(sha256_file "$ARTIFACT")" = "$EXPECTED_ARTIFACT_SHA256" ] \ || { printf 'Release artifact digest does not match the signed manifest\n' >&2; exit 1; } +EXPECTED_ARTIFACT_SIZE=$(jq -er '.artifactSizeBytes | numbers | floor | select(. > 0)' \ + release-manifest.json) +[ "$(wc -c < "$ARTIFACT" | tr -d '[:space:]')" -eq "$EXPECTED_ARTIFACT_SIZE" ] \ + || { printf 'Release artifact size does not match the signed manifest\n' >&2; exit 1; } [ "$(jq '.sboms | length' release-manifest.json)" -eq 2 ] \ || { printf 'Release manifest must bind two SBOMs\n' >&2; exit 1; } jq -e --arg version "$VERSION" \