fix: recover stuck update queue

server/app.ts

@@ -119,7 +119,7 @@ function enforceUpdateCooldown(
   adminId: string,
   operation: "check" | "download" | "apply",
   reply: FastifyReply,
-): void {
+): number {
   const state = updateRateState(database, adminId);
   const now = Date.now();
   const previous = operation === "check" ? state.checkedAt : operation === "download" ? state.downloadedAt : state.appliedAt;
@@ -134,6 +134,7 @@ function enforceUpdateCooldown(
   if (operation === "check") state.checkedAt = now;
   else if (operation === "download") state.downloadedAt = now;
   else state.appliedAt = now;
+  return now;
 }
 
 function adminSelect(alias = ""): string {
@@ -957,12 +958,15 @@ export async function buildApp(database: DatabaseContext, config: AppConfig) {
   });
 
   app.post("/api/update/check", { preHandler: guard(database, config), bodyLimit: 32 * 1024 }, async (request, reply) => {
+    const rateState = updateRateState(database.sqlite, request.auth!.admin.id);
+    const previousCheckedAt = rateState.checkedAt;
+    let reservedCheckedAt: number | null = null;
     try {
       reconcileOrphanedUpdateJobs(database.sqlite, config);
       // Disabled/dev installs do not contact a release endpoint, so repeated
       // checks are local status reads and should remain immediately usable.
       if (config.updateStrategy !== "disabled") {
-        enforceUpdateCooldown(database.sqlite, config, request.auth!.admin.id, "check", reply);
+        reservedCheckedAt = enforceUpdateCooldown(database.sqlite, config, request.auth!.admin.id, "check", reply);
       }
       const result = await checkForUpdate(database.sqlite, config);
       writeAudit(database.sqlite, {
@@ -981,6 +985,10 @@ export async function buildApp(database: DatabaseContext, config: AppConfig) {
       reply.header("Cache-Control", "no-store");
       return { ...result, strategy: config.updateStrategy };
     } catch (error) {
+      // A failed upstream request is not a successful check. Release the
+      // reservation only when this request still owns it, so a concurrent
+      // successful check cannot have its cooldown overwritten.
+      if (reservedCheckedAt !== null && rateState.checkedAt === reservedCheckedAt) rateState.checkedAt = previousCheckedAt;
       writeAudit(database.sqlite, {
         requestId: request.id,
         actorAdminId: request.auth!.admin.id,

server/update-service.ts

@@ -1,4 +1,4 @@
-import { lstatSync, realpathSync, unlinkSync } from "node:fs";
+import { lstatSync, realpathSync, readFileSync, unlinkSync } from "node:fs";
 import { chmod, mkdir, rename, writeFile } from "node:fs/promises";
 import path from "node:path";
 import { createPublicKey, randomUUID, verify as verifySignature } from "node:crypto";
@@ -385,6 +385,17 @@ function removeExpiredRequest(filePath: string, now: number): void {
   }
 }
 
+function requestJobId(filePath: string): string | null {
+  try {
+    const info = lstatSync(filePath);
+    if (!info.isFile() || info.isSymbolicLink()) return null;
+    const value = JSON.parse(readFileSync(filePath, "utf8")) as { jobId?: unknown };
+    return typeof value.jobId === "string" && /^[0-9a-f-]{36}$/.test(value.jobId) ? value.jobId : null;
+  } catch {
+    return null;
+  }
+}
+
 function currentReleaseVersion(config: AppConfig): string | null {
   try {
     const target = realpathSync(config.currentLink);
@@ -425,6 +436,7 @@ export function reconcileOrphanedUpdateJobs(database: Database.Database, config:
   // row is still safe to retry and must not block the queue forever.
   const releaseVersion = currentReleaseVersion(config);
   let reconciled = 0;
+  const reconciledIds = new Set<string>();
   for (const row of rows) {
     if (typeof row.updatedAt !== "number" || now - row.updatedAt < ORPHANED_UPDATE_TIMEOUT_MS) continue;
     // The runner refreshes the state marker while a download is in flight.
@@ -451,7 +463,10 @@ export function reconcileOrphanedUpdateJobs(database: Database.Database, config:
         });
         return true;
       })();
-      if (changed) reconciled += 1;
+      if (changed) {
+        reconciled += 1;
+        reconciledIds.add(row.id);
+      }
       continue;
     }
     if (requestFresh || stateFresh) continue;
@@ -476,7 +491,10 @@ export function reconcileOrphanedUpdateJobs(database: Database.Database, config:
       });
       return true;
     })();
-    if (changed) reconciled += 1;
+    if (changed) {
+      reconciled += 1;
+      reconciledIds.add(row.id);
+    }
   }
   // Prevent a stale request from being replayed after its DB row has been
   // marked failed. The path is fixed by the server configuration and the
@@ -484,7 +502,17 @@ export function reconcileOrphanedUpdateJobs(database: Database.Database, config:
   // A download runner refreshes the state marker while it is still using the
   // request. Keep the request until that lease also expires; otherwise a
   // long download can lose its job id and fail to finalize its row.
-  if (!stateFresh && (!requestPresent || (requestMtime !== null && now - requestMtime >= ORPHANED_UPDATE_TIMEOUT_MS))) {
+  const queuedRequestId = requestPresent ? requestJobId(config.updateRequestPath) : null;
+  const queuedRequest = queuedRequestId ? rows.find((row) => row.id === queuedRequestId) : undefined;
+  const requestStillNeeded = Boolean(
+    queuedRequest
+      && ACTIVE_UPDATE_STATUSES.includes(queuedRequest.status)
+      && !reconciledIds.has(queuedRequest.id)
+      && !(queuedRequest.status === "staged" && queuedRequest.operation === "download"),
+  );
+  if (!stateFresh
+    && (!requestPresent || (requestMtime !== null && now - requestMtime >= ORPHANED_UPDATE_TIMEOUT_MS))
+    && !requestStillNeeded) {
     removeExpiredRequest(config.updateRequestPath, now);
   }
   return reconciled;