feat: integrate platform backend and application interfaces
This commit is contained in:
Qiufeng
@@ -0,0 +1,94 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import type { MessageOutboxRecord, Store } from "../store.ts";
|
||||
import { decryptSecret, encryptSecret, hashChallenge } from "../shared/auth.ts";
|
||||
import { validateProviderUrlResolved } from "./provider.ts";
|
||||
import { safeOutboundFetch } from "../infra/outbound-url.ts";
|
||||
|
||||
export type MessageChannel = "email" | "sms";
|
||||
export type MessagePurpose = "register" | "reset" | "notification";
|
||||
const messageWorkerId = `message-worker-${process.pid}-${randomUUID().slice(0, 8)}`;
|
||||
const MAX_MESSAGE_ATTEMPTS = 5;
|
||||
const MESSAGE_LEASE_MS = 30_000;
|
||||
|
||||
export function enqueueMessage(store: Store, input: { channel: MessageChannel; target: string; purpose: MessagePurpose; templateData?: Record<string, string>; idempotencyKey?: string; deferProcessing?: boolean }): MessageOutboxRecord {
|
||||
const idempotencyKey = input.idempotencyKey?.trim() || undefined;
|
||||
if (idempotencyKey) {
|
||||
const existing = [...store.messageOutbox.values()].find((item) => item.idempotencyKey === idempotencyKey);
|
||||
if (existing) return existing;
|
||||
}
|
||||
const now = new Date().toISOString();
|
||||
const record: MessageOutboxRecord = { id: randomUUID(), idempotencyKey, channel: input.channel, targetHash: hashChallenge(input.target, store.channelEncryptionKey), targetEncrypted: encryptSecret(input.target, store.channelEncryptionKey), payloadEncrypted: input.templateData ? encryptSecret(JSON.stringify(input.templateData), store.channelEncryptionKey) : undefined, purpose: input.purpose, status: "queued", attempts: 0, createdAt: now, updatedAt: now };
|
||||
store.messageOutbox.set(record.id, record); store.persist();
|
||||
if (!input.deferProcessing) setTimeout(() => { void processMessageOutbox(store); }, 0).unref?.();
|
||||
return record;
|
||||
}
|
||||
|
||||
async function sendMessage(store: Store, record: MessageOutboxRecord): Promise<{ providerMessageId?: string }> {
|
||||
const payloadText = record.payloadEncrypted ? decryptSecret(record.payloadEncrypted, store.channelEncryptionKey) : undefined;
|
||||
const target = record.targetEncrypted ? decryptSecret(record.targetEncrypted, store.channelEncryptionKey) : undefined;
|
||||
const providers = [...store.messageProviders.values()].filter((provider) => provider.channel === record.channel && provider.enabled !== false).sort((a, b) => Number(a.priority || 0) - Number(b.priority || 0));
|
||||
const candidates = providers.length ? providers : process.env.NODE_ENV === "production" ? [] : [{ id: "mock", channel: record.channel, enabled: true }];
|
||||
let sentBy: Record<string, unknown> | undefined; let providerMessageId: string | undefined; let lastError = "message provider unavailable";
|
||||
for (const provider of candidates) {
|
||||
const providerId = String(provider.id || "mock");
|
||||
if (process.env.MIRAGENFLOW_MESSAGE_FAIL === record.channel || process.env.MIRAGENFLOW_MESSAGE_FAIL === providerId || provider.fixtureFailure === "retryable") { lastError = `${providerId}: message provider unavailable`; continue; }
|
||||
try {
|
||||
const endpoint = typeof provider.endpoint === "string" ? provider.endpoint.trim() : "";
|
||||
if (!endpoint) { if (process.env.NODE_ENV === "production") throw new Error(`${providerId}: provider endpoint is not configured`); }
|
||||
else {
|
||||
const url = await validateProviderUrlResolved(endpoint); const headers = new Headers({ "content-type": "application/json", accept: "application/json" });
|
||||
const secret = typeof provider.secretRef === "string" ? decryptSecret(provider.secretRef, store.channelEncryptionKey) : undefined; if (secret) headers.set("authorization", `Bearer ${secret}`);
|
||||
if (record.idempotencyKey) headers.set("idempotency-key", `miragenflow-message:${record.idempotencyKey}`);
|
||||
const response = await safeOutboundFetch(url, { method: "POST", headers, body: JSON.stringify({ to: target, channel: record.channel, purpose: record.purpose, data: payloadText ? JSON.parse(payloadText) : undefined }) }, { timeoutMs: 15_000, maxBytes: 1024 * 1024 });
|
||||
if (!response.ok) throw new Error(`${providerId}: provider HTTP ${response.status}`);
|
||||
const responseBody = await response.text(); let parsed: Record<string, unknown> = {}; try { parsed = responseBody ? JSON.parse(responseBody) as Record<string, unknown> : {}; } catch { /* empty/text body */ }
|
||||
providerMessageId = typeof parsed.id === "string" ? parsed.id : response.headers.get("x-message-id") || `${providerId}-${randomUUID()}`;
|
||||
}
|
||||
sentBy = provider; break;
|
||||
} catch (error) { lastError = error instanceof Error ? error.message : `${providerId}: provider request failed`; }
|
||||
}
|
||||
if (!sentBy) throw new Error(lastError);
|
||||
return { providerMessageId: providerMessageId || `${String(sentBy.id || "mock")}-${randomUUID()}` };
|
||||
}
|
||||
|
||||
async function processPostgresOutbox(store: Store) {
|
||||
const claim = store.repository.claimMessageOutbox; const complete = store.repository.completeMessageOutbox; const renew = store.repository.renewMessageOutbox; if (!claim || !complete) return;
|
||||
const worker = messageWorkerId; const records = await claim.call(store.repository, worker, MESSAGE_LEASE_MS);
|
||||
for (const record of records) {
|
||||
store.messageOutbox.set(record.id, record);
|
||||
let leaseLost = false;
|
||||
const renewTimer = renew ? setInterval(() => {
|
||||
void renew.call(store.repository, record.id, worker, MESSAGE_LEASE_MS).then((result) => {
|
||||
if (!result.renewed) leaseLost = true;
|
||||
else if (result.leaseExpiresAt) record.leaseExpiresAt = result.leaseExpiresAt;
|
||||
}).catch(() => { leaseLost = true; });
|
||||
}, Math.max(1_000, Math.floor(MESSAGE_LEASE_MS / 3))) : undefined;
|
||||
renewTimer?.unref?.();
|
||||
try {
|
||||
const result = await sendMessage(store, record); const completed = await complete.call(store.repository, record.id, worker, { success: true, providerMessageId: result.providerMessageId }); if (completed && !leaseLost) { record.status = "sent"; record.providerMessageId = result.providerMessageId; record.lastError = undefined; record.nextAttemptAt = undefined; record.leaseOwner = undefined; record.leaseExpiresAt = undefined; record.updatedAt = new Date().toISOString(); }
|
||||
} catch (error) {
|
||||
const nextAttemptAt = new Date(Date.now() + Math.min(15 * 60_000, 2 ** record.attempts * 1_000)).toISOString(); const completed = await complete.call(store.repository, record.id, worker, { success: false, error: error instanceof Error ? error.message : String(error), nextAttemptAt }); if (completed && !leaseLost) { record.status = record.attempts >= MAX_MESSAGE_ATTEMPTS ? "dead" : "failed"; record.lastError = error instanceof Error ? error.message : String(error); record.nextAttemptAt = nextAttemptAt; record.leaseOwner = undefined; record.leaseExpiresAt = undefined; record.updatedAt = new Date().toISOString(); }
|
||||
} finally {
|
||||
if (renewTimer) clearInterval(renewTimer);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function processMessageOutbox(store: Store) {
|
||||
if (store.repository.adapter === "postgres" && store.repository.claimMessageOutbox && store.repository.completeMessageOutbox) { await processPostgresOutbox(store); return; }
|
||||
const now = Date.now();
|
||||
for (const record of store.messageOutbox.values()) {
|
||||
if (!(record.status === "queued" || record.status === "failed" || record.status === "pending") || (record.nextAttemptAt && Date.parse(record.nextAttemptAt) > now)) continue;
|
||||
if (record.attempts >= MAX_MESSAGE_ATTEMPTS) { record.status = "dead"; record.updatedAt = new Date().toISOString(); continue; }
|
||||
record.status = "sending"; record.attempts += 1; record.leaseOwner = messageWorkerId; record.leaseExpiresAt = new Date(now + MESSAGE_LEASE_MS).toISOString(); record.updatedAt = new Date().toISOString(); store.persist();
|
||||
try { const result = await sendMessage(store, record); record.status = "sent"; record.providerMessageId = result.providerMessageId; record.lastError = undefined; record.nextAttemptAt = undefined; }
|
||||
catch (error) { record.status = record.attempts >= MAX_MESSAGE_ATTEMPTS ? "dead" : "failed"; record.lastError = error instanceof Error ? error.message : String(error); record.nextAttemptAt = new Date(Date.now() + Math.min(15 * 60_000, 2 ** record.attempts * 1_000)).toISOString(); }
|
||||
finally { record.leaseExpiresAt = undefined; record.leaseOwner = undefined; record.updatedAt = new Date().toISOString(); store.persist(); }
|
||||
}
|
||||
}
|
||||
|
||||
export function recoverMessageOutbox(store: Store) {
|
||||
if (store.repository.adapter === "postgres") return;
|
||||
const now = Date.now();
|
||||
for (const record of store.messageOutbox.values()) if (record.status === "sending" && record.leaseExpiresAt && Date.parse(record.leaseExpiresAt) <= now) { record.status = "failed"; record.nextAttemptAt = new Date(now).toISOString(); record.updatedAt = new Date(now).toISOString(); }
|
||||
}
|
||||
@@ -0,0 +1,293 @@
|
||||
import { readFile } from "node:fs/promises";
|
||||
import type { TaskType } from "@miragenflow/contracts";
|
||||
import type { ProviderChannel, Store } from "../store.ts";
|
||||
import { imageModelFor, normalizeImageParams } from "../image-options.ts";
|
||||
import { decryptSecret } from "../shared/auth.ts";
|
||||
import { safeStagingPath } from "../infra/staging.ts";
|
||||
import { isPublicAddress, resolvePublicHttpsUrl, safeOutboundFetch } from "../infra/outbound-url.ts";
|
||||
|
||||
export type ProviderOutput = { mimeType: string; data: string; metadata?: { width?: number; height?: number; format?: string; size?: string; revisedPrompt?: string; usage?: Record<string, number>; source?: "base64" | "url" } };
|
||||
export type ProviderResult = { status: "succeeded" | "unknown" | "failed"; providerRequestId?: string; outputs?: ProviderOutput[]; errorCode?: string; retryable?: boolean };
|
||||
export type ProviderInput = { taskId: string; taskType: TaskType; publicModelId?: string; prompt?: string; count?: number; references?: string[]; referenceImages?: Array<{ objectId: string; seq: number; name: string }>; maskObjectId?: string; params?: Record<string, unknown>; platformIdempotencyKey?: string; onRequestAttempted?: () => void };
|
||||
export type ProviderCancelResult = "confirmed" | "pending" | "unsupported";
|
||||
|
||||
function resolveChannelModel(channel: ProviderChannel, publicModelId?: string) {
|
||||
const mappedModel = channel.modelMappings?.find((mapping) => mapping.displayModelId === publicModelId)?.requestModelId;
|
||||
const directModel = publicModelId && (channel.enabledModelIds?.includes(publicModelId) || channel.providerModelId === publicModelId) ? publicModelId : undefined;
|
||||
if (channel.providerType === "openai-images" && publicModelId && !mappedModel && !directModel) {
|
||||
throw Object.assign(new Error("public model is not mapped to this channel"), { errorCode: "PROVIDER_MODEL_NOT_CONFIGURED", retryable: false });
|
||||
}
|
||||
return { model: mappedModel || directModel || channel.enabledModelIds?.[0] || channel.providerModelId, exact: Boolean(mappedModel || directModel) };
|
||||
}
|
||||
|
||||
const chineseImageNumbers = ["一", "二", "三", "四", "五", "六", "七", "八", "九", "十", "十一", "十二", "十三", "十四", "十五", "十六"];
|
||||
const imageMentionPattern = new RegExp(`@图片(${chineseImageNumbers.slice().sort((left, right) => right.length - left.length).join("|")})`, "g");
|
||||
|
||||
function mentionedImageNames(prompt: string) {
|
||||
return new Set([...prompt.matchAll(imageMentionPattern)].map((match) => `图片${match[1]}`));
|
||||
}
|
||||
|
||||
export function allowLocalProviderUrls() {
|
||||
const configured = process.env.MIRAGENFLOW_ALLOW_LOCAL_PROVIDER_URLS;
|
||||
return configured === "true" || (configured === undefined && process.env.NODE_ENV !== "production");
|
||||
}
|
||||
export function validateProviderUrl(value: string) { const url = new URL(value); const allowPrivateNetwork = allowLocalProviderUrls(); const protocolAllowed = url.protocol === "https:" || (allowPrivateNetwork && url.protocol === "http:"); if (!protocolAllowed || url.username || url.password) throw new Error(allowPrivateNetwork ? "provider URL must use HTTP or HTTPS without embedded credentials" : "provider URL must use HTTPS without embedded credentials"); const literal = url.hostname.replace(/^\[|\]$/g, ""); if (!allowPrivateNetwork && /^[\d.:a-f]+$/i.test(literal) && !isPublicAddress(literal)) throw new Error("provider URL targets a non-public address"); return url; }
|
||||
export async function validateProviderUrlResolved(value: string | URL) { return (await resolvePublicHttpsUrl(validateProviderUrl(String(value)), undefined, { allowPrivateNetwork: allowLocalProviderUrls() })).url; }
|
||||
const validateResolvedUrl = validateProviderUrlResolved;
|
||||
function providerEndpoint(base: URL, path: string) {
|
||||
const url = new URL(base);
|
||||
url.search = "";
|
||||
url.hash = "";
|
||||
const prefix = url.pathname.replace(/\/+$/, "");
|
||||
const normalizedPath = path.replace(/^\/+/, "").replace(/^v1(?:\/|$)/, "");
|
||||
const apiPrefix = prefix === "/v1" || prefix.endsWith("/v1") ? prefix : `${prefix}/v1`;
|
||||
url.pathname = `${apiPrefix}/${normalizedPath}`.replace(/\/{2,}/g, "/").replace(/\/$/, "") || "/";
|
||||
return url;
|
||||
}
|
||||
|
||||
/** Return the path used by the model-list probe without exposing credentials. */
|
||||
export function providerModelsPath(value: string | URL) {
|
||||
return providerEndpoint(value instanceof URL ? value : new URL(value), "v1/models").pathname;
|
||||
}
|
||||
|
||||
function providerRequestId(response: Response) {
|
||||
for (const header of ["x-request-id", "request-id", "openai-request-id", "x-correlation-id"]) {
|
||||
const value = response.headers.get(header)?.trim();
|
||||
if (value) return value;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
async function providerFetch(url: URL, secret: string | undefined, init: RequestInit, timeoutMs = 60_000, onRequestAttempted?: () => void) {
|
||||
const headers = new Headers(init.headers); if (secret) headers.set("authorization", `Bearer ${secret}`);
|
||||
return safeOutboundFetch(url, { ...init, headers }, { timeoutMs, maxBytes: 50 * 1024 * 1024, allowPrivateNetwork: allowLocalProviderUrls(), onRequestAttempted });
|
||||
}
|
||||
|
||||
async function jsonResponse(response: Response) { const text = await response.text(); if (Buffer.byteLength(text) > 50 * 1024 * 1024) throw new Error("provider response too large"); let parsed: Record<string, unknown> | undefined; try { parsed = text ? JSON.parse(text) as Record<string, unknown> : undefined; } catch { /* handled below */ } if (!response.ok) throw providerHttpError(response.status, parsed); if (!parsed) throw new Error("provider response is not valid JSON"); return parsed; }
|
||||
function providerHttpError(status: number, body?: Record<string, unknown>) { const error = new Error(`provider http ${status}`) as Error & { retryable?: boolean; errorCode?: string; providerStatusCode?: number }; const nested = body?.error && typeof body.error === "object" ? body.error as Record<string, unknown> : body; const code = `${nested?.code || ""} ${nested?.type || ""}`.toLowerCase(); error.providerStatusCode = status; error.retryable = status === 408 || status === 429 || status >= 500; error.errorCode = status === 401 || status === 403 ? "PROVIDER_AUTH" : /content|safety|moderation|policy/.test(code) ? "PROVIDER_CONTENT_REJECTED" : /model_not_found|invalid_request|invalid_model/.test(code) ? "PROVIDER_INVALID_REQUEST" : "PROVIDER_HTTP"; return error; }
|
||||
export function imageMimeFromBytes(bytes: Buffer) {
|
||||
if (bytes.length >= 24 && bytes.subarray(0, 8).equals(Buffer.from([137, 80, 78, 71, 13, 10, 26, 10])) && bytes.subarray(12, 16).toString() === "IHDR") return "image/png";
|
||||
if (bytes.length >= 4 && bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff && bytes.lastIndexOf(Buffer.from([0xff, 0xd9])) >= 2) return "image/jpeg";
|
||||
if (bytes.length >= 30 && bytes.subarray(0, 4).toString() === "RIFF" && bytes.subarray(8, 12).toString() === "WEBP" && imageDimensions(bytes, "image/webp")) return "image/webp";
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function invalidProviderOutput(message: string, cause?: unknown) {
|
||||
const error = Object.assign(new Error(message), { errorCode: "PROVIDER_INVALID_OUTPUT", retryable: false });
|
||||
if (cause !== undefined) (error as Error & { cause?: unknown }).cause = cause;
|
||||
return error;
|
||||
}
|
||||
|
||||
function decodeImageBase64(value: string) {
|
||||
const encoded = value.trim();
|
||||
if (!encoded || encoded.length % 4 !== 0 || !/^[A-Za-z0-9+/]*={0,2}$/.test(encoded)) throw invalidProviderOutput("provider image base64 is invalid");
|
||||
const bytes = Buffer.from(encoded, "base64");
|
||||
const mimeType = imageMimeFromBytes(bytes);
|
||||
if (!bytes.length || !mimeType) throw invalidProviderOutput("provider image bytes are invalid");
|
||||
if (!imageDimensions(bytes, mimeType)) throw invalidProviderOutput("provider image dimensions are invalid");
|
||||
return { bytes, mimeType };
|
||||
}
|
||||
|
||||
async function downloadOutput(url: string) {
|
||||
try {
|
||||
const response = await providerFetch(await validateResolvedUrl(url), undefined, { method: "GET" }, 30_000);
|
||||
if (!response.ok) throw new Error(`provider output http ${response.status}`);
|
||||
const bytes = Buffer.from(await response.arrayBuffer());
|
||||
if (bytes.byteLength > 50 * 1024 * 1024) throw new Error("provider output too large");
|
||||
const detectedMime = imageMimeFromBytes(bytes);
|
||||
const declaredMime = (response.headers.get("content-type") || "").split(";")[0].trim().toLowerCase();
|
||||
const normalizedDeclaredMime = declaredMime === "image/jpg" ? "image/jpeg" : declaredMime;
|
||||
if (!detectedMime || (normalizedDeclaredMime && normalizedDeclaredMime !== "application/octet-stream" && normalizedDeclaredMime !== detectedMime) || !imageDimensions(bytes, detectedMime)) throw new Error("provider output is not a valid image");
|
||||
return { mimeType: detectedMime, data: bytes.toString("base64"), source: "url" as const };
|
||||
} catch (error) {
|
||||
if ((error as Error & { errorCode?: string }).errorCode === "PROVIDER_INVALID_OUTPUT") throw error;
|
||||
throw invalidProviderOutput("provider output URL is invalid or unavailable", error);
|
||||
}
|
||||
}
|
||||
|
||||
export function imageDimensions(bytes: Buffer, mimeType: string) {
|
||||
if (mimeType === "image/png" && bytes.length >= 24 && bytes.subarray(0, 8).equals(Buffer.from([137, 80, 78, 71, 13, 10, 26, 10])) && bytes.subarray(12, 16).toString() === "IHDR") {
|
||||
const width = bytes.readUInt32BE(16); const height = bytes.readUInt32BE(20);
|
||||
return width > 0 && height > 0 ? { width, height } : undefined;
|
||||
}
|
||||
if (mimeType === "image/webp" && bytes.length >= 30 && bytes.subarray(0, 4).toString() === "RIFF" && bytes.subarray(8, 12).toString() === "WEBP") {
|
||||
const marker = bytes.subarray(12, 16).toString();
|
||||
const chunkSize = bytes.length >= 20 ? bytes.readUInt32LE(16) : 0;
|
||||
if (chunkSize > bytes.length - 20) return undefined;
|
||||
if (marker === "VP8X" && chunkSize >= 10 && bytes.length >= 30) return { width: 1 + bytes.readUIntLE(24, 3), height: 1 + bytes.readUIntLE(27, 3) };
|
||||
if (marker === "VP8L" && chunkSize >= 5 && bytes.length >= 25 && bytes[20] === 0x2f) { const bits = bytes.readUInt32LE(21); return { width: 1 + (bits & 0x3fff), height: 1 + ((bits >>> 14) & 0x3fff) }; }
|
||||
if (marker === "VP8 " && chunkSize >= 10 && bytes.length >= 30 && bytes[23] === 0x9d && bytes[24] === 0x01 && bytes[25] === 0x2a) return { width: bytes.readUInt16LE(26) & 0x3fff, height: bytes.readUInt16LE(28) & 0x3fff };
|
||||
}
|
||||
if (mimeType === "image/jpeg" && bytes.length > 4 && bytes[0] === 0xff && bytes[1] === 0xd8) {
|
||||
let offset = 2;
|
||||
while (offset + 9 < bytes.length) {
|
||||
if (bytes[offset] !== 0xff) { offset += 1; continue; }
|
||||
const marker = bytes[offset + 1]; const length = bytes.readUInt16BE(offset + 2);
|
||||
if (length < 2 || offset + 2 + length > bytes.length) break;
|
||||
if ((marker >= 0xc0 && marker <= 0xc3) || (marker >= 0xc5 && marker <= 0xc7) || (marker >= 0xc9 && marker <= 0xcb) || (marker >= 0xcd && marker <= 0xcf)) { const width = bytes.readUInt16BE(offset + 7); const height = bytes.readUInt16BE(offset + 5); return width > 0 && height > 0 ? { width, height } : undefined; }
|
||||
offset += 2 + length;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
function imageOutputMetadata(item: Record<string, unknown>, data: Record<string, unknown>, mimeType: string, source: "base64" | "url", bytes?: Buffer) { const size = typeof item.size === "string" ? item.size : typeof data.size === "string" ? data.size : undefined; const match = size?.match(/^(\d+)x(\d+)$/); const dimensions = bytes ? imageDimensions(bytes, mimeType) : undefined; return { width: dimensions?.width || (match ? Number(match[1]) : undefined), height: dimensions?.height || (match ? Number(match[2]) : undefined), format: mimeType.split("/")[1], size: dimensions ? `${dimensions.width}x${dimensions.height}` : size, revisedPrompt: typeof item.revised_prompt === "string" ? item.revised_prompt : undefined, usage: data.usage && typeof data.usage === "object" ? Object.fromEntries(Object.entries(data.usage as Record<string, unknown>).filter(([, value]) => typeof value === "number")) as Record<string, number> : undefined, source }; }
|
||||
|
||||
async function referenceBlob(store: Store, objectId: string) {
|
||||
const object = store.objects.get(objectId);
|
||||
if (!object || object.revoked || !object.expiresAt || Date.parse(object.expiresAt) <= Date.now()) throw Object.assign(new Error("reference object unavailable"), { errorCode: "PROVIDER_REFERENCE_UNAVAILABLE", retryable: false });
|
||||
let bytes: Buffer;
|
||||
try { bytes = object.stagingKey ? await readFile(safeStagingPath(store.stagingDir, object.stagingKey)) : Buffer.from(object.data || "", "base64"); }
|
||||
catch (error) { throw Object.assign(new Error("reference object unavailable"), { errorCode: "PROVIDER_REFERENCE_UNAVAILABLE", retryable: false, cause: error }); }
|
||||
if (!bytes.length || !/^image\/(png|jpeg|webp)$/.test(object.mimeType)) throw Object.assign(new Error("reference object is not a supported image"), { errorCode: "PROVIDER_REFERENCE_INVALID", retryable: false });
|
||||
if (imageMimeFromBytes(bytes) !== object.mimeType || !imageDimensions(bytes, object.mimeType)) throw Object.assign(new Error("reference object is not a valid image"), { errorCode: "PROVIDER_REFERENCE_INVALID", retryable: false });
|
||||
if (bytes.byteLength > 50 * 1024 * 1024) throw Object.assign(new Error("reference object is too large"), { errorCode: "PROVIDER_REFERENCE_TOO_LARGE", retryable: false });
|
||||
const copy = new Uint8Array(bytes.byteLength);
|
||||
copy.set(bytes);
|
||||
return new Blob([copy.buffer as ArrayBuffer], { type: object.mimeType });
|
||||
}
|
||||
|
||||
async function invokeImage(store: Store, base: URL, secret: string, channel: ProviderChannel, input: ProviderInput) {
|
||||
const count = input.count === undefined ? 1 : input.count;
|
||||
if (!Number.isSafeInteger(count) || count < 1) throw Object.assign(new Error("image count is invalid"), { errorCode: "PROVIDER_INVALID_REQUEST", retryable: false });
|
||||
const rawReferenceImages = input.referenceImages || [];
|
||||
const fallbackReferenceIds = [...new Set(input.references || [])];
|
||||
if (rawReferenceImages.length > chineseImageNumbers.length || fallbackReferenceIds.length > chineseImageNumbers.length) throw Object.assign(new Error("too many reference images"), { errorCode: "PROVIDER_REFERENCE_LIMIT", retryable: false });
|
||||
const orderedReferences = rawReferenceImages.map((item) => {
|
||||
const objectId = typeof item.objectId === "string" ? item.objectId.trim() : "";
|
||||
if (!objectId || !Number.isSafeInteger(item.seq) || item.seq < 1 || item.seq > chineseImageNumbers.length) throw Object.assign(new Error("reference image sequence is invalid"), { errorCode: "PROVIDER_REFERENCE_INVALID", retryable: false });
|
||||
return { ...item, objectId, name: `图片${chineseImageNumbers[item.seq - 1]}` };
|
||||
}).sort((a, b) => a.seq - b.seq);
|
||||
if (new Set(orderedReferences.map((item) => item.seq)).size !== orderedReferences.length) throw Object.assign(new Error("reference image sequences must be unique"), { errorCode: "PROVIDER_REFERENCE_INVALID", retryable: false });
|
||||
const seenReferenceIds = new Set<string>();
|
||||
const uniqueReferences = orderedReferences.filter((item) => {
|
||||
if (seenReferenceIds.has(item.objectId)) return false;
|
||||
seenReferenceIds.add(item.objectId);
|
||||
return true;
|
||||
});
|
||||
const referenceIds = uniqueReferences.length ? uniqueReferences.map((item) => item.objectId) : fallbackReferenceIds;
|
||||
const effectiveReferences = uniqueReferences.length ? uniqueReferences : referenceIds.map((objectId, index) => ({ objectId, seq: index + 1, name: `图片${chineseImageNumbers[index]}` }));
|
||||
const hasReferences = referenceIds.length > 0;
|
||||
const prompt = input.prompt?.trim() || "";
|
||||
const mentionedNames = mentionedImageNames(prompt);
|
||||
const availableNames = new Set(effectiveReferences.map((item) => item.name));
|
||||
for (const name of mentionedNames) if (!availableNames.has(name)) throw Object.assign(new Error("prompt references an unavailable image"), { errorCode: "PROVIDER_REFERENCE_UNAVAILABLE", retryable: false });
|
||||
const rawParams = input.params || {};
|
||||
const params = channel.providerType === "openai-images" ? normalizeImageParams(rawParams, hasReferences) : { ...rawParams };
|
||||
// The channel owns the upstream model identity. Do not let a public task
|
||||
// parameter override the configured resolution mapping.
|
||||
delete params.model;
|
||||
const channelModel = resolveChannelModel(channel, input.publicModelId);
|
||||
const model = channel.providerType === "openai-images" ? imageModelFor(channelModel.model, rawParams.resolution, channelModel.exact ? undefined : channel.resolutionModelMap) : channelModel.model || "configured-model";
|
||||
let rewrittenPrompt = [...effectiveReferences].sort((left, right) => right.name.length - left.name.length).reduce((value, reference) => value.split("@" + reference.name).join("第" + (chineseImageNumbers[reference.seq - 1] || reference.seq) + "张图"), prompt);
|
||||
if (hasReferences && mentionedNames.size === 0) rewrittenPrompt = rewrittenPrompt ? `${rewrittenPrompt}\n基于以下参考图生成。` : "基于以下参考图生成。";
|
||||
let response: Response;
|
||||
if (hasReferences) {
|
||||
const form = new FormData();
|
||||
form.set("model", model);
|
||||
form.set("prompt", rewrittenPrompt);
|
||||
form.set("n", String(count));
|
||||
for (const [index, objectId] of referenceIds.entries()) {
|
||||
const blob = await referenceBlob(store, objectId);
|
||||
const extension = blob.type.split("/")[1] || "png";
|
||||
form.append("image[]", blob, "reference-" + index + "." + extension);
|
||||
}
|
||||
if (input.maskObjectId) form.append("mask", await referenceBlob(store, input.maskObjectId), "mask.png");
|
||||
for (const [key, value] of Object.entries(params)) if (typeof value === "string" || typeof value === "number") form.set(key, String(value));
|
||||
response = await providerFetch(providerEndpoint(base, "v1/images/edits"), secret, { method: "POST", headers: { "Idempotency-Key": input.platformIdempotencyKey || input.taskId }, body: form }, 60_000, input.onRequestAttempted);
|
||||
} else {
|
||||
response = await providerFetch(providerEndpoint(base, "v1/images/generations"), secret, { method: "POST", headers: { "content-type": "application/json", "Idempotency-Key": input.platformIdempotencyKey || input.taskId }, body: JSON.stringify({ model, prompt: rewrittenPrompt, n: count, ...params }) }, 60_000, input.onRequestAttempted);
|
||||
}
|
||||
const providerRequestIdHeader = providerRequestId(response);
|
||||
const data = await jsonResponse(response);
|
||||
const items = Array.isArray(data.data) ? data.data as Array<Record<string, unknown>> : [];
|
||||
const outputs: ProviderOutput[] = [];
|
||||
for (const item of items) {
|
||||
if (typeof item.b64_json === "string") {
|
||||
const decoded = decodeImageBase64(item.b64_json);
|
||||
outputs.push({ mimeType: decoded.mimeType, data: item.b64_json, metadata: imageOutputMetadata(item, data, decoded.mimeType, "base64", decoded.bytes) });
|
||||
} else if (typeof item.url === "string") {
|
||||
const downloaded = await downloadOutput(item.url);
|
||||
outputs.push({ ...downloaded, metadata: imageOutputMetadata(item, data, downloaded.mimeType, "url", Buffer.from(downloaded.data, "base64")) });
|
||||
}
|
||||
}
|
||||
if (!outputs.length) throw Object.assign(new Error("provider returned no image"), { retryable: false, errorCode: "PROVIDER_EMPTY_RESULT" });
|
||||
return { id: typeof data.id === "string" && data.id.trim() ? data.id : providerRequestIdHeader, outputs };
|
||||
}
|
||||
|
||||
async function invokeText(base: URL, secret: string, channel: ProviderChannel, input: ProviderInput) { const model = resolveChannelModel(channel, input.publicModelId).model || "configured-model"; const response = await providerFetch(providerEndpoint(base, "v1/responses"), secret, { method: "POST", headers: { "content-type": "application/json", "Idempotency-Key": input.platformIdempotencyKey || input.taskId }, body: JSON.stringify({ ...input.params, model, input: input.prompt || "" }) }, 60_000, input.onRequestAttempted); const data = await jsonResponse(response); const nested = Array.isArray(data.output) ? (data.output as Array<{ content?: Array<{ text?: string }> }>).flatMap((item) => item.content || []).map((item) => item.text || "").join("") : ""; const text = typeof data.output_text === "string" ? data.output_text : nested; if (!text) throw Object.assign(new Error("provider returned no text"), { retryable: false, errorCode: "PROVIDER_EMPTY_RESULT" }); return { id: typeof data.id === "string" && data.id.trim() ? data.id : providerRequestId(response), outputs: [{ mimeType: "text/plain", data: Buffer.from(text).toString("base64") }] }; }
|
||||
async function invokeAudio(base: URL, secret: string, channel: ProviderChannel, input: ProviderInput) { const model = resolveChannelModel(channel, input.publicModelId).model || "configured-model"; const response = await providerFetch(providerEndpoint(base, "v1/audio/speech"), secret, { method: "POST", headers: { "content-type": "application/json", "Idempotency-Key": input.platformIdempotencyKey || input.taskId }, body: JSON.stringify({ ...input.params, model, input: input.prompt || "" }) }, 60_000, input.onRequestAttempted); if (!response.ok) throw providerHttpError(response.status); const bytes = Buffer.from(await response.arrayBuffer()); if (!bytes.length || bytes.byteLength > 50 * 1024 * 1024) throw new Error("provider audio response invalid"); const mimeType = (response.headers.get("content-type") || "audio/mpeg").split(";")[0].toLowerCase(); if (!/^audio\/(mpeg|wav|ogg|mp4)$/.test(mimeType)) throw new Error("provider audio MIME is not allowed"); return { id: providerRequestId(response), outputs: [{ mimeType, data: bytes.toString("base64") }] }; }
|
||||
|
||||
export async function invokeProvider(store: Store, channel: ProviderChannel, input: ProviderInput): Promise<ProviderResult> {
|
||||
if (!channel.baseUrl || !channel.secretRef) return { status: "failed", errorCode: channel.fixtureFailure === "unknown" ? "PROVIDER_TIMEOUT_UNKNOWN" : "CHANNEL_UNAVAILABLE", retryable: true };
|
||||
try { const secret = decryptSecret(channel.secretRef, store.channelEncryptionKey); if (!secret) return { status: "failed", errorCode: "PROVIDER_SECRET_UNAVAILABLE", retryable: false }; const base = await validateResolvedUrl(channel.baseUrl); const result = input.taskType === "audio" ? await invokeAudio(base, secret, channel, input) : input.taskType === "text" || input.taskType === "reverse-prompt" ? await invokeText(base, secret, channel, input) : await invokeImage(store, base, secret, channel, input); return { status: "succeeded", providerRequestId: result.id, outputs: result.outputs }; }
|
||||
catch (error) {
|
||||
const typed = error as Error & { retryable?: boolean; errorCode?: string };
|
||||
if (typed.name === "AbortError" || typed.name === "TimeoutError") return { status: "unknown", errorCode: "PROVIDER_TIMEOUT_UNKNOWN", retryable: false };
|
||||
// Once a request may have reached the provider, a transport failure is
|
||||
// not safe to retry because it could duplicate a billable generation.
|
||||
if (!typed.errorCode) return { status: "unknown", errorCode: "PROVIDER_TRANSPORT_UNKNOWN", retryable: false };
|
||||
return { status: "failed", errorCode: typed.errorCode, retryable: typed.retryable ?? true };
|
||||
}
|
||||
}
|
||||
|
||||
export async function probeProviderModels(baseUrl: string, secret: string) {
|
||||
let requestAttempted = false;
|
||||
try {
|
||||
const base = await validateResolvedUrl(baseUrl);
|
||||
const response = await providerFetch(providerEndpoint(base, "v1/models"), secret, { method: "GET" }, 15_000, () => { requestAttempted = true; });
|
||||
const data = await jsonResponse(response);
|
||||
const items = Array.isArray(data.data) ? data.data as Array<Record<string, unknown>> : [];
|
||||
const models = items.map((item) => typeof item.id === "string" ? item.id : undefined).filter((value): value is string => Boolean(value));
|
||||
if (!models.length) throw new Error("provider models response is empty");
|
||||
return { healthy: true, models };
|
||||
} catch (error) {
|
||||
const typed = error instanceof Error ? error as Error & { requestAttempted?: boolean } : Object.assign(new Error(String(error)), {} as { requestAttempted?: boolean });
|
||||
typed.requestAttempted = requestAttempted;
|
||||
throw typed;
|
||||
}
|
||||
}
|
||||
|
||||
/** Query/cancel hooks used by the unknown-result reconciler. Providers that do
|
||||
* not expose these endpoints simply remain in manual review. */
|
||||
export async function queryProvider(store: Store, channel: ProviderChannel, providerRequestId: string): Promise<ProviderResult> {
|
||||
if (channel.providerType === "openai-images") return { status: "unknown", providerRequestId, errorCode: "PROVIDER_QUERY_UNAVAILABLE" };
|
||||
if (!channel.baseUrl || !channel.secretRef) return { status: "unknown", providerRequestId, errorCode: "PROVIDER_QUERY_UNAVAILABLE" };
|
||||
try {
|
||||
const secret = decryptSecret(channel.secretRef, store.channelEncryptionKey); if (!secret) return { status: "unknown", providerRequestId, errorCode: "PROVIDER_SECRET_UNAVAILABLE" };
|
||||
const response = await providerFetch(providerEndpoint(await validateResolvedUrl(channel.baseUrl), `v1/requests/${encodeURIComponent(providerRequestId)}`), secret, { method: "GET" }, 15_000);
|
||||
const data = await jsonResponse(response); const status = typeof data.status === "string" ? data.status.toLowerCase() : "";
|
||||
if (["succeeded", "completed", "success"].includes(status)) {
|
||||
const items = Array.isArray(data.data) ? data.data as Array<Record<string, unknown>> : [];
|
||||
const outputs: ProviderOutput[] = [];
|
||||
for (const item of items) {
|
||||
if (typeof item.b64_json === "string") {
|
||||
const decoded = decodeImageBase64(item.b64_json);
|
||||
outputs.push({ mimeType: decoded.mimeType, data: item.b64_json, metadata: imageOutputMetadata(item, data, decoded.mimeType, "base64", decoded.bytes) });
|
||||
} else if (typeof item.url === "string") {
|
||||
const downloaded = await downloadOutput(item.url);
|
||||
outputs.push({ ...downloaded, metadata: imageOutputMetadata(item, data, downloaded.mimeType, "url", Buffer.from(downloaded.data, "base64")) });
|
||||
}
|
||||
}
|
||||
const text = typeof data.output_text === "string" ? data.output_text : typeof data.text === "string" ? data.text : undefined; if (text) outputs.push({ mimeType: "text/plain", data: Buffer.from(text).toString("base64") });
|
||||
if (typeof data.audio === "string") outputs.push({ mimeType: typeof data.mime_type === "string" && /^audio\//.test(data.mime_type) ? data.mime_type : "audio/mpeg", data: data.audio });
|
||||
return { status: outputs.length ? "succeeded" : "unknown", providerRequestId, outputs, errorCode: outputs.length ? undefined : "PROVIDER_QUERY_EMPTY" };
|
||||
}
|
||||
if (["failed", "error", "canceled", "cancelled"].includes(status)) return { status: "failed", providerRequestId, errorCode: status === "canceled" || status === "cancelled" ? "PROVIDER_CANCELED" : "PROVIDER_FAILED", retryable: false };
|
||||
return { status: "unknown", providerRequestId, errorCode: "PROVIDER_STILL_RUNNING" };
|
||||
} catch { return { status: "unknown", providerRequestId, errorCode: "PROVIDER_QUERY_FAILED" }; }
|
||||
}
|
||||
|
||||
export async function cancelProvider(store: Store, channel: ProviderChannel, providerRequestId: string): Promise<ProviderCancelResult> {
|
||||
if (channel.providerType === "openai-images") return "unsupported";
|
||||
if (!channel.baseUrl || !channel.secretRef) return "unsupported";
|
||||
try {
|
||||
const secret = decryptSecret(channel.secretRef, store.channelEncryptionKey); if (!secret) return "unsupported";
|
||||
const response = await providerFetch(providerEndpoint(await validateResolvedUrl(channel.baseUrl), `v1/requests/${encodeURIComponent(providerRequestId)}/cancel`), secret, { method: "POST" }, 15_000);
|
||||
if (response.status === 404 || response.status === 405 || response.status === 501) return "unsupported";
|
||||
if (!response.ok) return "pending";
|
||||
let status = ""; try { const data = JSON.parse(await response.text()) as { status?: unknown }; status = typeof data.status === "string" ? data.status.toLowerCase() : ""; } catch { return "pending"; }
|
||||
return status === "canceled" || status === "cancelled" ? "confirmed" : "pending";
|
||||
} catch { return "pending"; }
|
||||
}
|
||||
|
||||
export function providerSecretIsReference(secret: string) { return Boolean(secret.trim()); }
|
||||
@@ -0,0 +1,158 @@
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import { decryptSecret } from "../shared/auth.ts";
|
||||
import type { Store } from "../store.ts";
|
||||
import type { WebDavSyncMutation } from "../infra/repository.ts";
|
||||
import { resolvePublicHttpsUrl, safeOutboundFetch } from "../infra/outbound-url.ts";
|
||||
|
||||
type WebDavConnection = { url: string; username?: string; password?: string; directory: string };
|
||||
|
||||
function connectionFromStore(store: Store, userId: string, secret: string): WebDavConnection | undefined {
|
||||
const record = store.webdav.get(userId); if (!record) return undefined;
|
||||
return { url: decryptSecret(record.encryptedUrl, secret) || "", username: decryptSecret(record.encryptedUsername, secret), password: decryptSecret(record.encryptedPassword, secret), directory: record.directory };
|
||||
}
|
||||
|
||||
export async function validateWebDavUrl(value: string) {
|
||||
try { return (await resolvePublicHttpsUrl(value)).url; } catch (error) { throw new Error(error instanceof Error ? error.message.replace("出站地址", "WebDAV 地址") : "WebDAV 地址无效"); }
|
||||
}
|
||||
|
||||
function cleanPath(value: string) { const segments = value.replaceAll("\\", "/").split("/").filter(Boolean); if (!segments.length || segments.some((part) => part === "." || part === ".." || part.includes("\0"))) throw new Error("WebDAV 路径无效"); return segments.map(encodeURIComponent).join("/"); }
|
||||
export function validateWebDavPath(value: string) { return cleanPath(value); }
|
||||
|
||||
async function request(connection: WebDavConnection, path: string, init: RequestInit, maxBytes = 20 * 1024 * 1024) {
|
||||
const base = await validateWebDavUrl(connection.url); const root = cleanPath(connection.directory); const relative = path ? cleanPath(path) : ""; base.pathname = `${base.pathname.replace(/\/+$/, "")}/${root}${relative ? `/${relative}` : ""}`;
|
||||
const headers = new Headers(init.headers); if (connection.username || connection.password) headers.set("authorization", `Basic ${Buffer.from(`${connection.username || ""}:${connection.password || ""}`).toString("base64")}`);
|
||||
return safeOutboundFetch(base, { ...init, headers }, { timeoutMs: 15_000, maxBytes });
|
||||
}
|
||||
|
||||
export async function testWebDav(connection: WebDavConnection) { const response = await request(connection, "", { method: "PROPFIND", headers: { depth: "0" } }, 1024 * 1024); if (response.status !== 207 && !response.headers.get("dav")) throw new Error(`WebDAV 能力验证失败 (${response.status})`); const probe = `.miragenflow-probe-${randomUUID()}.json`; const written = await putWebDavFile(connection, probe, Buffer.from("{}"), "application/json", undefined, true); await deleteWebDavFile(connection, probe, written.etag); }
|
||||
export async function getWebDavFile(connection: WebDavConnection, path: string) { const response = await request(connection, path, { method: "GET" }); if (!response.ok) throw new Error(`WebDAV 文件读取失败 (${response.status})`); return { mimeType: response.headers.get("content-type") || "application/octet-stream", etag: response.headers.get("etag") || undefined, data: Buffer.from(await response.arrayBuffer()) }; }
|
||||
export async function putWebDavFile(connection: WebDavConnection, path: string, data: Uint8Array, mimeType: string, ifMatch?: string, ifNoneMatch = false) { const headers: Record<string, string> = { "content-type": mimeType }; if (ifMatch) headers["if-match"] = ifMatch; else if (ifNoneMatch) headers["if-none-match"] = "*"; const response = await request(connection, path, { method: "PUT", headers, body: data as BodyInit }); if (!response.ok) throw new Error(`WebDAV 文件写入失败 (${response.status})`); return { etag: response.headers.get("etag") || undefined }; }
|
||||
export async function deleteWebDavFile(connection: WebDavConnection, path: string, ifMatch?: string) { const headers: Record<string, string> = {}; if (ifMatch) headers["if-match"] = ifMatch; const response = await request(connection, path, { method: "DELETE", headers }); if (!response.ok && response.status !== 404) throw new Error(`WebDAV 文件删除失败 (${response.status})`); }
|
||||
|
||||
/** Queue remote files for retention cleanup. Only files tracked by our own
|
||||
* manifest proxy are eligible; unknown remote files are never guessed or
|
||||
* deleted without a listing/manifest reference. */
|
||||
export function enqueueExpiredWebDavRetention(store: Store, now = Date.now()) {
|
||||
let queued = 0; let changed = false;
|
||||
for (const record of store.webdav.values()) {
|
||||
if (!record.configured || record.retentionState === "deleted" || !record.manifestRetentionExpiresAt || Date.parse(record.manifestRetentionExpiresAt) > now) continue;
|
||||
const files = [...store.webdavFiles.values()].filter((file) => file.userId === record.userId && !file.deletedAt);
|
||||
for (const file of files) {
|
||||
// Terminal jobs do not suppress a later scan: a succeeded job may have
|
||||
// lost its file metadata before the commit, while failed/conflict jobs
|
||||
// must be retried. Only an active queued/running job is a dedupe hit.
|
||||
const existing = [...store.webdavJobs.values()].find((job) => job.userId === record.userId && job.intent === "retention-delete" && job.path === file.path && ["queued", "running"].includes(job.status));
|
||||
if (existing) continue;
|
||||
const id = randomUUID();
|
||||
store.webdavJobs.set(id, { id, userId: record.userId, operation: "delete", intent: "retention-delete", path: file.path, ifMatch: file.etag, attempts: 0, status: "queued", nextAttemptAt: new Date(now).toISOString(), createdAt: new Date(now).toISOString() });
|
||||
queued += 1; changed = true;
|
||||
}
|
||||
const nextRetentionState = files.length ? "deleting" : "deleted";
|
||||
if (record.retentionState !== nextRetentionState) { record.retentionState = nextRetentionState; changed = true; }
|
||||
if (files.length && record.state !== "syncing") { record.state = "syncing"; changed = true; }
|
||||
}
|
||||
if (changed) store.persist();
|
||||
return queued;
|
||||
}
|
||||
|
||||
/** PostgreSQL retention scanning uses a row-locked INSERT/UPDATE transaction
|
||||
* instead of the local full-snapshot writer. Memory/file fixtures retain the
|
||||
* synchronous helper above for compatibility with existing callers/tests. */
|
||||
export async function enqueueExpiredWebDavRetentionAsync(store: Store, now = Date.now()) {
|
||||
if (store.repository.adapter === "postgres" && store.repository.enqueueExpiredWebDavRetention) {
|
||||
return store.repository.enqueueExpiredWebDavRetention(new Date(now).toISOString());
|
||||
}
|
||||
return enqueueExpiredWebDavRetention(store, now);
|
||||
}
|
||||
|
||||
export async function processWebDavJobs(store: Store, secret: string) {
|
||||
let processed = 0;
|
||||
const now = Date.now();
|
||||
const pg = store.repository.adapter === "postgres" && !!store.repository.claimWebDavSyncJobs && !!store.repository.completeWebDavSyncJob;
|
||||
const renew = pg ? store.repository.renewWebDavSyncJob : undefined;
|
||||
const leaseMs = 30_000;
|
||||
const worker = `webdav-${process.pid}-${randomUUID().slice(0, 8)}`;
|
||||
const jobs = pg ? await store.repository.claimWebDavSyncJobs!(worker, leaseMs) : [...store.webdavJobs.values()];
|
||||
if (pg) for (const job of jobs) store.webdavJobs.set(job.id, job);
|
||||
for (const job of jobs) {
|
||||
if (!pg && job.status === "running" && (!job.leaseExpiresAt || Date.parse(job.leaseExpiresAt) <= now)) {
|
||||
job.status = "queued";
|
||||
job.nextAttemptAt = new Date(now).toISOString();
|
||||
job.lastError = "上一次同步进程未完成，已恢复重试";
|
||||
}
|
||||
if (!pg && (job.status !== "queued" || Date.parse(job.nextAttemptAt) > now)) continue;
|
||||
// A user may extend manifest retention after expiry has queued deletion
|
||||
// jobs. Re-check the current expiry immediately before dispatch so a stale
|
||||
// queue item cannot delete a file whose retention was just extended.
|
||||
if (job.intent === "retention-delete") {
|
||||
const record = store.webdav.get(job.userId);
|
||||
if (record?.manifestRetentionExpiresAt && Date.parse(record.manifestRetentionExpiresAt) > now) {
|
||||
if (record.retentionState === "deleting") record.retentionState = "active";
|
||||
if (record.state === "syncing") record.state = "ready";
|
||||
if (pg) { job.status = "succeeded"; job.leaseOwner = undefined; job.leaseExpiresAt = undefined; const completed = await store.repository.completeWebDavSyncJob!(job.id, worker, job, { config: record }); if (!completed) { await store.loadPersisted().catch(() => undefined); } }
|
||||
else { store.webdavJobs.delete(job.id); store.persist(); }
|
||||
continue;
|
||||
}
|
||||
}
|
||||
const connection = connectionFromStore(store, job.userId, secret); if (!connection) { job.status = "failed"; job.lastError = "WebDAV 未配置"; job.leaseOwner = undefined; job.leaseExpiresAt = undefined; if (pg) { const completed = await store.repository.completeWebDavSyncJob!(job.id, worker, job); if (!completed) { await store.loadPersisted().catch(() => undefined); } } else store.persist(); continue; }
|
||||
if (!pg) {
|
||||
job.status = "running";
|
||||
job.leaseExpiresAt = new Date(now + leaseMs).toISOString();
|
||||
store.persist();
|
||||
}
|
||||
let leaseLost = false;
|
||||
const renewTimer = renew ? setInterval(() => {
|
||||
void renew.call(store.repository, job.id, worker, leaseMs).then((result) => {
|
||||
if (!result.renewed) leaseLost = true;
|
||||
else if (result.leaseExpiresAt) job.leaseExpiresAt = result.leaseExpiresAt;
|
||||
}).catch(() => { leaseLost = true; });
|
||||
}, Math.max(1_000, Math.floor(leaseMs / 3))) : undefined;
|
||||
renewTimer?.unref?.();
|
||||
let mutation: WebDavSyncMutation | undefined;
|
||||
try {
|
||||
if (job.operation === "put") {
|
||||
const bytes = Buffer.from(job.data || "", "base64"); const result = await putWebDavFile(connection, job.path, bytes, job.mimeType || "application/octet-stream", job.ifMatch, !job.ifMatch);
|
||||
const checksum = createHash("sha256").update(bytes).digest("hex"); const key = `${job.userId}:${job.path}`; const current = store.webdavFiles.get(key); const file = { userId: job.userId, path: job.path, mimeType: job.mimeType || "application/octet-stream", data: job.data || "", checksum, etag: result.etag || `\"${checksum}\"`, version: (current?.version || 0) + 1, updatedAt: new Date().toISOString(), syncState: "synced" as const }; store.webdavFiles.set(key, file); mutation = { file };
|
||||
if (job.path.endsWith("manifest.json")) { const record = store.webdav.get(job.userId); if (record) { record.manifestEtag = result.etag || `\"${checksum}\"`; record.manifestChecksum = checksum; record.manifestVersion = (record.manifestVersion || 0) + 1; record.manifestRetentionExpiresAt ||= new Date(Date.now() + record.retentionDays * 86_400_000).toISOString(); record.manifestExtensionDays ||= 0; } }
|
||||
} else { await deleteWebDavFile(connection, job.path, job.ifMatch); const current = store.webdavFiles.get(`${job.userId}:${job.path}`); if (current) { current.deletedAt = new Date().toISOString(); current.version += 1; current.etag = `\"deleted-${current.version}\"`; current.data = ""; current.syncState = "synced"; mutation = { file: current }; } }
|
||||
job.status = "succeeded"; job.lastError = undefined; job.leaseOwner = undefined; job.leaseExpiresAt = undefined;
|
||||
if (job.intent === "archive-put") {
|
||||
const taskId = job.path.split("/")[2]; const task = store.tasks.get(taskId); const related = [...store.webdavJobs.values()].filter((item) => item.userId === job.userId && item.intent === "archive-put" && item.path.startsWith(`archive/tasks/${taskId}/`));
|
||||
if (task && related.length && related.every((item) => item.status === "succeeded")) { task.retentionState = "archived"; mutation = { ...(mutation || {}), taskRetention: { taskId, retentionState: "archived", updatedAt: new Date().toISOString() } }; }
|
||||
}
|
||||
if (job.intent === "retention-delete") {
|
||||
const record = store.webdav.get(job.userId);
|
||||
const pending = [...store.webdavJobs.values()].some((item) => item.userId === job.userId && item.intent === "retention-delete" && item.status !== "succeeded");
|
||||
const liveFiles = [...store.webdavFiles.values()].some((item) => item.userId === job.userId && !item.deletedAt);
|
||||
if (record && !pending && !liveFiles) { record.retentionState = "deleted"; record.state = "ready"; }
|
||||
}
|
||||
const record = store.webdav.get(job.userId); if (record) { const pending = [...store.webdavJobs.values()].some((item) => item.userId === job.userId && item.id !== job.id && item.status !== "succeeded"); record.state = pending ? "error" : "ready"; record.lastSyncedAt = new Date().toISOString(); mutation = { ...(mutation || {}), config: record }; }
|
||||
processed += 1;
|
||||
} catch (error) {
|
||||
if (!pg) job.attempts += 1;
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
const conflict = /409|etag|冲突|precondition/i.test(message);
|
||||
job.status = conflict ? "conflict" : job.attempts >= 8 ? "failed" : "queued";
|
||||
if (job.intent === "archive-put" && (job.status === "failed" || job.status === "conflict")) { const taskId = job.path.split("/")[2]; const task = store.tasks.get(taskId); if (task) { task.retentionState = "archive_failed"; mutation = { ...(mutation || {}), taskRetention: { taskId, retentionState: "archive_failed", updatedAt: new Date().toISOString() } }; } }
|
||||
if (job.intent === "retention-delete" && (job.status === "failed" || job.status === "conflict")) { const record = store.webdav.get(job.userId); if (record) { record.retentionState = "error"; record.state = "error"; mutation = { ...(mutation || {}), config: record }; } }
|
||||
if (conflict && job.operation === "put" && !job.path.startsWith(".conflicts/")) {
|
||||
const fileName = job.path.split("/").pop() || "manifest.json";
|
||||
const conflictCopyPath = `.conflicts/${randomUUID()}-${fileName}`;
|
||||
job.conflictCopyPath = conflictCopyPath;
|
||||
const copyJobId = randomUUID();
|
||||
const conflictJob = { id: copyJobId, userId: job.userId, operation: "put" as const, intent: "conflict-copy" as const, path: conflictCopyPath, data: job.data, mimeType: job.mimeType, attempts: 0, status: "queued" as const, nextAttemptAt: new Date().toISOString(), lastError: `冲突副本：${message}`, createdAt: new Date().toISOString() }; store.webdavJobs.set(copyJobId, conflictJob); mutation = { ...(mutation || {}), conflictJob };
|
||||
}
|
||||
job.leaseOwner = undefined;
|
||||
job.leaseExpiresAt = undefined;
|
||||
job.lastError = message;
|
||||
job.nextAttemptAt = new Date(Date.now() + Math.min(60 * 60_000, 2 ** job.attempts * 1000)).toISOString();
|
||||
}
|
||||
if (pg) {
|
||||
const completed = await store.repository.completeWebDavSyncJob!(job.id, worker, job, mutation);
|
||||
if (!completed) { await store.loadPersisted().catch(() => undefined); continue; }
|
||||
} else store.persist();
|
||||
if (leaseLost && pg) await store.loadPersisted().catch(() => undefined);
|
||||
if (renewTimer) clearInterval(renewTimer);
|
||||
}
|
||||
return processed;
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,111 @@
|
||||
import type { IncomingMessage, Server as HttpServer } from "node:http";
|
||||
import { WebSocketServer, type WebSocket } from "ws";
|
||||
import { CONTRACT_VERSION, WS_PATH, ErrorCodes, type ApiError, type TaskEvent, type WsClientMessage } from "@miragenflow/contracts";
|
||||
import type { Store } from "../store.ts";
|
||||
import { hashRefreshToken, verifyAccessToken } from "../shared/auth.ts";
|
||||
import { publicTaskEvent } from "../shared/task-event.ts";
|
||||
|
||||
function send(socket: WebSocket, message: unknown) { if (socket.readyState === socket.OPEN) socket.send(JSON.stringify(message)); }
|
||||
function cookieValue(request: IncomingMessage, name: string) { const raw = request.headers.cookie; if (typeof raw !== "string") return undefined; const part = raw.split(";").map((value) => value.trim()).find((value) => value.startsWith(`${name}=`)); return part ? decodeURIComponent(part.slice(name.length + 1)) : undefined; }
|
||||
type WsIdentity = { userId: string; familyId?: string; refreshHash?: string; issuedAt?: number };
|
||||
function userFromRequest(request: IncomingMessage, store: Store, secret: string, refreshCookieName: string): WsIdentity | undefined {
|
||||
const refresh = cookieValue(request, refreshCookieName);
|
||||
if (refresh) {
|
||||
const refreshHash = hashRefreshToken(refresh); const session = store.sessions.get(refreshHash);
|
||||
const user = session ? store.users.get(session.userId) : undefined;
|
||||
const now = Date.now();
|
||||
const family = session ? store.sessionFamilies.get(session.familyId) : undefined;
|
||||
if (session?.scope === "user" && session.expiresAt > now && session.absoluteExpiresAt > now && !session.revokedAt && !family?.revokedAt && user?.status === "active") return { userId: session.userId, familyId: session.familyId, refreshHash };
|
||||
}
|
||||
const auth = request.headers.authorization;
|
||||
const query = new URL(request.url || WS_PATH, "http://localhost").searchParams.get("access_token");
|
||||
if (query && process.env.NODE_ENV === "production") return undefined;
|
||||
const token = typeof auth === "string" && auth.startsWith("Bearer ") ? auth.slice(7).trim() : query;
|
||||
if (!token) return undefined;
|
||||
const provisional = verifyAccessToken(token, secret, "user");
|
||||
const revokedBefore = store.revokedAccessBefore.get(`user:${provisional.sub}`);
|
||||
const verified = verifyAccessToken(token, secret, "user", revokedBefore);
|
||||
if (verified.sid && store.sessionFamilies.get(verified.sid)?.revokedAt) return undefined;
|
||||
return { userId: verified.sub, familyId: verified.sid, issuedAt: verified.iat };
|
||||
}
|
||||
|
||||
function identityActive(identity: WsIdentity, store: Store) {
|
||||
const user = store.users.get(identity.userId);
|
||||
if (!user || user.status !== "active") return false;
|
||||
if (identity.familyId && store.sessionFamilies.get(identity.familyId)?.revokedAt) return false;
|
||||
if (identity.refreshHash && store.sessions.get(identity.refreshHash)?.revokedAt) return false;
|
||||
if (identity.issuedAt !== undefined) {
|
||||
const revokedBefore = store.revokedAccessBefore.get(`user:${identity.userId}`);
|
||||
if (revokedBefore !== undefined && identity.issuedAt <= revokedBefore) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
export function attachTaskWebSocket(server: HttpServer, store: Store, secret: string, refreshCookieName = "miragenflow_refresh") {
|
||||
const wss = new WebSocketServer({ noServer: true, maxPayload: 64 * 1024 });
|
||||
server.on("upgrade", (request, socket, head) => {
|
||||
if ((request.url || "").split("?")[0] !== WS_PATH) { socket.destroy(); return; }
|
||||
const origin = request.headers.origin;
|
||||
if (typeof origin === "string") {
|
||||
try {
|
||||
if (new URL(origin).host !== request.headers.host) { socket.destroy(); return; }
|
||||
} catch { socket.destroy(); return; }
|
||||
}
|
||||
try {
|
||||
const identity = userFromRequest(request, store, secret, refreshCookieName);
|
||||
if (!identity) throw new Error("missing session");
|
||||
wss.handleUpgrade(request, socket, head, (client) => wss.emit("connection", client, identity));
|
||||
} catch { socket.destroy(); }
|
||||
});
|
||||
wss.on("connection", (socket, identity: WsIdentity) => {
|
||||
const userId = identity.userId;
|
||||
const subscriptions = new Map<string, (event: TaskEvent) => void>();
|
||||
const sentEvents = new Set<string>();
|
||||
const sentSequences = new Map<string, number>();
|
||||
let messageWindowStartedAt = Date.now();
|
||||
let messageCount = 0;
|
||||
let alive = true;
|
||||
const heartbeat = setInterval(() => {
|
||||
if (!alive) { socket.terminate(); return; }
|
||||
if (!identityActive(identity, store)) { socket.close(1008, "session revoked"); return; }
|
||||
alive = false; socket.ping();
|
||||
}, 20_000);
|
||||
send(socket, { type: "ready", contractVersion: CONTRACT_VERSION, heartbeatIntervalMs: 20_000 });
|
||||
socket.on("pong", () => { alive = true; });
|
||||
socket.on("message", (raw) => {
|
||||
try {
|
||||
if (!identityActive(identity, store)) { socket.close(1008, "session revoked"); return; }
|
||||
const now = Date.now();
|
||||
if (now - messageWindowStartedAt >= 10_000) { messageWindowStartedAt = now; messageCount = 0; }
|
||||
if (++messageCount > 100) { socket.close(1008, "message rate exceeded"); return; }
|
||||
const message = JSON.parse(raw.toString()) as WsClientMessage & { type: string };
|
||||
if (message.type === "ping") { alive = true; send(socket, { type: "pong", at: new Date().toISOString() }); return; }
|
||||
if (message.type === "subscribe") {
|
||||
const task = store.tasks.get(message.taskId);
|
||||
if (!task || task.ownerId !== userId) { const error: ApiError = { code: ErrorCodes.Forbidden, message: "无权订阅此任务", requestId: "ws" }; send(socket, { type: "error", error }); return; }
|
||||
const old = subscriptions.get(message.taskId); if (old) store.taskSubscribers.get(message.taskId)?.delete(old);
|
||||
const cursor = Number.isSafeInteger(message.cursor) ? Math.max(0, Number(message.cursor)) : 0;
|
||||
if (subscriptions.size >= 8 && !subscriptions.has(message.taskId)) { socket.close(1008, "subscription limit exceeded"); return; }
|
||||
const sendEvent = (event: TaskEvent) => {
|
||||
const previous = sentSequences.get(event.taskId) || 0;
|
||||
if (event.sequence <= previous || sentEvents.has(event.eventId)) return;
|
||||
sentEvents.add(event.eventId);
|
||||
sentSequences.set(event.taskId, event.sequence);
|
||||
if (sentEvents.size > 5000) sentEvents.delete(sentEvents.values().next().value as string);
|
||||
send(socket, { type: "event", event: publicTaskEvent(event) });
|
||||
};
|
||||
const listener = (event: TaskEvent) => sendEvent(event);
|
||||
const subscribers = store.taskSubscribers.get(message.taskId) || new Set<(event: TaskEvent) => void>(); subscribers.add(listener); store.taskSubscribers.set(message.taskId, subscribers); subscriptions.set(message.taskId, listener);
|
||||
for (const event of (store.events.get(message.taskId) || []).filter((item) => item.sequence > cursor)) sendEvent(event);
|
||||
return;
|
||||
}
|
||||
if (message.type === "unsubscribe") {
|
||||
const listener = subscriptions.get(message.taskId); if (listener) { store.taskSubscribers.get(message.taskId)?.delete(listener); subscriptions.delete(message.taskId); } return;
|
||||
}
|
||||
const error: ApiError = { code: ErrorCodes.ValidationFailed, message: "不支持的 WebSocket 消息类型", requestId: "ws" }; send(socket, { type: "error", error });
|
||||
} catch { const error: ApiError = { code: ErrorCodes.ValidationFailed, message: "WebSocket 消息格式错误", requestId: "ws" }; send(socket, { type: "error", error }); }
|
||||
});
|
||||
socket.once("close", () => { clearInterval(heartbeat); for (const [taskId, listener] of subscriptions) store.taskSubscribers.get(taskId)?.delete(listener); subscriptions.clear(); });
|
||||
});
|
||||
return wss;
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
import { createHmac } from "node:crypto";
|
||||
|
||||
export type ServerConfig = {
|
||||
host: string;
|
||||
port: number;
|
||||
accessTokenSecret: string;
|
||||
accessTokenTtlSeconds: number;
|
||||
refreshCookieName: string;
|
||||
corsOrigin?: string;
|
||||
adminEmail: string;
|
||||
adminPassword: string;
|
||||
adminMfaRequired: boolean;
|
||||
adminSelfApprovalAllowed?: boolean;
|
||||
adminMfaSecret?: string;
|
||||
adminCaptchaRequired?: boolean;
|
||||
userMfaRequired?: boolean;
|
||||
userVerificationRequired?: boolean;
|
||||
emailRegistrationVerificationRequired?: boolean;
|
||||
phoneRegistrationVerificationRequired?: boolean;
|
||||
captchaRequired?: boolean;
|
||||
cookieSecure?: boolean;
|
||||
csrfCookieName?: string;
|
||||
maxBodyBytes?: number;
|
||||
stagingDir?: string;
|
||||
storeFile?: string;
|
||||
rateLimitWindowMs?: number;
|
||||
rateLimitMax?: number;
|
||||
adminRoles?: string[];
|
||||
databaseUrl?: string;
|
||||
redisUrl?: string;
|
||||
persistenceAdapter?: "memory" | "file" | "postgres";
|
||||
queueAdapter?: "memory" | "redis";
|
||||
stagingTtlSeconds?: number;
|
||||
channelEncryptionKey?: string;
|
||||
adminRole?: string;
|
||||
refreshAbsoluteTtlSeconds?: number;
|
||||
paymentWebhookSecret?: string;
|
||||
adminAccounts?: Array<{ id: string; email: string; password: string; role: string; mfaSecret?: string; mfaRequired: boolean; recoveryCodeHashes?: string[]; mfaLastTotpCounter?: number }>;
|
||||
};
|
||||
|
||||
function positiveInt(value: string | undefined, fallback: number) {
|
||||
const parsed = Number(value);
|
||||
return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback;
|
||||
}
|
||||
|
||||
function localMfaSecret(secret: string) {
|
||||
const alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567";
|
||||
const bytes = createHmac("sha256", secret).update("miragenflow-admin-mfa").digest().subarray(0, 20);
|
||||
let output = "";
|
||||
let buffer = 0;
|
||||
let bits = 0;
|
||||
for (const byte of bytes) {
|
||||
buffer = (buffer << 8) | byte;
|
||||
bits += 8;
|
||||
while (bits >= 5) {
|
||||
output += alphabet[(buffer >>> (bits - 5)) & 31];
|
||||
bits -= 5;
|
||||
}
|
||||
}
|
||||
if (bits > 0) output += alphabet[(buffer << (5 - bits)) & 31];
|
||||
return output;
|
||||
}
|
||||
|
||||
function adminAccounts(env: NodeJS.ProcessEnv, fallback: { email: string; password: string; role: string; mfaSecret: string; mfaRequired: boolean }) {
|
||||
if (!env.MIRAGENFLOW_ADMIN_ACCOUNTS_JSON?.trim()) return [{ id: `admin:${fallback.email.toLowerCase()}`, ...fallback }];
|
||||
const parsed = JSON.parse(env.MIRAGENFLOW_ADMIN_ACCOUNTS_JSON) as Array<Record<string, unknown>>;
|
||||
if (!Array.isArray(parsed) || !parsed.length) throw new Error("MIRAGENFLOW_ADMIN_ACCOUNTS_JSON must contain at least one administrator");
|
||||
return parsed.map((item, index) => {
|
||||
const email = typeof item.email === "string" ? item.email.trim().toLowerCase() : "";
|
||||
const password = typeof item.password === "string" ? item.password : "";
|
||||
if (!email || password.length < 12) throw new Error(`administrator account ${index + 1} is invalid`);
|
||||
return { id: typeof item.id === "string" && item.id.trim() ? item.id.trim() : `admin:${email}`, email, password, role: typeof item.role === "string" ? item.role : "operator", mfaSecret: typeof item.mfaSecret === "string" ? item.mfaSecret.trim() : undefined, mfaRequired: item.mfaRequired !== false, recoveryCodeHashes: Array.isArray(item.recoveryCodeHashes) ? item.recoveryCodeHashes.filter((value): value is string => typeof value === "string") : undefined, mfaLastTotpCounter: Number.isInteger(item.mfaLastTotpCounter) ? Number(item.mfaLastTotpCounter) : undefined };
|
||||
});
|
||||
}
|
||||
|
||||
export function loadConfig(env = process.env): ServerConfig {
|
||||
const isProduction = env.NODE_ENV === "production";
|
||||
const secret = env.MIRAGENFLOW_ACCESS_TOKEN_SECRET?.trim();
|
||||
const adminPassword = env.MIRAGENFLOW_ADMIN_PASSWORD;
|
||||
if (isProduction && (!secret || secret.length < 32)) throw new Error("MIRAGENFLOW_ACCESS_TOKEN_SECRET must be at least 32 characters in production");
|
||||
if (isProduction && (!adminPassword || adminPassword.length < 12)) throw new Error("MIRAGENFLOW_ADMIN_PASSWORD must be set in production");
|
||||
if (isProduction && env.MIRAGENFLOW_ADMIN_MFA_REQUIRED !== "false" && !env.MIRAGENFLOW_ADMIN_MFA_SECRET?.trim()) throw new Error("MIRAGENFLOW_ADMIN_MFA_SECRET must be set when administrator MFA is enabled");
|
||||
if (isProduction && !env.MIRAGENFLOW_CHANNEL_ENCRYPTION_KEY?.trim()) throw new Error("MIRAGENFLOW_CHANNEL_ENCRYPTION_KEY must be set in production");
|
||||
if (isProduction && env.MIRAGENFLOW_PERSISTENCE_ADAPTER !== "postgres") throw new Error("MIRAGENFLOW_PERSISTENCE_ADAPTER=postgres is required in production");
|
||||
if (isProduction && !(env.DATABASE_URL || env.MIRAGENFLOW_DATABASE_URL)) throw new Error("DATABASE_URL is required in production");
|
||||
if (isProduction && env.MIRAGENFLOW_QUEUE_ADAPTER !== "redis") throw new Error("MIRAGENFLOW_QUEUE_ADAPTER=redis is required in production");
|
||||
if (isProduction && !(env.REDIS_URL || env.MIRAGENFLOW_REDIS_URL)) throw new Error("REDIS_URL is required in production");
|
||||
const effectiveSecret = secret || "local-development-secret-change-me";
|
||||
// Administrator login stays lightweight by default. MFA/CAPTCHA remain
|
||||
// available as explicit security switches for deployments that enable them.
|
||||
const defaultAdminMfaRequired = env.MIRAGENFLOW_ADMIN_MFA_REQUIRED === "true";
|
||||
const defaultAdminMfaSecret = env.MIRAGENFLOW_ADMIN_MFA_SECRET?.trim() || localMfaSecret(effectiveSecret);
|
||||
const configuredAdminAccounts = adminAccounts(env, { email: env.MIRAGENFLOW_ADMIN_EMAIL?.trim().toLowerCase() || "admin@admin.com", password: adminPassword || "admin", role: env.MIRAGENFLOW_ADMIN_ROLE?.trim() || "super_admin", mfaSecret: defaultAdminMfaSecret, mfaRequired: defaultAdminMfaRequired }).map((account) => ({ ...account, mfaSecret: account.mfaSecret || localMfaSecret(`${effectiveSecret}:${account.email}`) }));
|
||||
return {
|
||||
host: env.MIRAGENFLOW_HOST?.trim() || "127.0.0.1",
|
||||
port: positiveInt(env.MIRAGENFLOW_PORT, 3100),
|
||||
accessTokenSecret: effectiveSecret,
|
||||
accessTokenTtlSeconds: positiveInt(env.MIRAGENFLOW_ACCESS_TOKEN_TTL_SECONDS, 900),
|
||||
refreshCookieName: env.MIRAGENFLOW_REFRESH_COOKIE?.trim() || "miragenflow_refresh",
|
||||
corsOrigin: env.MIRAGENFLOW_CORS_ORIGIN?.trim() || undefined,
|
||||
adminEmail: env.MIRAGENFLOW_ADMIN_EMAIL?.trim() || "admin@admin.com",
|
||||
adminPassword: adminPassword || "admin",
|
||||
adminMfaRequired: defaultAdminMfaRequired,
|
||||
adminSelfApprovalAllowed: !isProduction && env.MIRAGENFLOW_ADMIN_SELF_APPROVAL === "true",
|
||||
adminCaptchaRequired: env.MIRAGENFLOW_ADMIN_CAPTCHA_REQUIRED === "true",
|
||||
adminMfaSecret: defaultAdminMfaSecret,
|
||||
adminAccounts: configuredAdminAccounts,
|
||||
// Ordinary-user verification is an optional enhancement. The default
|
||||
// preview/production policy is email+password only; deployments may turn
|
||||
// CAPTCHA, registration verification, or user MFA on independently.
|
||||
userMfaRequired: env.MIRAGENFLOW_USER_MFA_REQUIRED === "true",
|
||||
userVerificationRequired: env.MIRAGENFLOW_USER_VERIFICATION_REQUIRED === "true",
|
||||
emailRegistrationVerificationRequired: env.MIRAGENFLOW_EMAIL_REGISTRATION_VERIFICATION_REQUIRED === undefined ? undefined : env.MIRAGENFLOW_EMAIL_REGISTRATION_VERIFICATION_REQUIRED === "true",
|
||||
phoneRegistrationVerificationRequired: env.MIRAGENFLOW_PHONE_REGISTRATION_VERIFICATION_REQUIRED === undefined ? undefined : env.MIRAGENFLOW_PHONE_REGISTRATION_VERIFICATION_REQUIRED === "true",
|
||||
captchaRequired: env.MIRAGENFLOW_CAPTCHA_REQUIRED === "true",
|
||||
// Secure cookies require an HTTPS public origin. Production keeps the
|
||||
// secure default, while explicit false is useful for the HTTP local
|
||||
// compose profile (and must not be swallowed by an `|| isProduction`
|
||||
// expression).
|
||||
cookieSecure: env.MIRAGENFLOW_COOKIE_SECURE === undefined ? isProduction : env.MIRAGENFLOW_COOKIE_SECURE === "true",
|
||||
csrfCookieName: env.MIRAGENFLOW_CSRF_COOKIE?.trim() || "miragenflow_csrf",
|
||||
maxBodyBytes: positiveInt(env.MIRAGENFLOW_MAX_BODY_BYTES, 10 * 1024 * 1024),
|
||||
stagingDir: env.MIRAGENFLOW_STAGING_DIR?.trim() || "/tmp/miragenflow-staging",
|
||||
storeFile: env.MIRAGENFLOW_STORE_FILE?.trim() || undefined,
|
||||
rateLimitWindowMs: positiveInt(env.MIRAGENFLOW_RATE_LIMIT_WINDOW_MS, 60_000),
|
||||
rateLimitMax: positiveInt(env.MIRAGENFLOW_RATE_LIMIT_MAX, 12),
|
||||
adminRoles: (env.MIRAGENFLOW_ADMIN_ROLES || "super_admin,operator,finance,support,auditor").split(",").map((role) => role.trim()).filter(Boolean),
|
||||
databaseUrl: env.DATABASE_URL?.trim() || env.MIRAGENFLOW_DATABASE_URL?.trim() || undefined,
|
||||
redisUrl: env.REDIS_URL?.trim() || env.MIRAGENFLOW_REDIS_URL?.trim() || undefined,
|
||||
persistenceAdapter: env.MIRAGENFLOW_PERSISTENCE_ADAPTER === "postgres" && (env.DATABASE_URL || env.MIRAGENFLOW_DATABASE_URL) ? "postgres" : env.MIRAGENFLOW_STORE_FILE?.trim() ? "file" : "memory",
|
||||
queueAdapter: env.MIRAGENFLOW_QUEUE_ADAPTER === "redis" && (env.REDIS_URL || env.MIRAGENFLOW_REDIS_URL) ? "redis" : "memory",
|
||||
stagingTtlSeconds: positiveInt(env.MIRAGENFLOW_STAGING_TTL_SECONDS, 7 * 24 * 60 * 60),
|
||||
channelEncryptionKey: env.MIRAGENFLOW_CHANNEL_ENCRYPTION_KEY?.trim() || effectiveSecret,
|
||||
adminRole: env.MIRAGENFLOW_ADMIN_ROLE?.trim() || "super_admin",
|
||||
refreshAbsoluteTtlSeconds: positiveInt(env.MIRAGENFLOW_REFRESH_ABSOLUTE_TTL_SECONDS, 90 * 24 * 60 * 60),
|
||||
paymentWebhookSecret: env.MIRAGENFLOW_PAYMENT_WEBHOOK_SECRET?.trim() || effectiveSecret,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,192 @@
|
||||
import type { BalanceLedgerEntry, GenerationTask, TaskEvent } from "@miragenflow/contracts";
|
||||
import type { AdminApprovalRecord, MessageOutboxRecord, PaymentEventRecord, RechargeOrder, Store } from "../store.ts";
|
||||
|
||||
const DAY = 24 * 60 * 60 * 1000;
|
||||
|
||||
function timestamp(daysAgo: number, hour: number, minute = 0) {
|
||||
const value = new Date(Date.now() - daysAgo * DAY);
|
||||
value.setHours(hour, minute, 0, 0);
|
||||
return value.toISOString();
|
||||
}
|
||||
|
||||
function addTask(store: Store, task: GenerationTask, events: TaskEvent[]) {
|
||||
if (!store.tasks.has(task.id)) store.tasks.set(task.id, task);
|
||||
if (!store.events.has(task.id)) store.events.set(task.id, events);
|
||||
}
|
||||
|
||||
function addLedger(store: Store, entry: BalanceLedgerEntry) {
|
||||
if (!store.ledger.some((item) => item.id === entry.id)) store.ledger.push(entry);
|
||||
}
|
||||
|
||||
function addChannelAwareAttempt(channelId: string, status: "succeeded" | "failed" | "unknown", startedAt: string, finishedAt: string, sequence: number) {
|
||||
return { id: `demo-attempt-${sequence}-${status}`, channelId, sequence, status, startedAt, finishedAt, retryable: status === "failed", ...(status === "unknown" ? { reconciliationStatus: "pending" as const } : {}) };
|
||||
}
|
||||
|
||||
function taskEvents(task: GenerationTask, terminalType: TaskEvent["type"], terminalAt: string): TaskEvent[] {
|
||||
return [
|
||||
{ eventId: `${task.id}-event-1`, sequence: 1, taskId: task.id, type: "task.queued", occurredAt: task.createdAt, payload: { status: "queued" } },
|
||||
{ eventId: `${task.id}-event-2`, sequence: 2, taskId: task.id, type: "task.running", occurredAt: new Date(Date.parse(task.createdAt) + 30_000).toISOString(), payload: { status: "running" } },
|
||||
{ eventId: `${task.id}-event-3`, sequence: 3, taskId: task.id, type: terminalType, occurredAt: terminalAt, payload: { status: task.status } },
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Seeds operational records for the local admin preview only. It deliberately
|
||||
* references existing users/channels instead of creating showcase identities or
|
||||
* channels, so destructive actions remain meaningful during manual testing.
|
||||
*/
|
||||
export function ensureAdminDemoData(store: Store) {
|
||||
// Demo records are an explicit opt-in. A normal local server must start
|
||||
// from persisted user data and user-created channels, not silently seed a
|
||||
// showcase dashboard that can be mistaken for real business data.
|
||||
if (process.env.NODE_ENV === "production" || process.env.NODE_ENV === "test" || process.env.MIRAGENFLOW_DEMO_DATA !== "true") return;
|
||||
// PostgreSQL data must satisfy foreign keys and must come from real workflows.
|
||||
if (store.repository.adapter === "postgres") return;
|
||||
|
||||
const ownerId = [...store.users.keys()][0] || "demo-user-reference";
|
||||
const channelId = store.channels[0]?.id || "demo-channel-reference";
|
||||
const channelGroupId = store.channelGroups[0]?.id || "image-default";
|
||||
const imageProduct = store.products.find((item) => item.capabilities.includes("image")) || store.products[0];
|
||||
const textProduct = store.products.find((item) => item.capabilities.includes("text")) || imageProduct;
|
||||
const imageModelId = imageProduct?.id || "basic-image";
|
||||
const imagePublicModelId = imageProduct?.publicModelId || "basic-image-v1";
|
||||
const textModelId = textProduct?.id || "basic-text";
|
||||
const textPublicModelId = textProduct?.publicModelId || "basic-text-v1";
|
||||
|
||||
const successStarted = timestamp(1, 9, 12);
|
||||
const successFinished = timestamp(1, 9, 13);
|
||||
const successTask: GenerationTask = {
|
||||
id: "demo-task-success",
|
||||
ownerId,
|
||||
taskType: "image",
|
||||
modelProductId: imageModelId,
|
||||
status: "succeeded",
|
||||
estimatedCost: 10,
|
||||
reservedCost: 0,
|
||||
createdAt: timestamp(1, 9),
|
||||
updatedAt: successFinished,
|
||||
eventSequence: 3,
|
||||
channelGroupId,
|
||||
routeSnapshotVersion: 1,
|
||||
attempts: [addChannelAwareAttempt(channelId, "succeeded", successStarted, successFinished, 1)],
|
||||
outputs: [{ id: "demo-output-success", mimeType: "image/png", status: "succeeded", unitPriceSnapshot: 10, chargedAmount: 10, metadata: { width: 1024, height: 1024, format: "png", resolution: "1K", source: "url" } }],
|
||||
prompt: "产品演示插画：明亮的工作室和一张白色桌面",
|
||||
resolutionPresetId: "low",
|
||||
count: 1,
|
||||
pricingSnapshot: { basePrice: 10, multiplier: 1, unitVersion: store.balanceUnit.version },
|
||||
modelSnapshot: { publicModelId: imagePublicModelId, version: imageProduct?.version || 1 },
|
||||
routeSnapshot: { groupId: channelGroupId, version: 1, channelIds: store.channels.length ? [channelId] : [] },
|
||||
planSnapshot: { planId: store.plans[0]?.id, version: store.plans[0]?.version, queuePriority: 0, maxConcurrent: 2 },
|
||||
};
|
||||
addTask(store, successTask, taskEvents(successTask, "task.succeeded", successFinished));
|
||||
|
||||
const partialStarted = timestamp(2, 14, 5);
|
||||
const partialFinished = timestamp(2, 14, 7);
|
||||
const partialTask: GenerationTask = {
|
||||
...successTask,
|
||||
id: "demo-task-partial",
|
||||
status: "partial",
|
||||
createdAt: timestamp(2, 14),
|
||||
updatedAt: partialFinished,
|
||||
attempts: [
|
||||
addChannelAwareAttempt(channelId, "succeeded", partialStarted, timestamp(2, 14, 6), 1),
|
||||
addChannelAwareAttempt(channelId, "failed", timestamp(2, 14, 6), partialFinished, 2),
|
||||
],
|
||||
outputs: [{ id: "demo-output-partial", mimeType: "image/png", status: "succeeded", unitPriceSnapshot: 10, chargedAmount: 10, metadata: { width: 1024, height: 1024, format: "png", resolution: "1K", source: "url" } }],
|
||||
prompt: "部分成功示例：一组双色几何海报",
|
||||
};
|
||||
addTask(store, partialTask, taskEvents(partialTask, "task.succeeded", partialFinished));
|
||||
|
||||
const failedStarted = timestamp(3, 16, 20);
|
||||
const failedFinished = timestamp(3, 16, 21);
|
||||
const failedTask: GenerationTask = {
|
||||
...successTask,
|
||||
id: "demo-task-failed",
|
||||
status: "failed",
|
||||
createdAt: timestamp(3, 16),
|
||||
updatedAt: failedFinished,
|
||||
attempts: [addChannelAwareAttempt(channelId, "failed", failedStarted, failedFinished, 1)],
|
||||
outputs: [],
|
||||
publicErrorCode: "INTERNAL_ERROR",
|
||||
prompt: "失败示例：供应商超时重试",
|
||||
};
|
||||
addTask(store, failedTask, taskEvents(failedTask, "task.failed", failedFinished));
|
||||
|
||||
const canceledFinished = timestamp(4, 11, 45);
|
||||
const canceledTask: GenerationTask = {
|
||||
...successTask,
|
||||
id: "demo-task-canceled",
|
||||
status: "canceled",
|
||||
taskType: "text",
|
||||
modelProductId: textModelId,
|
||||
createdAt: timestamp(4, 11),
|
||||
updatedAt: canceledFinished,
|
||||
attempts: [],
|
||||
outputs: [],
|
||||
prompt: "已取消示例：产品说明摘要",
|
||||
modelSnapshot: { publicModelId: textPublicModelId, version: textProduct?.version || 1 },
|
||||
};
|
||||
addTask(store, canceledTask, taskEvents(canceledTask, "task.canceled", canceledFinished));
|
||||
|
||||
const unknownFinished = timestamp(5, 13, 4);
|
||||
const unknownTask: GenerationTask = {
|
||||
...successTask,
|
||||
id: "demo-task-unknown",
|
||||
status: "unknown",
|
||||
createdAt: timestamp(5, 13),
|
||||
updatedAt: unknownFinished,
|
||||
attempts: [addChannelAwareAttempt(channelId, "unknown", timestamp(5, 13, 2), unknownFinished, 1)],
|
||||
outputs: [],
|
||||
publicErrorCode: "CONFLICT",
|
||||
prompt: "待对账示例：供应商返回状态不明确",
|
||||
};
|
||||
addTask(store, unknownTask, taskEvents(unknownTask, "task.unknown", unknownFinished));
|
||||
|
||||
addLedger(store, { id: "demo-ledger-recharge", userId: ownerId, type: "recharge", amount: 500, balanceUnitVersion: store.balanceUnit.version, idempotencyKey: "demo-ledger-recharge", createdAt: timestamp(6, 10), referenceId: "demo-recharge-paid" });
|
||||
addLedger(store, { id: "demo-ledger-recharge-today", userId: ownerId, type: "recharge", amount: 350, balanceUnitVersion: store.balanceUnit.version, idempotencyKey: "demo-ledger-recharge-today", createdAt: timestamp(0, 10, 2), referenceId: "demo-recharge-today" });
|
||||
addLedger(store, { id: "demo-ledger-settle", userId: ownerId, type: "settle", amount: -10, balanceUnitVersion: store.balanceUnit.version, idempotencyKey: "demo-ledger-settle", createdAt: timestamp(2, 14), referenceId: "demo-task-success", taskId: "demo-task-success" });
|
||||
addLedger(store, { id: "demo-ledger-adjustment", userId: ownerId, type: "adjustment", amount: 80, balanceUnitVersion: store.balanceUnit.version, idempotencyKey: "demo-ledger-adjustment", createdAt: timestamp(3, 8), referenceId: "demo-approval-executed" });
|
||||
addLedger(store, { id: "demo-ledger-refund", userId: ownerId, type: "refund", amount: -40, balanceUnitVersion: store.balanceUnit.version, idempotencyKey: "demo-ledger-refund", createdAt: timestamp(4, 17), referenceId: "demo-recharge-refunded" });
|
||||
|
||||
const addRecharge = (order: RechargeOrder) => { if (!store.rechargeOrders.has(order.id)) store.rechargeOrders.set(order.id, order); };
|
||||
addRecharge({ id: "demo-recharge-paid", userId: ownerId, provider: "mock", status: "paid", fiatAmount: 50, balanceAmount: 500, unitVersion: store.balanceUnit.version, idempotencyKey: "demo-recharge-paid", createdAt: timestamp(6, 9), paidAt: timestamp(6, 10) });
|
||||
addRecharge({ id: "demo-recharge-today", userId: ownerId, provider: "mock", status: "paid", fiatAmount: 35, balanceAmount: 350, unitVersion: store.balanceUnit.version, idempotencyKey: "demo-recharge-today", createdAt: timestamp(0, 10), paidAt: timestamp(0, 10, 2) });
|
||||
addRecharge({ id: "demo-recharge-pending", userId: ownerId, provider: "mock", status: "pending", fiatAmount: 20, balanceAmount: 200, unitVersion: store.balanceUnit.version, idempotencyKey: "demo-recharge-pending", createdAt: timestamp(0, 15) });
|
||||
addRecharge({ id: "demo-recharge-refunded", userId: ownerId, provider: "mock", status: "refunded", fiatAmount: 40, balanceAmount: 400, unitVersion: store.balanceUnit.version, idempotencyKey: "demo-recharge-refunded", createdAt: timestamp(7, 10), paidAt: timestamp(7, 11) });
|
||||
|
||||
const addPayment = (key: string, item: PaymentEventRecord) => { if (!store.paymentEvents.has(key)) store.paymentEvents.set(key, item); };
|
||||
addPayment("mock:demo-payment-applied", { id: "demo-payment-applied", provider: "mock", externalEventId: "demo-payment-applied", orderId: "demo-recharge-paid", status: "applied", payloadDigest: "demo-digest-applied", createdAt: timestamp(6, 10), appliedAt: timestamp(6, 10, 1) });
|
||||
addPayment("mock:demo-payment-today", { id: "demo-payment-today", provider: "mock", externalEventId: "demo-payment-today", orderId: "demo-recharge-today", status: "applied", payloadDigest: "demo-digest-today", createdAt: timestamp(0, 10, 2), appliedAt: timestamp(0, 10, 3) });
|
||||
addPayment("mock:demo-payment-received", { id: "demo-payment-received", provider: "mock", externalEventId: "demo-payment-received", orderId: "demo-recharge-pending", status: "received", payloadDigest: "demo-digest-received", createdAt: timestamp(0, 15) });
|
||||
addPayment("mock:demo-payment-rejected", { id: "demo-payment-rejected", provider: "mock", externalEventId: "demo-payment-rejected", orderId: "demo-recharge-refunded", status: "rejected", payloadDigest: "demo-digest-rejected", createdAt: timestamp(2, 18) });
|
||||
|
||||
const requesterId = [...store.adminAccounts.keys()][0] || "admin";
|
||||
const addApproval = (item: AdminApprovalRecord) => { if (!store.adminApprovals.has(item.id)) store.adminApprovals.set(item.id, item); };
|
||||
addApproval({ id: "demo-approval-pending", kind: "manual_recharge", requesterId, status: "pending", payload: { userId: ownerId, balanceAmount: 300, fiatAmount: 30 }, reason: "演示待审批的人工充值申请", version: 1, createdAt: timestamp(0, 9), expiresAt: timestamp(-1, 9) });
|
||||
addApproval({ id: "demo-approval-executed", kind: "high_risk_change", requesterId, approverId: requesterId, status: "executed", payload: { type: "balance_adjustment", userId: ownerId, amount: 80 }, reason: "演示已执行的余额调整", version: 2, createdAt: timestamp(3, 8), expiresAt: timestamp(-2, 8), approvedAt: timestamp(3, 8, 2), executedAt: timestamp(3, 8, 3) });
|
||||
addApproval({ id: "demo-approval-rejected", kind: "refund", requesterId, approverId: requesterId, status: "rejected", payload: { orderId: "demo-recharge-refunded" }, reason: "演示已驳回的退款申请", version: 2, createdAt: timestamp(5, 17), expiresAt: timestamp(-1, 17), approvedAt: timestamp(5, 18) });
|
||||
|
||||
const addMessage = (item: MessageOutboxRecord) => { if (!store.messageOutbox.has(item.id)) store.messageOutbox.set(item.id, item); };
|
||||
addMessage({ id: "demo-message-sent", idempotencyKey: "demo-message-sent", channel: "email", targetHash: "demo-target-sent", purpose: "register", status: "sent", attempts: 1, providerMessageId: "demo-provider-message", createdAt: timestamp(2, 10), updatedAt: timestamp(2, 10, 1) });
|
||||
addMessage({ id: "demo-message-failed", idempotencyKey: "demo-message-failed", channel: "sms", targetHash: "demo-target-failed", purpose: "reset", status: "dead", attempts: 5, lastError: "供应商连接超时", createdAt: timestamp(1, 18), updatedAt: timestamp(1, 18, 3) });
|
||||
addMessage({ id: "demo-message-dead", idempotencyKey: "demo-message-dead", channel: "email", targetHash: "demo-target-dead", purpose: "notification", status: "dead", attempts: 5, lastError: "供应商返回拒绝", createdAt: timestamp(4, 20), updatedAt: timestamp(4, 20, 2) });
|
||||
const failedMessage = store.messageOutbox.get("demo-message-failed");
|
||||
if (failedMessage && failedMessage.status !== "dead") { failedMessage.status = "dead"; failedMessage.attempts = Math.max(5, failedMessage.attempts); failedMessage.lastError = "供应商连接超时"; failedMessage.nextAttemptAt = undefined; }
|
||||
|
||||
if (!store.messageProviders.has("demo-email")) store.messageProviders.set("demo-email", { id: "demo-email", channel: "email", name: "演示邮件供应商", enabled: false, priority: 0, endpoint: "https://example.com/email", secretConfigured: false, version: 1 });
|
||||
if (!store.messageProviders.has("demo-sms")) store.messageProviders.set("demo-sms", { id: "demo-sms", channel: "sms", name: "演示短信供应商", enabled: false, priority: 1, endpoint: "https://example.com/sms", secretConfigured: false, version: 1 });
|
||||
if (!store.messageTemplates.has("demo-register-template")) store.messageTemplates.set("demo-register-template", { id: "demo-register-template", purpose: "register", channel: "email", content: "你的验证码是 {{code}}，10 分钟内有效。", enabled: true, version: 1 });
|
||||
if (!store.messageTemplates.has("demo-reset-template")) store.messageTemplates.set("demo-reset-template", { id: "demo-reset-template", purpose: "reset", channel: "sms", content: "密码重置验证码：{{code}}。", enabled: true, version: 1 });
|
||||
|
||||
const addAudit = (id: string, action: string, objectType: string, objectId: string, createdAt: string, actor = requesterId) => {
|
||||
if (store.audit.some((entry) => entry.id === id)) return;
|
||||
store.audit.push({ id, actorId: actor, action, objectType, objectId, requestId: `demo-request-${id}`, createdAt });
|
||||
};
|
||||
addAudit("demo-audit-user-update", "user.update", "user", ownerId, timestamp(1, 8));
|
||||
addAudit("demo-audit-task-failed", "task.failed", "generation_task", "demo-task-failed", timestamp(3, 16), "system");
|
||||
addAudit("demo-audit-payment-rejected", "payment.webhook.rejected", "recharge_order", "demo-recharge-refunded", timestamp(2, 18), "payment:webhook");
|
||||
addAudit("demo-audit-message-failed", "message.retry.failed", "message_outbox", "demo-message-failed", timestamp(1, 18), "system");
|
||||
addAudit("demo-audit-channel-test", "channel.test", "provider_channel", channelId, timestamp(0, 11));
|
||||
|
||||
try { store.persist(); } catch (error) { console.warn("后台演示数据未能持久化", error); }
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
export const OPENAI_IMAGE_RATIOS = ["auto", "1:1", "16:9", "9:16", "3:4", "4:5", "3:2", "2:3", "4:3", "1.91:1", "2.35:1", "21:9"] as const;
|
||||
export const OPENAI_IMAGE_RESOLUTIONS = ["1K", "2K", "4K"] as const;
|
||||
export type OpenAiImageRatio = (typeof OPENAI_IMAGE_RATIOS)[number];
|
||||
export type OpenAiImageResolution = (typeof OPENAI_IMAGE_RESOLUTIONS)[number];
|
||||
|
||||
const ratioParts: Record<Exclude<OpenAiImageRatio, "auto">, [number, number]> = {
|
||||
"1:1": [1, 1], "16:9": [16, 9], "9:16": [9, 16], "3:4": [3, 4], "4:5": [4, 5],
|
||||
"3:2": [3, 2], "2:3": [2, 3], "4:3": [4, 3], "1.91:1": [191, 100], "2.35:1": [235, 100], "21:9": [21, 9],
|
||||
};
|
||||
|
||||
function align16(value: number) { return Math.max(16, Math.round(value / 16) * 16); }
|
||||
function sizeForRatio(width: number, height: number, resolution: OpenAiImageResolution) {
|
||||
const longEdge = resolution === "1K" ? 1024 : resolution === "2K" ? 2048 : 3840;
|
||||
let nextWidth = width >= height ? longEdge : Math.round(longEdge * width / height);
|
||||
let nextHeight = width >= height ? Math.round(longEdge * height / width) : longEdge;
|
||||
nextWidth = align16(nextWidth); nextHeight = align16(nextHeight);
|
||||
if (resolution === "4K") {
|
||||
const budget = 8_294_400;
|
||||
while (nextWidth * nextHeight > budget) { nextWidth = align16(nextWidth * 0.99); nextHeight = align16(nextHeight * 0.99); }
|
||||
}
|
||||
return `${nextWidth}x${nextHeight}`;
|
||||
}
|
||||
|
||||
function validExplicitImageSize(value: unknown): value is string {
|
||||
if (typeof value !== "string") return false;
|
||||
const match = /^(\d+)x(\d+)$/.exec(value);
|
||||
if (!match) return false;
|
||||
const width = Number(match[1]);
|
||||
const height = Number(match[2]);
|
||||
const aspect = Math.max(width, height) / Math.min(width, height);
|
||||
return Number.isSafeInteger(width) && Number.isSafeInteger(height)
|
||||
&& width >= 16 && height >= 16 && width <= 3840 && height <= 3840
|
||||
&& width % 16 === 0 && height % 16 === 0
|
||||
&& width * height >= 400_000 && width * height <= 8_294_400 && aspect <= 3;
|
||||
}
|
||||
|
||||
const standardSizes: Record<OpenAiImageResolution, Record<Exclude<OpenAiImageRatio, "auto">, string>> = Object.fromEntries(
|
||||
OPENAI_IMAGE_RESOLUTIONS.map((resolution) => [resolution, Object.fromEntries(Object.entries(ratioParts).map(([ratio, [width, height]]) => [ratio, sizeForRatio(width, height, resolution)]))]),
|
||||
) as Record<OpenAiImageResolution, Record<Exclude<OpenAiImageRatio, "auto">, string>>;
|
||||
|
||||
export function imageSizeFor(ratio: unknown, resolution: unknown) {
|
||||
if (ratio === undefined || ratio === "auto" || !OPENAI_IMAGE_RATIOS.includes(ratio as OpenAiImageRatio)) return "auto";
|
||||
const tier = OPENAI_IMAGE_RESOLUTIONS.includes(resolution as OpenAiImageResolution) ? resolution as OpenAiImageResolution : "1K";
|
||||
return standardSizes[tier][ratio as Exclude<OpenAiImageRatio, "auto">];
|
||||
}
|
||||
|
||||
export function imageModelFor(baseModel: string | undefined, resolution: unknown, mapping?: Record<string, string>) {
|
||||
const tier = OPENAI_IMAGE_RESOLUTIONS.includes(resolution as OpenAiImageResolution) ? resolution as OpenAiImageResolution : "1K";
|
||||
return mapping?.[tier]?.trim() || baseModel?.trim() || "gpt-image-2";
|
||||
}
|
||||
|
||||
export function normalizeImageParams(params: Record<string, unknown> = {}, hasReferences = false) {
|
||||
const next = { ...params } as Record<string, unknown>;
|
||||
const ratio = typeof next.ratio === "string" ? next.ratio : "auto";
|
||||
const resolution = typeof next.resolution === "string" ? next.resolution : "1K";
|
||||
next.size = validExplicitImageSize(next.size) ? next.size : imageSizeFor(ratio, resolution);
|
||||
// Keep the public contract deterministic. OpenAI-compatible gateways often
|
||||
// apply different defaults, so the platform sends the documented values
|
||||
// explicitly instead of relying on each upstream implementation.
|
||||
if (next.background === undefined) next.background = "auto";
|
||||
if (next.quality === undefined) next.quality = "auto";
|
||||
if (next.moderation === undefined) next.moderation = "auto";
|
||||
if (next.output_format === undefined) next.output_format = "png";
|
||||
if (hasReferences && next.input_fidelity === undefined) next.input_fidelity = "high";
|
||||
delete next.ratio;
|
||||
delete next.resolution;
|
||||
if (!hasReferences) delete next.input_fidelity;
|
||||
const outputFormat = typeof next.output_format === "string" ? next.output_format.toLowerCase() : undefined;
|
||||
if (outputFormat) next.output_format = outputFormat;
|
||||
if (next.background === "transparent" && outputFormat === "jpeg") next.output_format = "png";
|
||||
return next;
|
||||
}
|
||||
|
||||
export function openAiImageCapabilityParameters(hasReferences = false) {
|
||||
const parameters: Record<string, { type: string; enum?: string[] }> = {
|
||||
ratio: { type: "string", enum: [...OPENAI_IMAGE_RATIOS] },
|
||||
resolution: { type: "string", enum: [...OPENAI_IMAGE_RESOLUTIONS] },
|
||||
size: { type: "string" },
|
||||
quality: { type: "string", enum: ["auto", "low", "medium", "high"] },
|
||||
background: { type: "string", enum: ["auto", "opaque", "transparent"] },
|
||||
moderation: { type: "string", enum: ["auto", "low"] },
|
||||
output_format: { type: "string", enum: ["png", "jpeg", "webp"] },
|
||||
};
|
||||
if (hasReferences) parameters.input_fidelity = { type: "string", enum: ["low", "high"] };
|
||||
return parameters;
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
import { createHttpServer } from "./app/http.ts";
|
||||
import { attachTaskWebSocket } from "./app/ws.ts";
|
||||
import { loadConfig } from "./config.ts";
|
||||
import { createStore } from "./store.ts";
|
||||
import { runMigrations } from "./infra/migrations.ts";
|
||||
import { ensureStagingDir, gcStaging } from "./infra/staging.ts";
|
||||
import { rm } from "node:fs/promises";
|
||||
import { safeStagingPath } from "./infra/staging.ts";
|
||||
import { dispatchTaskOutbox, reconcileUnknownTasks, recoverExpiredReservationsAsync, recoverExpiredTaskLeasesAsync, resumeQueuedTasks } from "./jobs/task-worker.ts";
|
||||
import { processMessageOutbox, recoverMessageOutbox } from "./adapters/messaging.ts";
|
||||
import { enqueueExpiredWebDavRetentionAsync, processWebDavJobs } from "./adapters/webdav.ts";
|
||||
import { ensureAdminDemoData } from "./fixtures/admin-demo.ts";
|
||||
|
||||
const config = loadConfig();
|
||||
await runMigrations(config.databaseUrl);
|
||||
const store = createStore(config.storeFile, { persistenceAdapter: config.persistenceAdapter, queueAdapter: config.queueAdapter, databaseUrl: config.databaseUrl, redisUrl: config.redisUrl, stagingDir: config.stagingDir, channelEncryptionKey: config.channelEncryptionKey });
|
||||
await store.loadPersisted();
|
||||
await recoverExpiredTaskLeasesAsync(store);
|
||||
recoverMessageOutbox(store);
|
||||
void processMessageOutbox(store);
|
||||
await enqueueExpiredWebDavRetentionAsync(store);
|
||||
await ensureStagingDir(store.stagingDir);
|
||||
void resumeQueuedTasks(store);
|
||||
void dispatchTaskOutbox(store);
|
||||
const stagingGc = setInterval(() => {
|
||||
const now = Date.now();
|
||||
void gcStaging(store.stagingDir, (config.stagingTtlSeconds || 7 * 24 * 60 * 60) * 1000, now, (key) => [...store.objects.values()].some((object) => object.stagingKey === key && !object.revoked && Date.parse(object.expiresAt) > now));
|
||||
}, 60 * 60 * 1000);
|
||||
const objectGc = setInterval(() => { void (async () => { const now = Date.now(); for (const [objectId, object] of store.objects) { if (!object.revoked && Date.parse(object.expiresAt) > now) continue; if (object.stagingKey) await rm(safeStagingPath(store.stagingDir, object.stagingKey), { force: true }); store.objects.delete(objectId); const upload = store.uploads.get(objectId); if (upload) upload.status = "deleted"; } store.persist(); })(); }, 60 * 60 * 1000);
|
||||
const taskWatchdog = setInterval(() => { void recoverExpiredTaskLeasesAsync(store); void recoverExpiredReservationsAsync(store); void reconcileUnknownTasks(store); void resumeQueuedTasks(store); }, 5_000);
|
||||
const taskDispatch = setInterval(() => { void dispatchTaskOutbox(store); }, 5_000);
|
||||
taskWatchdog.unref();
|
||||
taskDispatch.unref();
|
||||
objectGc.unref();
|
||||
const messageWorker = setInterval(() => { recoverMessageOutbox(store); void processMessageOutbox(store); }, 5_000);
|
||||
messageWorker.unref();
|
||||
const webdavWorker = setInterval(() => { void processWebDavJobs(store, config.accessTokenSecret); }, 5_000);
|
||||
const webdavRetention = setInterval(() => { void enqueueExpiredWebDavRetentionAsync(store); }, 60 * 60 * 1000);
|
||||
webdavWorker.unref();
|
||||
webdavRetention.unref();
|
||||
stagingGc.unref();
|
||||
const server = createHttpServer(store, config);
|
||||
ensureAdminDemoData(store);
|
||||
attachTaskWebSocket(server, store, config.accessTokenSecret, config.refreshCookieName);
|
||||
|
||||
server.listen(config.port, config.host, () => {
|
||||
console.log(`MiragenFlow server listening on http://${config.host}:${config.port}`);
|
||||
});
|
||||
|
||||
function shutdown() {
|
||||
clearInterval(stagingGc);
|
||||
clearInterval(objectGc);
|
||||
clearInterval(taskWatchdog);
|
||||
clearInterval(taskDispatch);
|
||||
clearInterval(messageWorker);
|
||||
clearInterval(webdavWorker);
|
||||
clearInterval(webdavRetention);
|
||||
server.close(() => process.exit(0));
|
||||
}
|
||||
|
||||
process.on("SIGINT", shutdown);
|
||||
process.on("SIGTERM", shutdown);
|
||||
@@ -0,0 +1,52 @@
|
||||
import { readdir, readFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
type QueryClient = { query: (sql: string, values?: unknown[]) => Promise<{ rows: unknown[] }>; end?: () => Promise<void> };
|
||||
|
||||
async function optionalPgClient(databaseUrl: string): Promise<QueryClient | undefined> {
|
||||
try {
|
||||
const dynamicImport = new Function("specifier", "return import(specifier)") as (specifier: string) => Promise<{ default?: { Client: new (options: { connectionString: string }) => QueryClient } }>;
|
||||
const module = await dynamicImport("pg");
|
||||
const Client = module.default?.Client;
|
||||
if (!Client) return undefined;
|
||||
const client = new Client({ connectionString: databaseUrl });
|
||||
await (client as QueryClient & { connect: () => Promise<void> }).connect();
|
||||
return client;
|
||||
} catch { return undefined; }
|
||||
}
|
||||
|
||||
export async function runMigrations(databaseUrl: string | undefined) {
|
||||
if (!databaseUrl) return { applied: 0, skipped: true };
|
||||
const client = await optionalPgClient(databaseUrl);
|
||||
if (!client) throw new Error("DATABASE_URL is configured but the pg adapter is unavailable");
|
||||
await client.query("SELECT pg_advisory_lock(hashtext('miragenflow:migrations'))");
|
||||
try {
|
||||
await client.query("CREATE TABLE IF NOT EXISTS schema_migrations (version TEXT PRIMARY KEY, applied_at TIMESTAMPTZ NOT NULL DEFAULT now())");
|
||||
const directory = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../../migrations");
|
||||
const files = (await readdir(directory)).filter((file) => file.endsWith(".sql")).sort();
|
||||
const knownVersions = new Set(files.map((file) => file.replace(/\.sql$/, "")));
|
||||
const existing = await client.query("SELECT version FROM schema_migrations");
|
||||
const unknown = existing.rows.map((row) => String((row as { version?: unknown }).version || "")).filter((version) => version && !knownVersions.has(version));
|
||||
if (unknown.length) throw new Error(`unknown schema migration versions: ${unknown.join(", ")}`);
|
||||
let applied = 0;
|
||||
for (const file of files) {
|
||||
const version = file.replace(/\.sql$/, "");
|
||||
await client.query("BEGIN");
|
||||
try {
|
||||
const result = await client.query("SELECT version FROM schema_migrations WHERE version = $1 FOR UPDATE", [version]);
|
||||
if (!result.rows.length) {
|
||||
const sql = await readFile(path.join(directory, file), "utf8");
|
||||
await client.query(sql);
|
||||
await client.query("INSERT INTO schema_migrations(version) VALUES ($1) ON CONFLICT (version) DO NOTHING", [version]);
|
||||
applied += 1;
|
||||
}
|
||||
await client.query("COMMIT");
|
||||
} catch (error) { await client.query("ROLLBACK"); throw error; }
|
||||
}
|
||||
return { applied, skipped: false };
|
||||
} finally {
|
||||
await client.query("SELECT pg_advisory_unlock(hashtext('miragenflow:migrations'))").catch(() => undefined);
|
||||
await client.end?.().catch(() => undefined);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
import { lookup } from "node:dns/promises";
|
||||
import { isIP } from "node:net";
|
||||
import ipaddr from "ipaddr.js";
|
||||
import { Agent } from "undici";
|
||||
|
||||
type ResolvedAddress = { address: string; family?: number };
|
||||
export type PublicTarget = { url: URL; address: string; family: 4 | 6 };
|
||||
export type OutboundResolver = (hostname: string) => Promise<ResolvedAddress[]>;
|
||||
export type OutboundUrlOptions = { allowPrivateNetwork?: boolean };
|
||||
|
||||
function embeddedIpv4(bytes: number[], offset: number) {
|
||||
return `${bytes[offset]}.${bytes[offset + 1]}.${bytes[offset + 2]}.${bytes[offset + 3]}`;
|
||||
}
|
||||
|
||||
export function isPublicAddress(value: string): boolean {
|
||||
try {
|
||||
const address = ipaddr.parse(value.replace(/^\[|\]$/g, ""));
|
||||
if (address.kind() === "ipv4") return address.range() === "unicast";
|
||||
const ipv6 = address as ipaddr.IPv6;
|
||||
if (ipv6.isIPv4MappedAddress()) return isPublicAddress(ipv6.toIPv4Address().toString());
|
||||
if (ipv6.range() !== "unicast") return false;
|
||||
const bytes = ipv6.toByteArray();
|
||||
const nat64 = ipv6.match(ipaddr.parse("64:ff9b::") as ipaddr.IPv6, 96);
|
||||
const sixToFour = ipv6.match(ipaddr.parse("2002::") as ipaddr.IPv6, 16);
|
||||
if (nat64 && !isPublicAddress(embeddedIpv4(bytes, 12))) return false;
|
||||
if (sixToFour && !isPublicAddress(embeddedIpv4(bytes, 2))) return false;
|
||||
return true;
|
||||
} catch { return false; }
|
||||
}
|
||||
|
||||
function configuredAllowlist() {
|
||||
return (process.env.MIRAGENFLOW_OUTBOUND_HOST_ALLOWLIST || "").split(",").map((item) => item.trim().toLowerCase()).filter(Boolean);
|
||||
}
|
||||
|
||||
export async function resolvePublicHttpsUrl(value: string | URL, resolver: OutboundResolver = async (hostname) => lookup(hostname, { all: true }), options: OutboundUrlOptions = {}) {
|
||||
const url = value instanceof URL ? new URL(value) : new URL(value);
|
||||
const allowPrivateNetwork = options.allowPrivateNetwork === true;
|
||||
const protocolAllowed = url.protocol === "https:" || (allowPrivateNetwork && url.protocol === "http:");
|
||||
if (!protocolAllowed || url.username || url.password) throw new Error(allowPrivateNetwork ? "出站地址必须是无内嵌凭证的 HTTP 或 HTTPS URL" : "出站地址必须是无内嵌凭证的 HTTPS URL");
|
||||
const allowlist = configuredAllowlist();
|
||||
const hostname = url.hostname.replace(/^\[|\]$/g, "").toLowerCase();
|
||||
if (allowlist.length && !allowlist.some((allowed) => hostname === allowed || hostname.endsWith(`.${allowed}`))) throw new Error("出站地址不在允许的域名范围内");
|
||||
const addresses = isIP(hostname) ? [{ address: hostname, family: isIP(hostname) }] : await resolver(hostname);
|
||||
if (!addresses.length || (!allowPrivateNetwork && addresses.some((item) => !isPublicAddress(item.address)))) throw new Error("出站地址不允许指向本机、私有或保留网络");
|
||||
const selected = addresses[0]; const family = (selected.family || isIP(selected.address)) as 4 | 6;
|
||||
if (family !== 4 && family !== 6) throw new Error("出站地址解析结果无效");
|
||||
return { url, address: selected.address, family } satisfies PublicTarget;
|
||||
}
|
||||
|
||||
export async function safeOutboundFetch(value: string | URL, init: RequestInit = {}, options: { maxBytes?: number; timeoutMs?: number; resolver?: OutboundResolver; allowPrivateNetwork?: boolean; onRequestAttempted?: () => void } = {}) {
|
||||
const target = await resolvePublicHttpsUrl(value, options.resolver, { allowPrivateNetwork: options.allowPrivateNetwork });
|
||||
// Node switches to the multi-address callback contract when `all: true` is
|
||||
// passed to dns.lookup. Undici uses that mode for connect race/fallback, so
|
||||
// returning a string here makes every real request fail with
|
||||
// ERR_INVALID_IP_ADDRESS before a socket is opened.
|
||||
const lookupPinned = ((_hostname: string, lookupOptions: { all?: boolean }, callback: (error: Error | null, address: string | Array<{ address: string; family: number }>, family?: number) => void) => {
|
||||
if (lookupOptions?.all) callback(null, [{ address: target.address, family: target.family }]);
|
||||
else callback(null, target.address, target.family);
|
||||
}) as never;
|
||||
const dispatcher = new Agent({ connect: { lookup: lookupPinned, servername: target.url.hostname } });
|
||||
const timeoutMs = options.timeoutMs || 15_000; const maxBytes = options.maxBytes || 20 * 1024 * 1024;
|
||||
try {
|
||||
// Use the WHATWG fetch implementation so FormData/Blob bodies used by the
|
||||
// image-edit adapter are encoded correctly. The custom dispatcher still
|
||||
// pins DNS to the address validated above and fetch is explicitly told not
|
||||
// to follow redirects.
|
||||
options.onRequestAttempted?.();
|
||||
const response = await fetch(target.url, { ...init, redirect: "error", dispatcher, signal: init.signal || AbortSignal.timeout(timeoutMs) } as RequestInit & { dispatcher: Agent });
|
||||
if (response.status >= 300 && response.status < 400) throw new Error(`出站请求拒绝重定向 (${response.status})`);
|
||||
if (!response.body) return response;
|
||||
const reader = response.body.getReader(); const chunks: Buffer[] = []; let total = 0;
|
||||
while (true) {
|
||||
const next = await reader.read(); if (next.done) break;
|
||||
const bytes = Buffer.from(next.value); total += bytes.byteLength;
|
||||
if (total > maxBytes) { await reader.cancel(); throw new Error("出站响应超过大小限制"); }
|
||||
chunks.push(bytes);
|
||||
}
|
||||
return new Response(Buffer.concat(chunks), { status: response.status, headers: response.headers });
|
||||
} finally { await dispatcher.close(); }
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
|
||||
export type QueueHealth = { adapter: "memory" | "redis"; status: "ready" | "degraded" | "unavailable"; detail?: string };
|
||||
export type QueueLease = { taskId: string; leaseToken: string; attempt?: number };
|
||||
export type QueueEnqueueOptions = { maxAttempts?: number; priority?: number };
|
||||
export type QueueNackOptions = { retry?: boolean; maxAttempts?: number };
|
||||
|
||||
/** Optional methods keep existing custom adapters source-compatible. */
|
||||
export interface TaskQueue {
|
||||
readonly adapter: QueueHealth["adapter"];
|
||||
enqueue(taskId: string, options?: QueueEnqueueOptions): Promise<void>;
|
||||
claim(workerId: string, leaseMs: number): Promise<QueueLease | undefined>;
|
||||
ack(taskId: string, leaseToken: string): Promise<void>;
|
||||
renew?(taskId: string, leaseToken: string, leaseMs: number): Promise<boolean>;
|
||||
heartbeat?(taskId: string, leaseToken: string, leaseMs: number): Promise<boolean>;
|
||||
nack?(taskId: string, leaseToken: string, options?: QueueNackOptions): Promise<boolean>;
|
||||
deadLetters?(): Promise<string[]>;
|
||||
health(): Promise<QueueHealth>;
|
||||
}
|
||||
|
||||
const DEFAULT_MAX_ATTEMPTS = 5;
|
||||
|
||||
export class MemoryTaskQueue implements TaskQueue {
|
||||
readonly adapter = "memory" as const;
|
||||
private readonly pending: Array<{ taskId: string; priority: number; enqueuedAt: number }> = [];
|
||||
private readonly leases = new Map<string, { workerId: string; token: string; expiresAt: number }>();
|
||||
private readonly attempts = new Map<string, number>();
|
||||
private readonly maxAttempts = new Map<string, number>();
|
||||
private readonly priorities = new Map<string, number>();
|
||||
private readonly dead = new Set<string>();
|
||||
|
||||
async enqueue(taskId: string, options: QueueEnqueueOptions = {}) {
|
||||
if (this.dead.has(taskId)) this.dead.delete(taskId);
|
||||
if (options.maxAttempts && options.maxAttempts > 0) this.maxAttempts.set(taskId, options.maxAttempts);
|
||||
if (Number.isFinite(options.priority)) this.priorities.set(taskId, Number(options.priority));
|
||||
if (!this.pending.some((item) => item.taskId === taskId) && !this.leases.has(taskId)) this.pending.push({ taskId, priority: Number.isFinite(options.priority) ? Number(options.priority) : 0, enqueuedAt: Date.now() });
|
||||
}
|
||||
private recoverExpired(now: number) {
|
||||
for (const [taskId, lease] of this.leases) {
|
||||
if (lease.expiresAt > now) continue;
|
||||
this.leases.delete(taskId);
|
||||
const count = this.attempts.get(taskId) || 0;
|
||||
if (count >= (this.maxAttempts.get(taskId) || DEFAULT_MAX_ATTEMPTS)) this.dead.add(taskId);
|
||||
else if (!this.pending.some((item) => item.taskId === taskId)) this.pending.push({ taskId, priority: this.priorities.get(taskId) || 0, enqueuedAt: now });
|
||||
}
|
||||
}
|
||||
async claim(workerId: string, leaseMs: number) {
|
||||
const now = Date.now(); this.recoverExpired(now);
|
||||
let taskId: string | undefined;
|
||||
this.pending.sort((a, b) => a.priority - b.priority || a.enqueuedAt - b.enqueuedAt);
|
||||
while (this.pending.length && !taskId) { const candidate = this.pending.shift()!; if (!this.dead.has(candidate.taskId)) taskId = candidate.taskId; }
|
||||
if (!taskId) return undefined;
|
||||
const leaseToken = `${workerId}:${randomUUID()}`;
|
||||
const attempt = (this.attempts.get(taskId) || 0) + 1; this.attempts.set(taskId, attempt);
|
||||
this.leases.set(taskId, { workerId, token: leaseToken, expiresAt: now + leaseMs });
|
||||
return { taskId, leaseToken, attempt };
|
||||
}
|
||||
async renew(taskId: string, leaseToken: string, leaseMs: number) { const lease = this.leases.get(taskId); if (!lease || lease.token !== leaseToken || lease.expiresAt <= Date.now()) return false; lease.expiresAt = Date.now() + leaseMs; return true; }
|
||||
async heartbeat(taskId: string, leaseToken: string, leaseMs: number) { return this.renew(taskId, leaseToken, leaseMs); }
|
||||
async ack(taskId: string, leaseToken: string) { const lease = this.leases.get(taskId); if (!lease || lease.token !== leaseToken || lease.expiresAt <= Date.now()) return; this.leases.delete(taskId); }
|
||||
async nack(taskId: string, leaseToken: string, options: QueueNackOptions = {}) { const lease = this.leases.get(taskId); if (!lease || lease.token !== leaseToken || lease.expiresAt <= Date.now()) return false; this.leases.delete(taskId); const count = this.attempts.get(taskId) || 0; const max = options.maxAttempts || this.maxAttempts.get(taskId) || DEFAULT_MAX_ATTEMPTS; if (options.retry !== false && count < max) this.pending.push({ taskId, priority: this.priorities.get(taskId) || 0, enqueuedAt: Date.now() }); else this.dead.add(taskId); return true; }
|
||||
async deadLetters() { return [...this.dead]; }
|
||||
async health(): Promise<QueueHealth> { return { adapter: this.adapter, status: "ready" }; }
|
||||
}
|
||||
|
||||
/** Redis is optional; a configured but unavailable adapter reports unavailable health. */
|
||||
export class RedisTaskQueue implements TaskQueue {
|
||||
readonly adapter = "redis" as const;
|
||||
private client?: { rpush(key: string, value: string): Promise<number>; eval(script: string, numKeys: number, ...args: Array<string | number>): Promise<unknown>; ping(): Promise<string> };
|
||||
private connectPromise?: Promise<typeof this.client>;
|
||||
constructor(readonly url: string, private readonly prefix = "miragenflow:tasks") {}
|
||||
private async connect() { if (!this.connectPromise) this.connectPromise = import("ioredis").then((module) => { const Redis = (module.default || module.Redis) as unknown as new (url: string) => NonNullable<typeof this.client>; return new Redis(this.url); }).catch(() => undefined); this.client ||= await this.connectPromise; if (!this.client) throw new Error("Redis queue adapter unavailable: install ioredis and verify REDIS_URL"); return this.client; }
|
||||
private pendingKey() { return `${this.prefix}:pending`; } private leasesKey() { return `${this.prefix}:leases`; } private deadKey() { return `${this.prefix}:dead`; } private attemptsKey() { return `${this.prefix}:attempts`; } private leasePrefix() { return `${this.prefix}:lease:`; } private priorityKey() { return `${this.prefix}:priority`; }
|
||||
async enqueue(taskId: string, options: QueueEnqueueOptions = {}) { const client = await this.connect(); const priority = Number.isFinite(options.priority) ? Number(options.priority) : 0; await client.eval("if redis.call('EXISTS', KEYS[3]) == 1 then return 0 end; redis.call('SREM', KEYS[4], ARGV[2]); redis.call('ZADD', KEYS[1], ARGV[1], ARGV[2]); redis.call('HSET', KEYS[2], ARGV[2], ARGV[3]); return 1", 4, this.pendingKey(), this.priorityKey(), `${this.leasePrefix()}${taskId}`, this.deadKey(), priority * 1000000000000000 + Date.now(), taskId, priority); if (options.maxAttempts && options.maxAttempts > 0) await client.eval("redis.call('HSET', KEYS[1], ARGV[1], ARGV[2]); return 1", 1, `${this.prefix}:max-attempts`, taskId, options.maxAttempts); }
|
||||
async claim(workerId: string, leaseMs: number) {
|
||||
const client = await this.connect(); const token = `${workerId}:${randomUUID()}`;
|
||||
const script = `local now=tonumber(ARGV[1]); local ttl=tonumber(ARGV[2]); local expired=redis.call('ZRANGEBYSCORE',KEYS[2],'-inf',now); for _,id in ipairs(expired) do local lk=KEYS[5]..id; local count=tonumber(redis.call('HGET',KEYS[4],id) or '0'); local max=tonumber(redis.call('HGET',KEYS[7],id) or '${DEFAULT_MAX_ATTEMPTS}'); redis.call('DEL',lk); redis.call('ZREM',KEYS[2],id); if count >= max then redis.call('SADD',KEYS[3],id); else local p=tonumber(redis.call('HGET',KEYS[8],id) or '0'); redis.call('ZADD',KEYS[1],p*1000000000000000+now,id); end end; local ids=redis.call('ZRANGE',KEYS[1],0,0); local id=ids[1]; if not id then return nil end; redis.call('ZREM',KEYS[1],id); local attempt=redis.call('HINCRBY',KEYS[4],id,1); local lk=KEYS[5]..id; redis.call('SET',lk,ARGV[3],'PX',ttl); redis.call('ZADD',KEYS[2],now+ttl,id); return {id,ARGV[3],attempt}`;
|
||||
const result = await client.eval(script, 8, this.pendingKey(), this.leasesKey(), this.deadKey(), this.attemptsKey(), this.leasePrefix(), `${this.prefix}:max-attempts`, `${this.prefix}:max-attempts`, this.priorityKey(), Date.now(), leaseMs, token) as [string, string, number] | null;
|
||||
return result ? { taskId: String(result[0]), leaseToken: String(result[1]), attempt: Number(result[2]) } : undefined;
|
||||
}
|
||||
async renew(taskId: string, leaseToken: string, leaseMs: number) { const client = await this.connect(); const script = "local k=KEYS[1]; if redis.call('GET',k) ~= ARGV[1] or redis.call('PTTL',k) <= 0 then return 0 end; redis.call('PEXPIRE',k,ARGV[2]); redis.call('ZADD',KEYS[2],ARGV[3]+ARGV[2],ARGV[4]); return 1"; return Number(await client.eval(script, 2, `${this.leasePrefix()}${taskId}`, this.leasesKey(), leaseToken, leaseMs, Date.now(), taskId)) === 1; }
|
||||
async heartbeat(taskId: string, leaseToken: string, leaseMs: number) { return this.renew(taskId, leaseToken, leaseMs); }
|
||||
async ack(taskId: string, leaseToken: string) { const client = await this.connect(); await client.eval("if redis.call('GET',KEYS[1]) ~= ARGV[1] then return 0 end; redis.call('DEL',KEYS[1]); redis.call('ZREM',KEYS[2],ARGV[2]); return 1", 2, `${this.leasePrefix()}${taskId}`, this.leasesKey(), leaseToken, taskId); }
|
||||
async nack(taskId: string, leaseToken: string, options: QueueNackOptions = {}) { const client = await this.connect(); const script = "if redis.call('GET',KEYS[1]) ~= ARGV[1] then return 0 end; redis.call('DEL',KEYS[1]); redis.call('ZREM',KEYS[2],ARGV[2]); local n=tonumber(redis.call('HGET',KEYS[4],ARGV[2]) or '0'); local m=tonumber(ARGV[3]); if ARGV[4]=='1' and n < m then local p=tonumber(redis.call('HGET',KEYS[6],ARGV[2]) or '0'); redis.call('ZADD',KEYS[3],p*1000000000000000+ARGV[5],ARGV[2]); else redis.call('SADD',KEYS[5],ARGV[2]); end; return 1"; return Number(await client.eval(script, 6, `${this.leasePrefix()}${taskId}`, this.leasesKey(), this.pendingKey(), this.attemptsKey(), this.deadKey(), this.priorityKey(), leaseToken, taskId, options.maxAttempts || DEFAULT_MAX_ATTEMPTS, options.retry === false ? "0" : "1", Date.now())) === 1; }
|
||||
async deadLetters() { const client = await this.connect(); const result = await client.eval("return redis.call('SMEMBERS',KEYS[1])", 1, this.deadKey()); return (result as string[]).map(String); }
|
||||
async health(): Promise<QueueHealth> { try { const client = await this.connect(); await client.ping(); return { adapter: this.adapter, status: "ready" }; } catch (error) { return { adapter: this.adapter, status: "unavailable", detail: error instanceof Error ? error.message : String(error) }; } }
|
||||
}
|
||||
|
||||
export function createTaskQueue(options: { adapter?: "memory" | "redis"; redisUrl?: string } = {}): TaskQueue { return options.adapter === "redis" && options.redisUrl ? new RedisTaskQueue(options.redisUrl) : new MemoryTaskQueue(); }
|
||||
@@ -0,0 +1,722 @@
|
||||
import { mkdir, readFile, rename, writeFile } from "node:fs/promises";
|
||||
import { dirname } from "node:path";
|
||||
import { createHash } from "node:crypto";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import type { GenerationTask } from "@miragenflow/contracts";
|
||||
import type { WebDavFileRecord, WebDavRecord, WebDavSyncJob } from "../store.ts";
|
||||
import { ErrorCodes } from "@miragenflow/contracts";
|
||||
import { DomainError } from "../shared/errors.ts";
|
||||
import type { MessageOutboxRecord } from "../store.ts";
|
||||
|
||||
export type RepositoryHealth = { adapter: "memory" | "file" | "postgres"; status: "ready" | "degraded" | "unavailable"; detail?: string };
|
||||
export type AtomicTaskInput = { userId: string; task: GenerationTask; estimatedCost: number; unitVersion: number; maxConcurrent: number; preferredBucketId?: string; requestFingerprint: string };
|
||||
export type AtomicTaskResult = { created: boolean; taskId: string; task?: GenerationTask };
|
||||
export type MessageOutboxCompletion = { success: boolean; providerMessageId?: string; error?: string; nextAttemptAt?: string };
|
||||
export type WebDavSyncMutation = {
|
||||
file?: WebDavFileRecord;
|
||||
config?: WebDavRecord;
|
||||
taskRetention?: { taskId: string; retentionState: string; updatedAt: string };
|
||||
conflictJob?: WebDavSyncJob;
|
||||
};
|
||||
|
||||
/**
|
||||
* Persistence is deliberately kept behind this small boundary. The domain
|
||||
* store remains synchronous for the local fixture, while production adapters
|
||||
* can be swapped in without making request handlers depend on a driver.
|
||||
*/
|
||||
export interface PersistenceRepository<T = unknown> {
|
||||
readonly adapter: RepositoryHealth["adapter"];
|
||||
load(): Promise<T | undefined>;
|
||||
save(value: T, expectedRevision?: number): Promise<void>;
|
||||
getRevision?(): number;
|
||||
health(): Promise<RepositoryHealth>;
|
||||
claimTaskDispatchOutbox?(workerId: string, leaseMs: number): Promise<string[]>;
|
||||
completeTaskDispatchOutbox?(taskId: string, workerId: string, success: boolean, error?: string): Promise<boolean>;
|
||||
claimWebDavSyncJobs?(workerId: string, leaseMs: number): Promise<WebDavSyncJob[]>;
|
||||
/** Renew a WebDAV job lease while a remote request is in flight. */
|
||||
renewWebDavSyncJob?(jobId: string, workerId: string, leaseMs: number): Promise<{ renewed: boolean; leaseExpiresAt?: string }>;
|
||||
completeWebDavSyncJob?(jobId: string, workerId: string, job: WebDavSyncJob, mutation?: WebDavSyncMutation): Promise<boolean>;
|
||||
enqueueExpiredWebDavRetention?(now?: string): Promise<number>;
|
||||
claimMessageOutbox?(workerId: string, leaseMs: number): Promise<MessageOutboxRecord[]>;
|
||||
completeMessageOutbox?(id: string, workerId: string, result: MessageOutboxCompletion): Promise<boolean>;
|
||||
/** Renew a message outbox lease while a provider request is in flight. */
|
||||
renewMessageOutbox?(id: string, workerId: string, leaseMs: number): Promise<{ renewed: boolean; leaseExpiresAt?: string }>;
|
||||
/** Renew a durable generation task lease with owner/token fencing. */
|
||||
renewTaskLease?(taskId: string, leaseToken: string, leaseMs: number): Promise<{ renewed: boolean; leaseExpiresAt?: string }>;
|
||||
/** Load one durable task for a worker that claimed it on another instance. */
|
||||
loadTaskById?(taskId: string): Promise<GenerationTask | undefined>;
|
||||
createTaskAtomic?(input: AtomicTaskInput): Promise<AtomicTaskResult>;
|
||||
}
|
||||
|
||||
export class MemoryRepository<T> implements PersistenceRepository<T> {
|
||||
readonly adapter = "memory" as const;
|
||||
private value?: T;
|
||||
constructor(initial?: T) { this.value = initial; }
|
||||
async load() { return this.value; }
|
||||
async save(value: T, _expectedRevision?: number) { this.value = value; }
|
||||
async health(): Promise<RepositoryHealth> { return { adapter: this.adapter, status: "ready" }; }
|
||||
}
|
||||
|
||||
export class UnavailableRepository<T> implements PersistenceRepository<T> {
|
||||
constructor(readonly adapter: "postgres", private readonly detail: string) {}
|
||||
async load() { return undefined; }
|
||||
async save(_value: T) { throw new Error(this.detail); }
|
||||
async health(): Promise<RepositoryHealth> { return { adapter: this.adapter, status: "unavailable", detail: this.detail }; }
|
||||
}
|
||||
|
||||
export class FileRepository<T> implements PersistenceRepository<T> {
|
||||
readonly adapter = "file" as const;
|
||||
constructor(private readonly filePath: string) {}
|
||||
async load() {
|
||||
try { return JSON.parse(await readFile(this.filePath, "utf8")) as T; }
|
||||
catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
async save(value: T, _expectedRevision?: number) {
|
||||
await mkdir(dirname(this.filePath), { recursive: true, mode: 0o700 });
|
||||
const temporary = `${this.filePath}.${process.pid}.${randomUUID()}.tmp`;
|
||||
await writeFile(temporary, JSON.stringify(value), { encoding: "utf8", mode: 0o600 });
|
||||
await rename(temporary, this.filePath);
|
||||
}
|
||||
async health(): Promise<RepositoryHealth> {
|
||||
try { await mkdir(dirname(this.filePath), { recursive: true, mode: 0o700 }); return { adapter: this.adapter, status: "ready" }; }
|
||||
catch (error) { return { adapter: this.adapter, status: "unavailable", detail: error instanceof Error ? error.message : String(error) }; }
|
||||
}
|
||||
}
|
||||
|
||||
type PgClient = { query(sql: string, values?: unknown[]): Promise<{ rows: Array<Record<string, unknown>> }> };
|
||||
type PgPool = PgClient & { connect(): Promise<PgClient & { release(): void }> };
|
||||
|
||||
/**
|
||||
* The Store-shaped local domain is projected into one row per entity in
|
||||
* `miragenflow_domain_rows`. The legacy JSON snapshot remains as a migration
|
||||
* fallback only; PostgreSQL rows are the authoritative representation once
|
||||
* the repository has written its first commit.
|
||||
*/
|
||||
export class PostgresRepository<T> implements PersistenceRepository<T> {
|
||||
readonly adapter = "postgres" as const;
|
||||
private client?: PgPool;
|
||||
private connectPromise?: Promise<PgPool | undefined>;
|
||||
// A missing revision row is revision zero. Treating it as undefined lets two
|
||||
// fresh instances both write the first snapshot and silently overwrite each
|
||||
// other before fencing is established.
|
||||
private revision = 0;
|
||||
constructor(private readonly databaseUrl: string, private readonly key = "default") {}
|
||||
getRevision() { return this.revision; }
|
||||
private async connect() {
|
||||
if (!this.connectPromise) this.connectPromise = (async () => {
|
||||
try {
|
||||
const dynamicImport = new Function("specifier", "return import(specifier)") as (specifier: string) => Promise<{ default?: { Pool: new (options: { connectionString: string }) => PgPool } }>;
|
||||
const module = await dynamicImport("pg");
|
||||
const Pool = module.default?.Pool;
|
||||
if (!Pool) return undefined;
|
||||
const client = new Pool({ connectionString: this.databaseUrl });
|
||||
await client.query("CREATE TABLE IF NOT EXISTS miragenflow_store_snapshots (snapshot_key TEXT PRIMARY KEY, payload JSONB NOT NULL, updated_at TIMESTAMPTZ NOT NULL DEFAULT now())");
|
||||
await client.query("CREATE TABLE IF NOT EXISTS miragenflow_domain_rows (root_key TEXT NOT NULL, entity_id TEXT NOT NULL, shape TEXT NOT NULL, payload JSONB, updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), PRIMARY KEY (root_key, entity_id))");
|
||||
await client.query("CREATE TABLE IF NOT EXISTS miragenflow_store_revisions (snapshot_key TEXT PRIMARY KEY, revision BIGINT NOT NULL DEFAULT 0, updated_at TIMESTAMPTZ NOT NULL DEFAULT now())");
|
||||
return client;
|
||||
} catch { return undefined; }
|
||||
})();
|
||||
return this.connectPromise;
|
||||
}
|
||||
/** Advance the shared Store revision for direct domain-table writes. This
|
||||
* keeps subsequent snapshot saves from overwriting a task/outbox mutation
|
||||
* committed by another repository instance. Callers must already be inside
|
||||
* a transaction and hold the relevant advisory/row locks. */
|
||||
private async bumpRevision(client: PgClient) {
|
||||
await client.query("INSERT INTO miragenflow_store_revisions(snapshot_key, revision) VALUES ($1, 0) ON CONFLICT (snapshot_key) DO NOTHING", [this.key]);
|
||||
const current = await client.query("SELECT revision FROM miragenflow_store_revisions WHERE snapshot_key = $1 FOR UPDATE", [this.key]);
|
||||
const next = Number(current.rows[0]?.revision || 0) + 1;
|
||||
await client.query("UPDATE miragenflow_store_revisions SET revision = $2, updated_at = now() WHERE snapshot_key = $1", [this.key, next]);
|
||||
this.revision = next;
|
||||
}
|
||||
async load() {
|
||||
const pool = this.client || (this.client = await this.connect());
|
||||
if (!pool) return undefined;
|
||||
const client = await pool.connect();
|
||||
try {
|
||||
await client.query("BEGIN ISOLATION LEVEL REPEATABLE READ");
|
||||
const revision = await client.query("SELECT revision FROM miragenflow_store_revisions WHERE snapshot_key = $1", [this.key]);
|
||||
this.revision = Number(revision.rows[0]?.revision || 0);
|
||||
const rows = await client.query("SELECT root_key, entity_id, shape, payload FROM miragenflow_domain_rows WHERE root_key LIKE $1 ORDER BY root_key, entity_id", [`${this.key}:%`]);
|
||||
const projected = rows.rows.length ? inflateDomainRows(rows.rows) : undefined;
|
||||
const canonical = await loadCanonicalSnapshot(client);
|
||||
if (canonical) await enrichCanonicalOutputMetadata(client, canonical);
|
||||
const value = canonical ? mergeSnapshots(projected, canonical) as T : projected ? projected as T : (await client.query("SELECT payload FROM miragenflow_store_snapshots WHERE snapshot_key = $1", [this.key])).rows[0]?.payload as T | undefined;
|
||||
await client.query("COMMIT");
|
||||
return value;
|
||||
} catch (error) {
|
||||
await client.query("ROLLBACK").catch(() => undefined);
|
||||
throw error;
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
async loadTaskById(taskId: string): Promise<GenerationTask | undefined> {
|
||||
const pool = this.client || (this.client = await this.connect());
|
||||
if (!pool) throw new Error("PostgreSQL adapter unavailable: install pg and verify DATABASE_URL");
|
||||
const client = await pool.connect();
|
||||
try {
|
||||
await client.query("BEGIN ISOLATION LEVEL REPEATABLE READ");
|
||||
const snapshot = await loadCanonicalSnapshot(client);
|
||||
if (snapshot) await enrichCanonicalOutputMetadata(client, snapshot);
|
||||
const entry = mapEntries(record(snapshot).tasks).find(([, value]) => record(value).id === taskId);
|
||||
const task = entry?.[1] as GenerationTask | undefined;
|
||||
await client.query("COMMIT");
|
||||
return task;
|
||||
} catch (error) {
|
||||
await client.query("ROLLBACK").catch(() => undefined);
|
||||
throw error;
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
async save(value: T, expectedRevision = this.revision) {
|
||||
const pool = this.client || (this.client = await this.connect());
|
||||
if (!pool) throw new Error("PostgreSQL adapter unavailable: install pg and verify DATABASE_URL");
|
||||
const client = await pool.connect();
|
||||
await client.query("BEGIN");
|
||||
try {
|
||||
await client.query("SELECT pg_advisory_xact_lock(hashtext($1))", [this.key]);
|
||||
await client.query("INSERT INTO miragenflow_store_revisions(snapshot_key, revision) VALUES ($1, 0) ON CONFLICT (snapshot_key) DO NOTHING", [this.key]);
|
||||
const currentRevision = await client.query("SELECT revision FROM miragenflow_store_revisions WHERE snapshot_key = $1 FOR UPDATE", [this.key]);
|
||||
const persistedRevision = Number(currentRevision.rows[0]?.revision || 0);
|
||||
if (persistedRevision !== expectedRevision) throw new Error("PostgreSQL persistence conflict: store revision changed; reload before writing");
|
||||
await client.query("DELETE FROM miragenflow_domain_rows WHERE root_key LIKE $1", [`${this.key}:%`]);
|
||||
for (const row of flattenDomainRows(value)) {
|
||||
await client.query("INSERT INTO miragenflow_domain_rows(root_key, entity_id, shape, payload, updated_at) VALUES ($1, $2, $3, $4, now())", [`${this.key}:${row.root}`, row.entityId, row.shape, row.payload === undefined ? null : JSON.stringify(row.payload)]);
|
||||
}
|
||||
// Keep the relational V1 tables populated in the same transaction. The
|
||||
// generic projection remains for fields that do not yet have a first
|
||||
// class table and for forward-compatible snapshot recovery.
|
||||
await persistCanonicalSnapshot(client, value);
|
||||
// Keep a compatibility snapshot for operators and pre-domain upgrades.
|
||||
await client.query("INSERT INTO miragenflow_store_snapshots(snapshot_key, payload, updated_at) VALUES ($1, $2, now()) ON CONFLICT (snapshot_key) DO UPDATE SET payload = EXCLUDED.payload, updated_at = now()", [this.key, JSON.stringify(value)]);
|
||||
await client.query("UPDATE miragenflow_store_revisions SET revision = revision + 1, updated_at = now() WHERE snapshot_key = $1", [this.key]);
|
||||
this.revision = persistedRevision + 1;
|
||||
await client.query("COMMIT");
|
||||
} catch (error) {
|
||||
await client.query("ROLLBACK").catch(() => undefined);
|
||||
throw error;
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
async health(): Promise<RepositoryHealth> {
|
||||
try { const pool = this.client || (this.client = await this.connect()); if (!pool) return { adapter: this.adapter, status: "unavailable", detail: "pg driver or DATABASE_URL unavailable" }; await pool.query("SELECT 1"); return { adapter: this.adapter, status: "ready" }; }
|
||||
catch (error) { return { adapter: this.adapter, status: "degraded", detail: error instanceof Error ? error.message : String(error) }; }
|
||||
}
|
||||
async renewTaskLease(taskId: string, leaseToken: string, leaseMs: number): Promise<{ renewed: boolean; leaseExpiresAt?: string }> {
|
||||
const pool = this.client || (this.client = await this.connect());
|
||||
if (!pool) throw new Error("PostgreSQL adapter unavailable: install pg and verify DATABASE_URL");
|
||||
const client = await pool.connect();
|
||||
try {
|
||||
await client.query("BEGIN");
|
||||
// Keep the lock order identical to save(): advisory lock, then the
|
||||
// revision row, then the task row. This prevents a heartbeat racing a
|
||||
// full snapshot save from deadlocking while still fencing stale owners.
|
||||
await client.query("SELECT pg_advisory_xact_lock(hashtext($1))", [this.key]);
|
||||
const result = await client.query(
|
||||
"UPDATE generation_tasks SET lease_expires_at = now() + ($3 || ' milliseconds')::interval, updated_at = now() WHERE id = $1 AND status = 'running' AND lease_token = $2 AND lease_expires_at > now() RETURNING lease_expires_at",
|
||||
[taskId, leaseToken, String(leaseMs)],
|
||||
);
|
||||
if (!result.rows.length) {
|
||||
await client.query("COMMIT");
|
||||
return { renewed: false };
|
||||
}
|
||||
await this.bumpRevision(client);
|
||||
await client.query("COMMIT");
|
||||
const raw = result.rows[0]?.lease_expires_at;
|
||||
return { renewed: true, leaseExpiresAt: raw ? new Date(String(raw)).toISOString() : undefined };
|
||||
} catch (error) {
|
||||
await client.query("ROLLBACK").catch(() => undefined);
|
||||
throw error;
|
||||
} finally { client.release(); }
|
||||
}
|
||||
async enqueueExpiredWebDavRetention(now = new Date().toISOString()): Promise<number> {
|
||||
const pool = this.client || (this.client = await this.connect());
|
||||
if (!pool) throw new Error("PostgreSQL adapter unavailable: install pg and verify DATABASE_URL");
|
||||
const client = await pool.connect();
|
||||
let queued = 0;
|
||||
try {
|
||||
await client.query("BEGIN");
|
||||
await client.query("SELECT pg_advisory_xact_lock(hashtext($1))", [`${this.key}:webdav-retention`]);
|
||||
const configs = await client.query("SELECT user_id, retention_state, state FROM webdav_configs WHERE manifest_retention_expires_at IS NOT NULL AND manifest_retention_expires_at <= $1 AND COALESCE(retention_state, 'active') <> 'deleted' FOR UPDATE SKIP LOCKED", [now]);
|
||||
for (const config of configs.rows) {
|
||||
const userId = String(config.user_id);
|
||||
const files = await client.query("SELECT path, etag FROM webdav_files WHERE user_id = $1 AND deleted_at IS NULL FOR UPDATE", [userId]);
|
||||
for (const file of files.rows) {
|
||||
const existing = await client.query("SELECT id FROM webdav_sync_jobs WHERE user_id = $1 AND intent = 'retention-delete' AND path = $2 AND status IN ('queued', 'running') LIMIT 1 FOR UPDATE", [userId, String(file.path)]);
|
||||
if (existing.rows.length) continue;
|
||||
const id = stableUuid(`webdav-retention:${userId}:${String(file.path)}`);
|
||||
await client.query("INSERT INTO webdav_sync_jobs(id, user_id, operation, intent, path, if_match, attempts, status, next_attempt_at, created_at) VALUES ($1, $2, 'delete', 'retention-delete', $3, $4, 0, 'queued', now(), now()) ON CONFLICT (id) DO UPDATE SET if_match = EXCLUDED.if_match, attempts = 0, status = 'queued', next_attempt_at = now(), lease_owner = NULL, lease_expires_at = NULL, last_error = NULL", [id, userId, String(file.path), file.etag || null]);
|
||||
queued += 1;
|
||||
}
|
||||
const hasFiles = files.rows.length > 0;
|
||||
await client.query("UPDATE webdav_configs SET retention_state = $2, state = $3 WHERE user_id = $1", [userId, hasFiles ? "deleting" : "deleted", hasFiles ? "syncing" : "ready"]);
|
||||
}
|
||||
if (configs.rows.length) await this.bumpRevision(client);
|
||||
await client.query("COMMIT");
|
||||
return queued;
|
||||
} catch (error) {
|
||||
await client.query("ROLLBACK").catch(() => undefined);
|
||||
throw error;
|
||||
} finally { client.release(); }
|
||||
}
|
||||
async claimTaskDispatchOutbox(workerId: string, leaseMs: number) {
|
||||
const pool = this.client || (this.client = await this.connect());
|
||||
if (!pool) throw new Error("PostgreSQL adapter unavailable: install pg and verify DATABASE_URL");
|
||||
const client = await pool.connect(); const claimed: string[] = [];
|
||||
try {
|
||||
await client.query("BEGIN");
|
||||
const rows = await client.query("SELECT task_id FROM task_dispatch_outbox WHERE ((status = 'pending' AND available_at <= now()) OR (status = 'leased' AND lease_expires_at <= now())) ORDER BY available_at, created_at FOR UPDATE SKIP LOCKED LIMIT 32");
|
||||
for (const row of rows.rows) {
|
||||
const taskId = String(row.task_id); claimed.push(taskId);
|
||||
await client.query("UPDATE task_dispatch_outbox SET status = 'leased', lease_owner = $2, lease_expires_at = now() + ($3 || ' milliseconds')::interval, attempts = attempts + 1 WHERE task_id = $1", [taskId, workerId, String(leaseMs)]);
|
||||
}
|
||||
if (claimed.length) await this.bumpRevision(client);
|
||||
await client.query("COMMIT"); return claimed;
|
||||
} catch (error) { await client.query("ROLLBACK").catch(() => undefined); throw error; }
|
||||
finally { client.release(); }
|
||||
}
|
||||
async completeTaskDispatchOutbox(taskId: string, workerId: string, success: boolean, error?: string): Promise<boolean> {
|
||||
const pool = this.client || (this.client = await this.connect());
|
||||
if (!pool) throw new Error("PostgreSQL adapter unavailable: install pg and verify DATABASE_URL");
|
||||
const client = await pool.connect();
|
||||
try {
|
||||
await client.query("BEGIN");
|
||||
const result = await client.query("UPDATE task_dispatch_outbox SET status = $3, lease_owner = NULL, lease_expires_at = NULL, dispatched_at = CASE WHEN $3 = 'sent' THEN now() ELSE dispatched_at END, available_at = CASE WHEN $3 = 'pending' THEN now() + interval '5 seconds' ELSE available_at END, last_error = $4 WHERE task_id = $1 AND lease_owner = $2 AND lease_expires_at > now() RETURNING task_id", [taskId, workerId, success ? "sent" : "pending", error || null]);
|
||||
if (result.rows.length) await this.bumpRevision(client);
|
||||
await client.query("COMMIT");
|
||||
return result.rows.length > 0;
|
||||
} catch (error) { await client.query("ROLLBACK").catch(() => undefined); throw error; }
|
||||
finally { client.release(); }
|
||||
}
|
||||
async claimWebDavSyncJobs(workerId: string, leaseMs: number): Promise<WebDavSyncJob[]> {
|
||||
const pool = this.client || (this.client = await this.connect());
|
||||
if (!pool) throw new Error("PostgreSQL adapter unavailable: install pg and verify DATABASE_URL");
|
||||
const client = await pool.connect();
|
||||
try {
|
||||
await client.query("BEGIN");
|
||||
const result = await client.query("WITH candidates AS (SELECT id FROM webdav_sync_jobs WHERE ((status = 'queued' AND next_attempt_at <= now()) OR (status = 'running' AND lease_expires_at <= now())) ORDER BY next_attempt_at, created_at, id FOR UPDATE SKIP LOCKED LIMIT 32) UPDATE webdav_sync_jobs AS jobs SET status = 'running', lease_owner = $1, lease_expires_at = now() + ($2 || ' milliseconds')::interval, attempts = jobs.attempts + 1 FROM candidates WHERE jobs.id = candidates.id RETURNING jobs.id, jobs.user_id, jobs.operation, jobs.intent, jobs.path, jobs.data, jobs.mime_type, jobs.if_match, jobs.attempts, jobs.status, jobs.next_attempt_at, jobs.lease_owner, jobs.lease_expires_at, jobs.last_error, jobs.conflict_copy_path, jobs.created_at", [workerId, String(leaseMs)]);
|
||||
if (result.rows.length) await this.bumpRevision(client);
|
||||
await client.query("COMMIT");
|
||||
return result.rows.map((row) => ({ id: String(row.id), userId: String(row.user_id), operation: row.operation as WebDavSyncJob["operation"], intent: typeof row.intent === "string" ? row.intent as WebDavSyncJob["intent"] : undefined, path: String(row.path), data: typeof row.data === "string" ? row.data : undefined, mimeType: typeof row.mime_type === "string" ? row.mime_type : undefined, ifMatch: typeof row.if_match === "string" ? row.if_match : undefined, attempts: Number(row.attempts || 0), status: row.status as WebDavSyncJob["status"], nextAttemptAt: new Date(String(row.next_attempt_at)).toISOString(), leaseOwner: row.lease_owner ? String(row.lease_owner) : undefined, leaseExpiresAt: row.lease_expires_at ? new Date(String(row.lease_expires_at)).toISOString() : undefined, lastError: typeof row.last_error === "string" ? row.last_error : undefined, conflictCopyPath: typeof row.conflict_copy_path === "string" ? row.conflict_copy_path : undefined, createdAt: new Date(String(row.created_at)).toISOString() }));
|
||||
} catch (error) { await client.query("ROLLBACK").catch(() => undefined); throw error; }
|
||||
finally { client.release(); }
|
||||
}
|
||||
async renewWebDavSyncJob(jobId: string, workerId: string, leaseMs: number): Promise<{ renewed: boolean; leaseExpiresAt?: string }> {
|
||||
const pool = this.client || (this.client = await this.connect());
|
||||
if (!pool) throw new Error("PostgreSQL adapter unavailable: install pg and verify DATABASE_URL");
|
||||
const client = await pool.connect();
|
||||
try {
|
||||
await client.query("BEGIN");
|
||||
await client.query("SELECT pg_advisory_xact_lock(hashtext($1))", [`${this.key}:webdav-job`]);
|
||||
const result = await client.query("UPDATE webdav_sync_jobs SET lease_expires_at = now() + ($3 || ' milliseconds')::interval WHERE id = $1 AND status = 'running' AND lease_owner = $2 AND lease_expires_at > now() RETURNING lease_expires_at", [jobId, workerId, String(leaseMs)]);
|
||||
if (!result.rows.length) { await client.query("COMMIT"); return { renewed: false }; }
|
||||
await this.bumpRevision(client);
|
||||
await client.query("COMMIT");
|
||||
const raw = result.rows[0]?.lease_expires_at;
|
||||
return { renewed: true, leaseExpiresAt: raw ? new Date(String(raw)).toISOString() : undefined };
|
||||
} catch (error) { await client.query("ROLLBACK").catch(() => undefined); throw error; }
|
||||
finally { client.release(); }
|
||||
}
|
||||
async completeWebDavSyncJob(jobId: string, workerId: string, job: WebDavSyncJob, mutation?: WebDavSyncMutation): Promise<boolean> {
|
||||
const pool = this.client || (this.client = await this.connect());
|
||||
if (!pool) throw new Error("PostgreSQL adapter unavailable: install pg and verify DATABASE_URL");
|
||||
const client = await pool.connect();
|
||||
try {
|
||||
await client.query("BEGIN");
|
||||
const result = await client.query("UPDATE webdav_sync_jobs SET status = $3, intent = $4, path = $5, data = $6, mime_type = $7, if_match = $8, attempts = $9, next_attempt_at = $10, lease_owner = NULL, lease_expires_at = NULL, last_error = $11, conflict_copy_path = $12 WHERE id = $1 AND lease_owner = $2 AND lease_expires_at > now() RETURNING id", [jobId, workerId, job.status, job.intent || null, job.path, job.data || null, job.mimeType || null, job.ifMatch || null, job.attempts, iso(job.nextAttemptAt), job.lastError || null, job.conflictCopyPath || null]);
|
||||
if (!result.rows.length) { await client.query("ROLLBACK").catch(() => undefined); return false; }
|
||||
const file = mutation?.file;
|
||||
const fileUserId = file ? String(file.userId) : undefined;
|
||||
if (file && fileUserId) {
|
||||
await client.query("INSERT INTO webdav_files(user_id, path, mime_type, data, checksum, etag, version, updated_at, deleted_at, sync_state) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) ON CONFLICT (user_id, path) DO UPDATE SET mime_type = EXCLUDED.mime_type, data = EXCLUDED.data, checksum = EXCLUDED.checksum, etag = EXCLUDED.etag, version = EXCLUDED.version, updated_at = EXCLUDED.updated_at, deleted_at = EXCLUDED.deleted_at, sync_state = EXCLUDED.sync_state", [fileUserId, file.path, file.mimeType, file.data, file.checksum, file.etag, file.version, iso(file.updatedAt), file.deletedAt ? iso(file.deletedAt) : null, file.syncState || "synced"]);
|
||||
}
|
||||
const config = mutation?.config;
|
||||
const configUserId = uuid(config?.userId);
|
||||
if (config && configUserId) {
|
||||
await client.query("UPDATE webdav_configs SET encrypted_url_ref = $2, encrypted_username_ref = $3, encrypted_password_ref = $4, directory = $5, state = $6, retention_state = $7, last_synced_at = $8, retention_days = $9, manifest_retention_expires_at = $10, manifest_extension_days = $11 WHERE user_id = $1", [configUserId, config.encryptedUrl || "", config.encryptedUsername || null, config.encryptedPassword || null, config.directory, config.state, config.retentionState || "active", config.lastSyncedAt ? iso(config.lastSyncedAt) : null, config.retentionDays, config.manifestRetentionExpiresAt ? iso(config.manifestRetentionExpiresAt) : null, config.manifestExtensionDays || 0]);
|
||||
if (config.manifestVersion !== undefined || config.manifestChecksum || config.manifestEtag) {
|
||||
const version = numberValue(config.manifestVersion, 0);
|
||||
const latest = await client.query("SELECT COALESCE(MAX(version), -1) AS version FROM webdav_manifests WHERE user_id = $1 FOR UPDATE", [configUserId]);
|
||||
if (version > Number(latest.rows[0]?.version ?? -1)) {
|
||||
// A stale worker must never overwrite a manifest at the same or a
|
||||
// newer version. Version fencing also makes restart reads select
|
||||
// one authoritative manifest row.
|
||||
await client.query("INSERT INTO webdav_manifests(id, user_id, version, checksum, manifest, state, created_at) VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT (user_id, version) DO NOTHING", [stableUuid(`webdav-manifest:${configUserId}:${version}`), configUserId, version, config.manifestChecksum || "", jsonValue({ etag: config.manifestEtag, version: config.manifestVersion, checksum: config.manifestChecksum }), config.state, iso(config.lastSyncedAt)]);
|
||||
}
|
||||
}
|
||||
}
|
||||
const taskRetention = mutation?.taskRetention;
|
||||
const taskId = uuid(taskRetention?.taskId);
|
||||
if (taskRetention && taskId) {
|
||||
await client.query("UPDATE generation_tasks SET task_snapshot = jsonb_set(COALESCE(task_snapshot, '{}'::jsonb), '{retentionState}', to_jsonb($2::text), true), updated_at = $3 WHERE id = $1", [taskId, taskRetention.retentionState, iso(taskRetention.updatedAt)]);
|
||||
}
|
||||
const conflictJob = mutation?.conflictJob;
|
||||
if (conflictJob) {
|
||||
await client.query("INSERT INTO webdav_sync_jobs(id, user_id, operation, intent, path, data, mime_type, if_match, attempts, status, next_attempt_at, lease_owner, lease_expires_at, last_error, conflict_copy_path, created_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, NULL, NULL, $12, $13, $14) ON CONFLICT (id) DO NOTHING", [conflictJob.id, conflictJob.userId, conflictJob.operation, conflictJob.intent || null, conflictJob.path, conflictJob.data || null, conflictJob.mimeType || null, conflictJob.ifMatch || null, conflictJob.attempts, conflictJob.status, iso(conflictJob.nextAttemptAt), conflictJob.lastError || null, conflictJob.conflictCopyPath || null, iso(conflictJob.createdAt)]);
|
||||
}
|
||||
await this.bumpRevision(client);
|
||||
await client.query("COMMIT");
|
||||
return result.rows.length > 0;
|
||||
} catch (error) { await client.query("ROLLBACK").catch(() => undefined); throw error; }
|
||||
finally { client.release(); }
|
||||
}
|
||||
/** Claim message rows atomically so multiple server instances cannot send the
|
||||
* same message. The row is selected with SKIP LOCKED and the complete
|
||||
* encrypted payload is returned to the worker after the lease is recorded. */
|
||||
async claimMessageOutbox(workerId: string, leaseMs: number): Promise<MessageOutboxRecord[]> {
|
||||
const pool = this.client || (this.client = await this.connect());
|
||||
if (!pool) throw new Error("PostgreSQL adapter unavailable: install pg and verify DATABASE_URL");
|
||||
const client = await pool.connect(); const claimed: MessageOutboxRecord[] = [];
|
||||
try {
|
||||
await client.query("BEGIN");
|
||||
// A worker can crash after its fifth send attempt. Finalize that
|
||||
// expired lease before selecting retryable rows so it cannot remain
|
||||
// stuck in `sending` forever.
|
||||
await client.query("UPDATE email_sms_outbox SET status = 'dead', lease_owner = NULL, lease_expires_at = NULL, updated_at = now() WHERE status = 'sending' AND attempts >= 5 AND lease_expires_at <= now()");
|
||||
const rows = await client.query("SELECT id, idempotency_key, target_hash, channel, purpose, payload, status, attempts, next_attempt_at, lease_owner, lease_expires_at, provider_message_id, last_error, created_at, updated_at FROM email_sms_outbox WHERE attempts < 5 AND (((status IN ('pending', 'queued', 'failed')) AND next_attempt_at <= now()) OR (status = 'sending' AND lease_expires_at <= now())) ORDER BY next_attempt_at, created_at, id FOR UPDATE SKIP LOCKED LIMIT 32");
|
||||
for (const row of rows.rows) {
|
||||
const attempts = Number(row.attempts || 0) + 1;
|
||||
await client.query("UPDATE email_sms_outbox SET status = 'sending', lease_owner = $2, lease_expires_at = now() + ($3 || ' milliseconds')::interval, attempts = $4, updated_at = now() WHERE id = $1", [String(row.id), workerId, String(leaseMs), attempts]);
|
||||
const payload = record(parseJson(row.payload));
|
||||
claimed.push({
|
||||
id: String(row.id), idempotencyKey: typeof row.idempotency_key === "string" ? row.idempotency_key : undefined, targetHash: String(row.target_hash), channel: row.channel === "sms" ? "sms" : "email", purpose: row.purpose === "reset" ? "reset" : row.purpose === "notification" ? "notification" : "register",
|
||||
targetEncrypted: typeof payload.targetEncrypted === "string" ? payload.targetEncrypted : undefined,
|
||||
payloadEncrypted: typeof payload.payloadEncrypted === "string" ? payload.payloadEncrypted : undefined,
|
||||
status: "sending", attempts, nextAttemptAt: row.next_attempt_at ? new Date(String(row.next_attempt_at)).toISOString() : undefined,
|
||||
leaseOwner: workerId, leaseExpiresAt: new Date(Date.now() + leaseMs).toISOString(), providerMessageId: typeof row.provider_message_id === "string" ? row.provider_message_id : undefined, lastError: typeof row.last_error === "string" ? row.last_error : undefined,
|
||||
createdAt: new Date(String(row.created_at)).toISOString(), updatedAt: new Date().toISOString(),
|
||||
});
|
||||
}
|
||||
if (claimed.length) await this.bumpRevision(client);
|
||||
await client.query("COMMIT"); return claimed;
|
||||
} catch (error) { await client.query("ROLLBACK").catch(() => undefined); throw error; }
|
||||
finally { client.release(); }
|
||||
}
|
||||
/** Complete a message only while holding its current lease owner. A stale
|
||||
* worker therefore gets undefined and cannot overwrite a newer attempt. */
|
||||
async completeMessageOutbox(id: string, workerId: string, result: MessageOutboxCompletion): Promise<boolean> {
|
||||
const pool = this.client || (this.client = await this.connect());
|
||||
if (!pool) throw new Error("PostgreSQL adapter unavailable: install pg and verify DATABASE_URL");
|
||||
const client = await pool.connect();
|
||||
try {
|
||||
await client.query("BEGIN");
|
||||
const updated = await client.query("UPDATE email_sms_outbox SET status = CASE WHEN $3 THEN 'sent' WHEN attempts >= 5 THEN 'dead' ELSE 'failed' END, provider_message_id = COALESCE($4, provider_message_id), last_error = CASE WHEN $3 THEN NULL ELSE $5 END, next_attempt_at = CASE WHEN $3 OR attempts >= 5 THEN next_attempt_at ELSE COALESCE($6, now()) END, lease_owner = NULL, lease_expires_at = NULL, updated_at = now() WHERE id = $1 AND lease_owner = $2 AND lease_expires_at > now() RETURNING id, target_hash, channel, purpose, payload, status, attempts, next_attempt_at, lease_owner, lease_expires_at, provider_message_id, last_error, created_at, updated_at", [id, workerId, result.success, result.providerMessageId || null, result.error || null, result.nextAttemptAt || null]);
|
||||
if (!updated.rows.length) { await client.query("COMMIT"); return false; }
|
||||
await this.bumpRevision(client);
|
||||
await client.query("COMMIT"); return true;
|
||||
} catch (error) { await client.query("ROLLBACK").catch(() => undefined); throw error; }
|
||||
finally { client.release(); }
|
||||
}
|
||||
async renewMessageOutbox(id: string, workerId: string, leaseMs: number): Promise<{ renewed: boolean; leaseExpiresAt?: string }> {
|
||||
const pool = this.client || (this.client = await this.connect());
|
||||
if (!pool) throw new Error("PostgreSQL adapter unavailable: install pg and verify DATABASE_URL");
|
||||
const client = await pool.connect();
|
||||
try {
|
||||
await client.query("BEGIN");
|
||||
await client.query("SELECT pg_advisory_xact_lock(hashtext($1))", [`${this.key}:message-outbox`]);
|
||||
const result = await client.query("UPDATE email_sms_outbox SET lease_expires_at = now() + ($3 || ' milliseconds')::interval, updated_at = now() WHERE id = $1 AND status = 'sending' AND lease_owner = $2 AND lease_expires_at > now() RETURNING lease_expires_at", [id, workerId, String(leaseMs)]);
|
||||
if (!result.rows.length) { await client.query("COMMIT"); return { renewed: false }; }
|
||||
await this.bumpRevision(client);
|
||||
await client.query("COMMIT");
|
||||
const leaseExpiresAt = result.rows[0]?.lease_expires_at;
|
||||
return { renewed: true, leaseExpiresAt: leaseExpiresAt ? new Date(String(leaseExpiresAt)).toISOString() : undefined };
|
||||
} catch (error) { await client.query("ROLLBACK").catch(() => undefined); throw error; }
|
||||
finally { client.release(); }
|
||||
}
|
||||
async createTaskAtomic(input: AtomicTaskInput): Promise<AtomicTaskResult> {
|
||||
if (!Number.isSafeInteger(input.maxConcurrent) || input.maxConcurrent < 1) throw new DomainError(ErrorCodes.ValidationFailed, "任务并发上限无效", undefined, 422);
|
||||
const pool = this.client || (this.client = await this.connect());
|
||||
if (!pool) throw new Error("PostgreSQL adapter unavailable: install pg and verify DATABASE_URL");
|
||||
const client = await pool.connect(); const task = input.task; const taskSnapshot = { modelProductId: task.modelProductId, prompt: task.prompt, resolutionPresetId: task.resolutionPresetId, count: task.count, estimatedCost: input.estimatedCost, channelGroupId: task.channelGroupId, routeSnapshotVersion: task.routeSnapshot?.version, retentionExpiresAt: task.retentionExpiresAt, retentionState: task.retentionState, retentionExtensionDays: task.retentionExtensionDays, reserveExpiresAt: task.reserveExpiresAt, inputSnapshot: task.inputSnapshot, planSnapshot: task.planSnapshot, requestFingerprint: input.requestFingerprint, providerSubmitted: task.providerSubmitted, publicErrorCode: task.publicErrorCode };
|
||||
try {
|
||||
await client.query("BEGIN");
|
||||
await client.query("SELECT pg_advisory_xact_lock_shared(hashtext($1))", [this.key]);
|
||||
await client.query("SELECT pg_advisory_xact_lock(hashtext('balance:' || $1))", [input.userId]);
|
||||
await client.query("INSERT INTO balance_accounts(user_id, available, reserved, version) VALUES ($1, 0, 0, 0) ON CONFLICT (user_id) DO NOTHING", [input.userId]);
|
||||
const account = await client.query("SELECT available, reserved, version FROM balance_accounts WHERE user_id = $1 FOR UPDATE", [input.userId]);
|
||||
const current = account.rows[0]; if (!current) throw new DomainError(ErrorCodes.ObjectNotFound, "余额账户不存在", undefined, 404);
|
||||
const existing = await client.query("SELECT id, task_snapshot, status FROM generation_tasks WHERE user_id = $1 AND idempotency_key = $2 FOR UPDATE", [input.userId, task.idempotencyKey]);
|
||||
if (existing.rows[0]) {
|
||||
const existingSnapshot = record(parseJson(existing.rows[0].task_snapshot, {}));
|
||||
if (existingSnapshot.requestFingerprint && existingSnapshot.requestFingerprint !== input.requestFingerprint) throw new DomainError(ErrorCodes.IdempotencyConflict, "幂等键已用于其他请求", undefined, 409);
|
||||
await client.query("INSERT INTO task_dispatch_outbox(task_id, status) VALUES ($1, 'pending') ON CONFLICT (task_id) DO NOTHING", [String(existing.rows[0].id)]);
|
||||
await this.bumpRevision(client);
|
||||
await client.query("COMMIT");
|
||||
return { created: false, taskId: String(existing.rows[0].id), task: parseJson(existingSnapshot, undefined) as GenerationTask | undefined };
|
||||
}
|
||||
const activeTasks = await client.query("SELECT COUNT(*)::int AS count FROM generation_tasks WHERE user_id = $1 AND status IN ('queued', 'running')", [input.userId]);
|
||||
if (Number(activeTasks.rows[0]?.count || 0) >= input.maxConcurrent) throw new DomainError(ErrorCodes.RateLimited, "已达到当前套餐并发上限，请等待已有任务完成", { maxConcurrent: input.maxConcurrent }, 429);
|
||||
let available = Number(current.available);
|
||||
const expiredBuckets = await client.query("SELECT id, remaining FROM balance_buckets WHERE user_id = $1 AND remaining > 0 AND expires_at IS NOT NULL AND expires_at <= now() ORDER BY priority, expires_at, id FOR UPDATE", [input.userId]);
|
||||
let expiredAmount = 0;
|
||||
for (const row of expiredBuckets.rows) {
|
||||
const effective = Math.min(Number(row.remaining), Math.max(0, available));
|
||||
if (effective > 0) {
|
||||
const posted = await client.query("INSERT INTO balance_ledger(id, user_id, type, amount, unit_version, idempotency_key, reference_id, bucket_id, status, created_at) VALUES ($1, $2, 'adjustment', $3, $4, $5, $6, $6, 'posted', now()) ON CONFLICT (idempotency_key) DO NOTHING RETURNING id", [randomUUID(), input.userId, -effective, input.unitVersion, `expire:${input.userId}:${row.id}`, String(row.id)]);
|
||||
if (posted.rows.length) { expiredAmount += effective; available -= effective; }
|
||||
}
|
||||
await client.query("UPDATE balance_buckets SET remaining = 0 WHERE id = $1", [String(row.id)]);
|
||||
}
|
||||
if (expiredAmount > 0) await client.query("UPDATE balance_accounts SET available = GREATEST(0, available - $2), version = version + 1 WHERE user_id = $1", [input.userId, expiredAmount]);
|
||||
let buckets = await client.query("SELECT id, remaining, priority, expires_at FROM balance_buckets WHERE user_id = $1 AND remaining > 0 AND (expires_at IS NULL OR expires_at > now()) ORDER BY priority, expires_at NULLS LAST, id FOR UPDATE", [input.userId]);
|
||||
const tracked = buckets.rows.reduce((total, row) => total + Number(row.remaining), 0);
|
||||
if (tracked < available) {
|
||||
const syntheticBucketId = randomUUID();
|
||||
const priority = buckets.rows.reduce((highest, row) => Math.max(highest, Number(row.priority)), -1) + 1;
|
||||
const remaining = available - tracked;
|
||||
await client.query("INSERT INTO balance_buckets(id, user_id, source, remaining, priority) VALUES ($1, $2, 'manual', $3, $4)", [syntheticBucketId, input.userId, remaining, priority]);
|
||||
buckets.rows.push({ id: syntheticBucketId, remaining, priority, expires_at: null });
|
||||
}
|
||||
// Package benefits and credit sources are independent: prefer the plan
|
||||
// bucket while it has funds, then continue through ordinary buckets.
|
||||
const ordered = [...buckets.rows].sort((a, b) => Number(String(a.id) !== input.preferredBucketId) - Number(String(b.id) !== input.preferredBucketId));
|
||||
let remaining = input.estimatedCost; const allocations: Array<{ id: string; amount: number }> = [];
|
||||
for (const row of ordered) { if (remaining <= 0) break; const amount = Math.min(Number(row.remaining), remaining); if (amount <= 0) continue; allocations.push({ id: String(row.id), amount }); remaining -= amount; }
|
||||
if (remaining > 0 || available < input.estimatedCost) throw new DomainError(ErrorCodes.InsufficientBalance, "金币余额不足", { required: input.estimatedCost, available }, 409);
|
||||
const accountUpdate = await client.query("UPDATE balance_accounts SET available = available - $2, reserved = reserved + $2, version = version + 1 WHERE user_id = $1 AND available >= $2 RETURNING version", [input.userId, input.estimatedCost]);
|
||||
if (!accountUpdate.rows.length) throw new DomainError(ErrorCodes.InsufficientBalance, "金币余额不足", undefined, 409);
|
||||
for (const allocation of allocations) {
|
||||
await client.query("UPDATE balance_buckets SET remaining = remaining - $2 WHERE id = $1", [allocation.id, allocation.amount]);
|
||||
await client.query("INSERT INTO reserve_allocations(user_id, reference_id, bucket_id, amount) VALUES ($1, $2, $3, $4) ON CONFLICT DO NOTHING", [input.userId, task.id, allocation.id, allocation.amount]);
|
||||
await client.query("INSERT INTO balance_ledger(id, user_id, type, amount, unit_version, idempotency_key, reference_id, task_id, bucket_id, status, created_at) VALUES ($1, $2, 'reserve', $3, $4, $5, $6, $7, $8, 'posted', now()) ON CONFLICT (idempotency_key) DO NOTHING", [randomUUID(), input.userId, allocation.amount, input.unitVersion, `reserve:${input.userId}:${task.id}:${allocation.id}`, task.id, task.id, allocation.id]);
|
||||
}
|
||||
await client.query("INSERT INTO generation_tasks(id, user_id, task_type, status, idempotency_key, model_snapshot, route_snapshot, pricing_snapshot, task_snapshot, reserved_amount, state_version, event_sequence, reserve_expires_at, updated_at, created_at) VALUES ($1, $2, $3, 'queued', $4, $5, $6, $7, $8, $9, 0, 1, $10, $11, $12)", [task.id, input.userId, task.taskType, task.idempotencyKey, jsonValue(task.modelSnapshot), jsonValue(task.routeSnapshot), jsonValue(task.pricingSnapshot), jsonValue(taskSnapshot), input.estimatedCost, task.reserveExpiresAt ? iso(task.reserveExpiresAt) : null, iso(task.updatedAt, iso(task.createdAt)), iso(task.createdAt)]);
|
||||
await client.query("INSERT INTO task_event_log(task_id, sequence, event_id, event_type, payload, occurred_at) VALUES ($1, 1, $2, 'task.queued', $3, now())", [task.id, randomUUID(), jsonValue({ estimatedCost: input.estimatedCost })]);
|
||||
await client.query("INSERT INTO task_dispatch_outbox(task_id, status) VALUES ($1, 'pending') ON CONFLICT (task_id) DO NOTHING", [task.id]);
|
||||
await this.bumpRevision(client);
|
||||
await client.query("COMMIT");
|
||||
return { created: true, taskId: task.id, task: { ...task, reservedCost: input.estimatedCost, eventSequence: 1 } };
|
||||
} catch (error) { await client.query("ROLLBACK").catch(() => undefined); throw error; }
|
||||
finally { client.release(); }
|
||||
}
|
||||
}
|
||||
|
||||
type DomainShape = "map" | "array" | "scalar";
|
||||
type DomainRow = { entityId: string; shape: DomainShape; payload?: unknown };
|
||||
|
||||
type SnapshotRecord = Record<string, unknown>;
|
||||
|
||||
const uuidPattern = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
|
||||
|
||||
function uuid(value: unknown) { return typeof value === "string" && uuidPattern.test(value) ? value : undefined; }
|
||||
function stableUuid(value: unknown) {
|
||||
const hex = createHash("sha256").update(String(value)).digest("hex").slice(0, 32);
|
||||
return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-4${hex.slice(13, 16)}-8${hex.slice(17, 20)}-${hex.slice(20, 32)}`;
|
||||
}
|
||||
function isRecord(value: unknown): value is SnapshotRecord { return value !== null && typeof value === "object" && !Array.isArray(value); }
|
||||
function record(value: unknown): SnapshotRecord { return isRecord(value) ? value : {}; }
|
||||
function mapEntries(value: unknown) { return Array.isArray(value) ? value.filter((entry): entry is [unknown, unknown] => Array.isArray(entry) && entry.length === 2) : []; }
|
||||
function arrayEntries(value: unknown) { return Array.isArray(value) ? value : []; }
|
||||
function jsonValue(value: unknown, fallback: unknown = {}) { return JSON.stringify(value === undefined ? fallback : value); }
|
||||
function iso(value: unknown, fallback = new Date().toISOString()) { return typeof value === "string" && !Number.isNaN(Date.parse(value)) ? value : fallback; }
|
||||
function dateValue(value: unknown, fallback = new Date().toISOString()) { return typeof value === "number" && Number.isFinite(value) ? new Date(value).toISOString() : iso(value, fallback); }
|
||||
function numberValue(value: unknown, fallback = 0) { return typeof value === "number" && Number.isFinite(value) ? value : fallback; }
|
||||
function parseJson(value: unknown, fallback: unknown = {}) { if (typeof value !== "string") return value ?? fallback; try { return JSON.parse(value); } catch { return fallback; } }
|
||||
|
||||
async function canonicalTablesAvailable(client: PgClient) {
|
||||
try {
|
||||
const result = await client.query("SELECT to_regclass('public.users') AS users, to_regclass('public.generation_tasks') AS generation_tasks, to_regclass('public.balance_ledger') AS balance_ledger");
|
||||
return Boolean(result.rows[0]?.users && result.rows[0]?.generation_tasks && result.rows[0]?.balance_ledger);
|
||||
} catch { return false; }
|
||||
}
|
||||
|
||||
async function clearCanonicalTables(client: PgClient) {
|
||||
// Delete children first so the migration-defined foreign keys remain active.
|
||||
for (const table of ["task_dispatch_outbox", "task_event_log", "generation_attempts", "generation_outputs", "generation_tasks", "reserve_allocations", "plan_payment_events", "balance_ledger", "balance_buckets", "user_plan_entitlements", "plan_purchases", "payment_events", "recharge_orders", "assets", "upload_objects", "email_sms_outbox", "webdav_sync_jobs", "webdav_files", "webdav_manifests", "webdav_configs", "admin_approval_requests", "admin_idempotency", "captcha_challenges", "mfa_challenges", "session_families", "admin_accounts", "message_providers", "message_templates", "payment_providers", "stored_objects", "runtime_records", "verification_codes", "sessions", "password_credentials", "user_identities", "balance_accounts", "users", "provider_channels", "channel_groups", "model_products", "membership_plans", "balance_units", "storage_policies", "audit_logs"]) await client.query(`DELETE FROM ${table}`);
|
||||
}
|
||||
|
||||
async function persistCanonicalSnapshot(client: PgClient, value: unknown) {
|
||||
if (!(await canonicalTablesAvailable(client))) return;
|
||||
const source = record(value);
|
||||
await clearCanonicalTables(client);
|
||||
const users = mapEntries(source.users);
|
||||
for (const [key, raw] of users) {
|
||||
const item = record(raw); const userId = uuid(key) || uuid(item.id); if (!userId) continue;
|
||||
const createdAt = iso(item.createdAt); const status = typeof item.status === "string" ? item.status : "active";
|
||||
await client.query("INSERT INTO users(id, status, display_name, created_at, updated_at, mfa_secret_encrypted, mfa_recovery_code_hashes, mfa_last_totp_counter, mfa_required, mfa_enabled, roles, failed_login_count, locked_until) VALUES ($1, $2, $3, $4, $4, $5, $6, $7, $8, $9, $10, $11, $12)", [userId, status, typeof item.displayName === "string" ? item.displayName : "", createdAt, item.mfaSecretEncrypted || null, jsonValue(item.mfaRecoveryCodeHashes, []), item.mfaLastTotpCounter === undefined ? null : numberValue(item.mfaLastTotpCounter), item.mfaRequired === true, item.mfaEnabled === true, jsonValue(item.roles, []), numberValue(item.failedLoginCount), item.lockedUntil === undefined ? null : numberValue(item.lockedUntil)]);
|
||||
const credentials = typeof item.password === "string" ? item.password : "";
|
||||
if (credentials) await client.query("INSERT INTO password_credentials(user_id, password_hash, algorithm_version, changed_at) VALUES ($1, $2, $3, $4)", [userId, credentials, credentials.startsWith("scrypt:") ? "scrypt-v1" : "legacy", createdAt]);
|
||||
for (const kind of ["username", "email", "phone"] as const) {
|
||||
const normalized = typeof item[kind] === "string" ? item[kind] : undefined; if (!normalized) continue;
|
||||
await client.query("INSERT INTO user_identities(id, user_id, kind, normalized_value, verified_at) VALUES ($1, $2, $3, $4, $5)", [stableUuid(`${userId}:${kind}:${normalized}`), userId, kind, normalized, item.verified ? createdAt : null]);
|
||||
}
|
||||
}
|
||||
for (const [key, raw] of mapEntries(source.sessions)) {
|
||||
const item = record(raw); const userId = uuid(item.userId); if (!userId) continue;
|
||||
const refreshHash = String(key); const rotationId = uuid(item.rotationId) || stableUuid(`rotation:${refreshHash}`); const familyId = uuid(item.familyId) || stableUuid(`family:${refreshHash}`);
|
||||
await client.query("INSERT INTO sessions(id, subject_id, scope, refresh_hash, rotation_id, expires_at, absolute_expires_at, revoked_at, created_at, family_id, device_id, replaced_by, roles, scopes) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)", [stableUuid(`session:${refreshHash}`), userId, item.scope === "admin" ? "admin" : "user", refreshHash, rotationId, dateValue(item.expiresAt), dateValue(item.absoluteExpiresAt, dateValue(item.expiresAt)), item.revokedAt ? dateValue(item.revokedAt) : null, dateValue(item.createdAt), familyId, typeof item.deviceId === "string" ? item.deviceId : null, item.replacedBy || null, jsonValue(item.roles, []), jsonValue(item.scopes, [])]);
|
||||
}
|
||||
for (const [key, raw] of mapEntries(source.balances)) { const userId = uuid(key); const item = record(raw); if (!userId) continue; await client.query("INSERT INTO balance_accounts(user_id, available, reserved, version) VALUES ($1, $2, $3, $4)", [userId, numberValue(item.available), numberValue(item.reserved), numberValue(item.version)]); }
|
||||
for (const raw of arrayEntries(source.ledger)) { const item = record(raw); const userId = uuid(item.userId); const id = uuid(item.id); if (!userId || !id) continue; await client.query("INSERT INTO balance_ledger(id, user_id, type, amount, unit_version, idempotency_key, reference_id, task_id, output_id, bucket_id, expires_at, status, created_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)", [id, userId, String(item.type || "adjustment"), numberValue(item.amount), numberValue(item.balanceUnitVersion, 1), String(item.idempotencyKey || id), typeof item.referenceId === "string" ? item.referenceId : null, uuid(item.taskId) || null, uuid(item.outputId) || null, uuid(item.bucketId) || null, item.expiresAt ? iso(item.expiresAt) : null, String(item.status || "posted"), iso(item.createdAt)]); }
|
||||
// `buckets` is persisted as a map keyed by user id, so iterate its values
|
||||
// rather than treating the outer `[userId, list]` tuple as a bucket.
|
||||
for (const [, raw] of mapEntries(source.buckets)) for (const bucket of arrayEntries(raw)) { const item = record(bucket); const id = uuid(item.id); const userId = uuid(item.userId); if (!id || !userId) continue; await client.query("INSERT INTO balance_buckets(id, user_id, source, remaining, priority, expires_at) VALUES ($1, $2, $3, $4, $5, $6)", [id, userId, String(item.source || "recharge"), numberValue(item.remaining), numberValue(item.priority), item.expiresAt ? iso(item.expiresAt) : null]); }
|
||||
const unit = record(source.balanceUnit); if (unit.shortCode) await client.query("INSERT INTO balance_units(id, version, name, short_code, icon, precision, currency_code, conversion_rate, rounding, published_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, now())", [String(unit.shortCode), numberValue(unit.version, 1), String(unit.name || "金币"), String(unit.shortCode), String(unit.icon || "coins"), numberValue(unit.precision), String(unit.currencyCode || "CNY"), numberValue(unit.conversionRate, 1), String(unit.rounding || "round")]);
|
||||
for (const raw of arrayEntries(source.products)) { const item = record(raw); if (!item.id || !item.publicModelId) continue; await client.query("INSERT INTO model_products(id, public_model_id, tier, enabled, version, snapshot) VALUES ($1, $2, $3, $4, $5, $6)", [String(item.id), String(item.publicModelId), String(item.tier || "basic"), item.enabled !== false, numberValue(item.version, 1), jsonValue(item)]); }
|
||||
for (const raw of arrayEntries(source.channels)) { const item = record(raw); if (!item.id) continue; const health = { label: item.label, enabled: item.enabled, priority: item.priority, providerType: item.providerType, providerName: item.providerName, providerModelId: item.providerModelId, enabledModelIds: item.enabledModelIds, modelMappings: item.modelMappings, resolutionModelMap: item.resolutionModelMap, retryBudget: item.retryBudget, fixtureFailure: item.fixtureFailure, health: item.health, consecutiveFailures: item.consecutiveFailures, circuitOpenedAt: item.circuitOpenedAt, circuitResetAt: item.circuitResetAt }; await client.query("INSERT INTO provider_channels(id, status, encrypted_secret_ref, base_url, health, version) VALUES ($1, $2, $3, $4, $5, $6)", [String(item.id), String(item.health || "healthy"), typeof item.secretRef === "string" ? item.secretRef : "", typeof item.baseUrl === "string" ? item.baseUrl : "", jsonValue(health), numberValue(item.version, 1)]); }
|
||||
for (const raw of arrayEntries(source.channelGroups)) { const item = record(raw); if (!item.id) continue; await client.query("INSERT INTO channel_groups(id, version, enabled, channel_order) VALUES ($1, $2, $3, $4)", [String(item.id), numberValue(item.version, 1), item.enabled !== false, jsonValue({ name: item.name, capabilities: item.capabilities || [], channelIds: item.channelIds || [], channelPriorities: item.channelPriorities || {}, retryBudget: item.retryBudget })]); }
|
||||
const taskEntries = mapEntries(source.tasks); const taskIds = new Set<string>();
|
||||
for (const [key, raw] of taskEntries) { const item = record(raw); const id = uuid(key) || uuid(item.id); const userId = uuid(item.ownerId); if (!id || !userId) continue; taskIds.add(id); await client.query("INSERT INTO generation_tasks(id, user_id, task_type, status, idempotency_key, model_snapshot, route_snapshot, pricing_snapshot, task_snapshot, reserved_amount, state_version, event_sequence, reserve_expires_at, lease_expires_at, lease_token, updated_at, created_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $11, $12, $13, $14, $15, $16)", [id, userId, String(item.taskType || "image"), String(item.status || "queued"), String(item.idempotencyKey || id), jsonValue(item.modelSnapshot), jsonValue(item.routeSnapshot), jsonValue(item.pricingSnapshot), jsonValue({ modelProductId: item.modelProductId, prompt: item.prompt, resolutionPresetId: item.resolutionPresetId, count: item.count, estimatedCost: item.estimatedCost, channelGroupId: item.channelGroupId, routeSnapshotVersion: record(item.routeSnapshot).version, retentionExpiresAt: item.retentionExpiresAt, retentionState: item.retentionState, retentionExtensionDays: item.retentionExtensionDays, reserveExpiresAt: item.reserveExpiresAt, inputSnapshot: item.inputSnapshot, planSnapshot: item.planSnapshot, requestFingerprint: item.requestFingerprint, providerSubmitted: item.providerSubmitted, publicErrorCode: item.publicErrorCode }), numberValue(item.reservedCost), numberValue(item.eventSequence, 0), item.reserveExpiresAt ? iso(item.reserveExpiresAt) : null, item.leaseExpiresAt ? iso(item.leaseExpiresAt) : null, item.leaseToken || null, iso(item.updatedAt, iso(item.createdAt)), iso(item.createdAt)]); for (const rawAttempt of arrayEntries(item.attempts)) { const attempt = record(rawAttempt); const attemptId = uuid(attempt.id); if (!attemptId || !attempt.channelId) continue; await client.query("INSERT INTO generation_attempts(id, task_id, channel_id, sequence, status, provider_request_id, error_code, platform_idempotency_key, reconciliation_status, lease_expires_at, started_at, finished_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)", [attemptId, id, String(attempt.channelId), numberValue(attempt.sequence, 1), String(attempt.status || "started"), typeof attempt.providerRequestId === "string" ? attempt.providerRequestId : null, typeof attempt.errorCode === "string" ? attempt.errorCode : null, attempt.platformIdempotencyKey || null, attempt.reconciliationStatus || null, attempt.leaseExpiresAt ? iso(attempt.leaseExpiresAt) : null, iso(attempt.startedAt), attempt.finishedAt ? iso(attempt.finishedAt) : null]); } for (const rawOutput of arrayEntries(item.outputs)) { const output = record(rawOutput); const outputId = uuid(output.id); if (!outputId) continue; await client.query("INSERT INTO generation_outputs(id, task_id, owner_id, object_key, mime_type, status, expires_at, revoked_at, unit_price_snapshot, charged_amount, metadata, created_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)", [outputId, id, userId, String(output.url || output.id), String(output.mimeType || "application/octet-stream"), String(output.status || "succeeded"), iso(item.retentionExpiresAt, new Date(Date.now() + 86_400_000).toISOString()), null, output.unitPriceSnapshot === undefined ? null : numberValue(output.unitPriceSnapshot), output.chargedAmount === undefined ? null : numberValue(output.chargedAmount), jsonValue(output.metadata), iso(item.createdAt)]); } }
|
||||
for (const [key, raw] of mapEntries(source.taskDispatchOutbox)) { const item = record(raw); const taskId = uuid(key) || uuid(item.taskId); if (!taskId || !taskIds.has(taskId)) continue; await client.query("INSERT INTO task_dispatch_outbox(task_id, status, available_at, attempts, lease_owner, lease_expires_at, last_error, dispatched_at, created_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) ON CONFLICT (task_id) DO UPDATE SET status = EXCLUDED.status, available_at = EXCLUDED.available_at, attempts = EXCLUDED.attempts, lease_owner = EXCLUDED.lease_owner, lease_expires_at = EXCLUDED.lease_expires_at, last_error = EXCLUDED.last_error, dispatched_at = EXCLUDED.dispatched_at", [taskId, String(item.status || "pending"), iso(item.availableAt), numberValue(item.attempts), item.leaseOwner || null, item.leaseExpiresAt ? iso(item.leaseExpiresAt) : null, item.lastError || null, item.dispatchedAt ? iso(item.dispatchedAt) : null, iso(item.createdAt)]); }
|
||||
const events = mapEntries(source.events); for (const [taskKey, rawEvents] of events) { const taskId = uuid(taskKey); if (!taskId || !taskIds.has(taskId)) continue; for (const rawEvent of arrayEntries(rawEvents)) { const event = record(rawEvent); const eventId = uuid(event.eventId); if (!eventId) continue; await client.query("INSERT INTO task_event_log(task_id, sequence, event_id, event_type, payload, occurred_at) VALUES ($1, $2, $3, $4, $5, $6)", [taskId, numberValue(event.sequence, 1), eventId, String(event.type || "task.progress"), jsonValue(event.payload), iso(event.occurredAt)]); } }
|
||||
const plans = arrayEntries(source.plans); for (const raw of plans) { const item = record(raw); if (!item.id || !item.code) continue; await client.query("INSERT INTO membership_plans(id, code, version, snapshot, published) VALUES ($1, $2, $3, $4, $5)", [String(item.id), String(item.code), numberValue(item.version, 1), jsonValue(item), item.published === true]); }
|
||||
for (const [key, raw] of mapEntries(source.planPurchases)) { const item = record(raw); const id = uuid(key) || uuid(item.id); const userId = uuid(item.userId); if (!id || !userId) continue; await client.query("INSERT INTO plan_purchases(id, user_id, plan_id, plan_version, provider, status, purchase_amount, included_balance, idempotency_key, request_fingerprint, plan_snapshot, credit_bucket_id, entitlement_id, created_at, paid_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15)", [id, userId, String(item.planId || ""), numberValue(item.planVersion, 1), String(item.provider || "adapter"), String(item.status || "pending"), numberValue(item.purchaseAmount), numberValue(item.includedBalance), String(item.idempotencyKey || id), typeof item.requestFingerprint === "string" ? item.requestFingerprint : null, jsonValue(item.planSnapshot), uuid(item.creditBucketId) || null, uuid(item.entitlementId) || null, iso(item.createdAt), item.paidAt ? iso(item.paidAt) : null]); }
|
||||
for (const [key, raw] of mapEntries(source.entitlements)) { const item = record(raw); const userId = uuid(item.userId); if (!userId || !item.planId) continue; await client.query("INSERT INTO user_plan_entitlements(user_id, plan_id, version, starts_at, expires_at, snapshot) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (user_id, plan_id, version) DO UPDATE SET expires_at = EXCLUDED.expires_at, snapshot = EXCLUDED.snapshot", [userId, String(item.planId), numberValue(item.planVersion, 1), iso(item.createdAt), iso(item.expiresAt), jsonValue(item)]); }
|
||||
for (const [key, raw] of mapEntries(source.reserveAllocations)) { const userId = uuid(String(key).split(":")[0]); const referenceId = uuid(String(key).split(":").slice(1).join(":")); if (!userId || !referenceId) continue; for (const allocation of arrayEntries(raw)) { const item = record(allocation); const bucketId = uuid(item.bucketId); if (!bucketId) continue; await client.query("INSERT INTO reserve_allocations(user_id, reference_id, bucket_id, amount) VALUES ($1, $2, $3, $4)", [userId, referenceId, bucketId, numberValue(item.amount)]); } }
|
||||
for (const [key, raw] of mapEntries(source.rechargeOrders)) { const item = record(raw); const id = uuid(key) || uuid(item.id); const userId = uuid(item.userId); if (!id || !userId) continue; await client.query("INSERT INTO recharge_orders(id, user_id, provider, status, fiat_amount, balance_amount, unit_version, idempotency_key, request_fingerprint, credit_bucket_id, created_at, paid_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)", [id, userId, String(item.provider || "adapter"), String(item.status || "pending"), numberValue(item.fiatAmount), numberValue(item.balanceAmount), numberValue(item.unitVersion, 1), String(item.idempotencyKey || id), typeof item.requestFingerprint === "string" ? item.requestFingerprint : null, uuid(item.creditBucketId) || null, iso(item.createdAt), item.paidAt ? iso(item.paidAt) : null]); }
|
||||
const orderIds = new Set(mapEntries(source.rechargeOrders).map(([key, raw]) => uuid(key) || uuid(record(raw).id)).filter((value): value is string => Boolean(value)));
|
||||
for (const [key, raw] of mapEntries(source.paymentEvents)) { const item = record(raw); const id = uuid(item.id) || stableUuid(`payment:${key}`); const orderId = uuid(item.orderId); if (!orderId || !orderIds.has(orderId)) continue; const externalEventId = String(item.externalEventId || key); await client.query("INSERT INTO payment_events(id, provider, provider_event_id, external_event_id, order_id, payload_digest, status, created_at) VALUES ($1, $2, $3, $3, $4, $5, $6, $7)", [id, String(item.provider || "adapter"), externalEventId, orderId, String(item.payloadDigest || ""), String(item.status || "received"), iso(item.createdAt)]); }
|
||||
const purchaseIds = new Set(mapEntries(source.planPurchases).map(([key, raw]) => uuid(key) || uuid(record(raw).id)).filter((value): value is string => Boolean(value)));
|
||||
for (const [key, raw] of mapEntries(source.paymentEvents)) { const item = record(raw); const id = uuid(item.id) || stableUuid(`payment:${key}`); const purchaseId = uuid(item.orderId); if (!purchaseId || !purchaseIds.has(purchaseId)) continue; await client.query("INSERT INTO plan_payment_events(id, provider, external_event_id, purchase_id, payload_digest, status, created_at, applied_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)", [id, String(item.provider || "adapter"), String(item.externalEventId || key), purchaseId, String(item.payloadDigest || ""), String(item.status || "received"), iso(item.createdAt), item.appliedAt ? iso(item.appliedAt) : null]); }
|
||||
for (const [key, raw] of mapEntries(source.uploads)) { const item = record(raw); const id = uuid(key) || uuid(item.id); const userId = uuid(item.ownerId); if (!id || !userId) continue; await client.query("INSERT INTO upload_objects(id, owner_id, object_key, mime_type, byte_size, checksum, scan_status, expires_at, deleted_at, idempotency_key) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)", [id, userId, String(item.stagingKey || id), String(item.mimeType || "application/octet-stream"), numberValue(item.size), String(item.checksum || ""), String(item.status || "pending"), iso(item.expiresAt), item.status === "deleted" ? new Date().toISOString() : null, item.idempotencyKey || null]); }
|
||||
const uploadIds = new Set(mapEntries(source.uploads).map(([key, raw]) => uuid(key) || uuid(record(raw).id)).filter((value): value is string => Boolean(value)));
|
||||
for (const [key, raw] of mapEntries(source.assets)) { const item = record(raw); const id = uuid(key) || uuid(item.id); const ownerId = uuid(item.ownerId); if (!id || !ownerId) continue; const objectId = uuid(item.objectId); const taskId = uuid(item.taskId); await client.query("INSERT INTO assets(id, owner_id, task_id, object_id, asset_type, snapshot, deleted_at, retention_expires_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)", [id, ownerId, taskId && taskIds.has(taskId) ? taskId : null, objectId && uploadIds.has(objectId) ? objectId : null, String(item.type || "image"), jsonValue(item), item.deletedAt ? iso(item.deletedAt) : null, item.retentionExpiresAt ? iso(item.retentionExpiresAt) : null]); }
|
||||
const storagePolicy = record(source.storagePolicy); if (storagePolicy.id) await client.query("INSERT INTO storage_policies(id, version, snapshot, published) VALUES ($1, $2, $3, true) ON CONFLICT (id) DO UPDATE SET version = EXCLUDED.version, snapshot = EXCLUDED.snapshot, published = true", [String(storagePolicy.id), numberValue(storagePolicy.version, 1), jsonValue(storagePolicy)]);
|
||||
for (const [key, raw] of mapEntries(source.messageOutbox)) { const item = record(raw); const id = uuid(key) || uuid(item.id); if (!id) continue; await client.query("INSERT INTO email_sms_outbox(id, idempotency_key, target_hash, channel, purpose, payload, status, attempts, next_attempt_at, lease_owner, lease_expires_at, provider_message_id, last_error, created_at, updated_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15)", [id, item.idempotencyKey || null, String(item.targetHash || ""), String(item.channel || "email"), String(item.purpose || "notification"), jsonValue({ targetEncrypted: item.targetEncrypted, payloadEncrypted: item.payloadEncrypted }), String(item.status || "queued"), numberValue(item.attempts), item.nextAttemptAt ? iso(item.nextAttemptAt) : new Date().toISOString(), item.leaseOwner || null, item.leaseExpiresAt ? iso(item.leaseExpiresAt) : null, item.providerMessageId || null, item.lastError || null, iso(item.createdAt), iso(item.updatedAt, iso(item.createdAt))]); }
|
||||
for (const [key, raw] of mapEntries(source.webdav)) { const item = record(raw); const userId = uuid(key) || uuid(item.userId); if (!userId) continue; await client.query("INSERT INTO webdav_configs(user_id, encrypted_url_ref, encrypted_username_ref, encrypted_password_ref, directory, state, retention_state, last_synced_at, retention_days, manifest_retention_expires_at, manifest_extension_days) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)", [userId, String(item.encryptedUrl || ""), item.encryptedUsername || null, item.encryptedPassword || null, String(item.directory || "miragenflow"), String(item.state || "unconfigured"), String(item.retentionState || "active"), item.lastSyncedAt ? iso(item.lastSyncedAt) : null, numberValue(item.retentionDays, 30), item.manifestRetentionExpiresAt ? iso(item.manifestRetentionExpiresAt) : null, numberValue(item.manifestExtensionDays)]); if (item.manifestVersion !== undefined || item.manifestChecksum || item.manifestEtag) await client.query("INSERT INTO webdav_manifests(id, user_id, version, checksum, manifest, state, created_at) VALUES ($1, $2, $3, $4, $5, $6, $7)", [stableUuid(`webdav-manifest:${userId}`), userId, numberValue(item.manifestVersion, 0), String(item.manifestChecksum || ""), jsonValue({ etag: item.manifestEtag, version: item.manifestVersion, checksum: item.manifestChecksum }), String(item.state || "ready"), iso(item.lastSyncedAt)]); }
|
||||
for (const raw of arrayEntries(source.audit)) { const item = record(raw); const id = uuid(item.id); if (!id) continue; await client.query("INSERT INTO audit_logs(id, actor_id, action, object_type, object_id, request_id, payload_digest, created_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)", [id, uuid(item.actorId) || null, String(item.action || "unknown"), String(item.objectType || "unknown"), typeof item.objectId === "string" ? item.objectId : null, String(item.requestId || id), String(item.hash || item.previousHash || ""), iso(item.createdAt)]); }
|
||||
for (const [key, raw] of mapEntries(source.verificationCodes)) { const item = record(raw); await client.query("INSERT INTO verification_codes(id, record_key, record_kind, target_hash, purpose, code_hash, expires_at, attempts, max_attempts, consumed_at, ip, device_id) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)", [stableUuid(`verification:${key}`), String(key), "verification", String(item.targetHash || ""), String(item.purpose || "register"), String(item.codeHash || ""), dateValue(item.expiresAt), numberValue(item.attempts), numberValue(item.maxAttempts, 5), item.consumedAt ? dateValue(item.consumedAt) : null, item.ip || null, item.deviceId || null]); }
|
||||
for (const [key, raw] of mapEntries(source.passwordResetCodes)) { const item = record(raw); await client.query("INSERT INTO verification_codes(id, record_key, record_kind, target_hash, purpose, code_hash, expires_at, attempts, max_attempts, consumed_at, ip, device_id) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)", [stableUuid(`reset:${key}`), String(key), "password_reset", String(item.targetHash || ""), String(item.purpose || "reset"), String(item.codeHash || ""), dateValue(item.expiresAt), numberValue(item.attempts), numberValue(item.maxAttempts, 5), item.consumedAt ? dateValue(item.consumedAt) : null, item.ip || null, item.deviceId || null]); }
|
||||
for (const [key, raw] of mapEntries(source.mfaChallenges)) { const item = record(raw); await client.query("INSERT INTO mfa_challenges(id, subject_id, scope, purpose, expires_at, attempts, pending_secret_encrypted, pending_recovery_code_hashes) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)", [String(key), String(item.userId || ""), String(item.scope || "user"), String(item.purpose || "login"), dateValue(item.expiresAt), numberValue(item.attempts), item.pendingSecretEncrypted || null, jsonValue(item.pendingRecoveryCodeHashes, [])]); }
|
||||
for (const [key, raw] of mapEntries(source.captchaChallenges)) { const item = record(raw); await client.query("INSERT INTO captcha_challenges(id, action, answer_hash, target_hash, expires_at, attempts, max_attempts, consumed_at, ip, device_id) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)", [String(item.id || key), String(item.action || "login"), String(item.answerHash || ""), String(item.targetHash || ""), dateValue(item.expiresAt), numberValue(item.attempts), numberValue(item.maxAttempts, 5), item.consumedAt ? dateValue(item.consumedAt) : null, String(item.ip || ""), item.deviceId || null]); }
|
||||
for (const [key, raw] of mapEntries(source.sessionFamilies)) { const item = record(raw); await client.query("INSERT INTO session_families(family_id, subject_id, scope, revoked_at) VALUES ($1, $2, $3, $4)", [String(key), String(item.userId || ""), String(item.scope || "user"), item.revokedAt ? dateValue(item.revokedAt) : null]); }
|
||||
for (const [key, raw] of mapEntries(source.messageProviders)) { const item = record(raw); const id = String(item.id || key); await client.query("INSERT INTO message_providers(id, snapshot) VALUES ($1, $2)", [id, jsonValue(item)]); }
|
||||
for (const [key, raw] of mapEntries(source.messageTemplates)) { const item = record(raw); const id = String(item.id || key); await client.query("INSERT INTO message_templates(id, snapshot) VALUES ($1, $2)", [id, jsonValue(item)]); }
|
||||
for (const [key, raw] of mapEntries(source.paymentProviders)) { const item = record(raw); const id = String(item.id || key); await client.query("INSERT INTO payment_providers(id, snapshot) VALUES ($1, $2)", [id, jsonValue(item)]); }
|
||||
for (const [key, raw] of mapEntries(source.webdavFiles)) { const item = record(raw); const userId = String(item.userId || String(key).split(":", 1)[0]); const path = String(item.path || String(key).slice(userId.length + 1)); await client.query("INSERT INTO webdav_files(user_id, path, mime_type, data, checksum, etag, version, updated_at, deleted_at, sync_state) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)", [userId, path, String(item.mimeType || "application/octet-stream"), String(item.data || ""), String(item.checksum || ""), String(item.etag || ""), numberValue(item.version, 1), iso(item.updatedAt), item.deletedAt ? iso(item.deletedAt) : null, String(item.syncState || "synced")]); }
|
||||
for (const [key, raw] of mapEntries(source.webdavJobs)) { const item = record(raw); await client.query("INSERT INTO webdav_sync_jobs(id, user_id, operation, intent, path, data, mime_type, if_match, attempts, status, next_attempt_at, lease_owner, lease_expires_at, last_error, conflict_copy_path, created_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16)", [String(item.id || key), String(item.userId || ""), String(item.operation || "put"), item.intent || null, String(item.path || ""), item.data || null, item.mimeType || null, item.ifMatch || null, numberValue(item.attempts), String(item.status || "queued"), iso(item.nextAttemptAt), item.leaseOwner || null, item.leaseExpiresAt ? iso(item.leaseExpiresAt) : null, item.lastError || null, item.conflictCopyPath || null, iso(item.createdAt)]); }
|
||||
for (const [key, raw] of mapEntries(source.objects)) { const item = record(raw); await client.query("INSERT INTO stored_objects(id, owner_id, mime_type, data, staging_key, expires_at, task_id, revoked) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)", [String(item.id || key), String(item.ownerId || ""), String(item.mimeType || "application/octet-stream"), item.data || null, item.stagingKey || null, iso(item.expiresAt), item.taskId || null, item.revoked === true]); }
|
||||
const runtimeRoots = ["sessionHashes", "refreshReuse", "revokedAccessBefore", "idempotency", "idempotencyBodies", "passwordResetRequests", "rateLimits", "rechargeIdempotency", "queue"];
|
||||
for (const root of runtimeRoots) for (const [key, raw] of mapEntries(source[root])) await client.query("INSERT INTO runtime_records(record_type, record_key, payload) VALUES ($1, $2, $3)", [root, String(key), jsonValue(raw)]);
|
||||
for (const [key, raw] of mapEntries(source.adminIdempotency)) { const parts = String(key).split(":"); const actorId = parts.shift() || ""; const idemKey = parts.pop() || ""; const route = parts.join(":"); const item = record(raw); await client.query("INSERT INTO admin_idempotency(actor_id, route, idem_key, request_fingerprint, claim_token, lease_expires_at, state, status_code, response, created_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)", [actorId, route, idemKey, String(item.fingerprint || ""), item.claimToken || null, item.leaseExpiresAt ? iso(item.leaseExpiresAt) : null, String(item.state || "pending"), item.statusCode === undefined ? null : numberValue(item.statusCode), item.data === undefined ? null : jsonValue(item.data), iso(item.createdAt)]); }
|
||||
for (const [key, raw] of mapEntries(source.adminApprovals)) { const item = record(raw); await client.query("INSERT INTO admin_approval_requests(id, kind, requester_id, approver_id, status, payload, reason, version, created_at, expires_at, approved_at, executed_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)", [String(key), String(item.kind || "high_risk_change"), String(item.requesterId || ""), item.approverId || null, String(item.status || "pending"), jsonValue(item.payload), String(item.reason || ""), numberValue(item.version, 1), iso(item.createdAt), iso(item.expiresAt), item.approvedAt ? iso(item.approvedAt) : null, item.executedAt ? iso(item.executedAt) : null]); }
|
||||
for (const [key, raw] of mapEntries(source.adminAccounts)) { const item = record(raw); await client.query("INSERT INTO admin_accounts(id, email, password_hash, role, mfa_secret_encrypted, mfa_required, recovery_code_hashes, mfa_last_totp_counter) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)", [String(item.id || key), String(item.email || ""), String(item.password || ""), String(item.role || "operator"), item.mfaSecretEncrypted || null, item.mfaRequired === true, jsonValue(item.recoveryCodeHashes, []), item.mfaLastTotpCounter === undefined ? null : numberValue(item.mfaLastTotpCounter)]); }
|
||||
}
|
||||
|
||||
function mergeMapRoot(base: SnapshotRecord | undefined, incoming: SnapshotRecord, root: string) {
|
||||
const baseEntries = mapEntries(base?.[root]); const incomingEntries = mapEntries(incoming[root]); if (!incomingEntries.length) return;
|
||||
const merged = new Map(baseEntries.map(([key, value]) => [String(key), value])); for (const [key, value] of incomingEntries) { const previous = merged.get(String(key)); merged.set(String(key), isRecord(previous) && isRecord(value) ? { ...previous, ...value } : value); } incoming[root] = [...merged.entries()];
|
||||
}
|
||||
function mergeArrayRoot(base: SnapshotRecord | undefined, incoming: SnapshotRecord, root: string) {
|
||||
const values = arrayEntries(incoming[root]); if (!values.length) return;
|
||||
const existing = new Map(arrayEntries(base?.[root]).map((value) => [String(record(value).id || ""), value])); for (const value of values) { const key = String(record(value).id || ""); existing.set(key, existing.has(key) ? { ...record(existing.get(key)), ...record(value) } : value); } incoming[root] = [...existing.values()];
|
||||
}
|
||||
function mergeSnapshots(base: Record<string, unknown> | undefined, canonical: SnapshotRecord) {
|
||||
const output = { ...(base || {}), ...canonical } as SnapshotRecord;
|
||||
for (const root of ["users", "sessions", "balances", "buckets", "tasks", "planPurchases", "entitlements", "reserveAllocations", "rechargeOrders", "paymentEvents", "uploads", "assets", "messageOutbox", "webdav", "webdavFiles", "webdavJobs", "captchaChallenges", "adminAccounts"]) mergeMapRoot(base, output, root);
|
||||
for (const root of ["products", "channels", "channelGroups", "plans", "ledger", "audit"]) mergeArrayRoot(base, output, root);
|
||||
for (const [key, value] of Object.entries(base || {})) if (output[key] === undefined) output[key] = value;
|
||||
output.snapshotVersion = 1;
|
||||
return output;
|
||||
}
|
||||
|
||||
async function loadCanonicalSnapshot(client: PgClient): Promise<SnapshotRecord | undefined> {
|
||||
if (!(await canonicalTablesAvailable(client))) return undefined;
|
||||
const output: SnapshotRecord = {};
|
||||
const users = await client.query("SELECT id, status, display_name, created_at, mfa_secret_encrypted, mfa_recovery_code_hashes, mfa_last_totp_counter, mfa_required, mfa_enabled, roles, failed_login_count, locked_until FROM users ORDER BY created_at, id");
|
||||
if (users.rows.length) {
|
||||
const identities = await client.query("SELECT user_id, kind, normalized_value, verified_at FROM user_identities"); const passwords = await client.query("SELECT user_id, password_hash FROM password_credentials");
|
||||
const identityMap = new Map<string, SnapshotRecord>(); for (const row of identities.rows) { const userId = String(row.user_id); const item = identityMap.get(userId) || {}; if (row.kind === "username" || row.kind === "email" || row.kind === "phone") item[String(row.kind)] = row.normalized_value; item.verified = Boolean(row.verified_at) || item.verified === true; identityMap.set(userId, item); }
|
||||
const passwordMap = new Map(passwords.rows.map((row) => [String(row.user_id), String(row.password_hash || "")]));
|
||||
output.users = users.rows.map((row) => { const id = String(row.id); const extra = identityMap.get(id) || {}; return [id, { id, version: 1, username: extra.username, status: row.status, verified: row.status === "active" || extra.verified === true, password: passwordMap.get(id) || "", email: extra.email, phone: extra.phone, displayName: row.display_name || undefined, createdAt: iso(row.created_at), mfaRequired: row.mfa_required === true, mfaEnabled: row.mfa_enabled === true, mfaSecretEncrypted: row.mfa_secret_encrypted || undefined, mfaRecoveryCodeHashes: arrayEntries(parseJson(row.mfa_recovery_code_hashes, [])), mfaLastTotpCounter: row.mfa_last_totp_counter === null ? undefined : Number(row.mfa_last_totp_counter), roles: arrayEntries(parseJson(row.roles, [])), failedLoginCount: Number(row.failed_login_count || 0), lockedUntil: row.locked_until === null ? undefined : Number(row.locked_until) }]; });
|
||||
}
|
||||
const sessions = await client.query("SELECT refresh_hash, subject_id, scope, expires_at, absolute_expires_at, revoked_at, created_at, rotation_id, family_id, device_id, replaced_by, roles, scopes FROM sessions"); if (sessions.rows.length) output.sessions = sessions.rows.map((row) => [String(row.refresh_hash), { userId: String(row.subject_id), scope: row.scope, expiresAt: new Date(String(row.expires_at)).getTime(), absoluteExpiresAt: new Date(String(row.absolute_expires_at || row.expires_at)).getTime(), createdAt: new Date(String(row.created_at)).getTime(), rotationId: String(row.rotation_id), familyId: String(row.family_id), revokedAt: row.revoked_at ? new Date(String(row.revoked_at)).getTime() : undefined, replacedBy: row.replaced_by || undefined, deviceId: row.device_id || undefined, roles: arrayEntries(parseJson(row.roles, [])), scopes: arrayEntries(parseJson(row.scopes, [])) }]);
|
||||
const families = await client.query("SELECT family_id, subject_id, scope, revoked_at FROM session_families"); if (families.rows.length) output.sessionFamilies = families.rows.map((row) => [String(row.family_id), { userId: String(row.subject_id), scope: row.scope, revokedAt: row.revoked_at ? new Date(String(row.revoked_at)).getTime() : undefined }]);
|
||||
const verification = await client.query("SELECT record_key, record_kind, target_hash, purpose, code_hash, expires_at, attempts, max_attempts, consumed_at, ip, device_id FROM verification_codes"); for (const row of verification.rows) { const key = String(row.record_key || ""); if (!key) continue; const value = { codeHash: String(row.code_hash || ""), expiresAt: new Date(String(row.expires_at)).getTime(), attempts: Number(row.attempts), maxAttempts: Number(row.max_attempts || 5), consumedAt: row.consumed_at ? new Date(String(row.consumed_at)).getTime() : undefined, targetHash: String(row.target_hash || ""), purpose: row.record_kind === "password_reset" ? "reset" : String(row.purpose || "register"), ip: row.ip || undefined, deviceId: row.device_id || undefined }; const root = row.record_kind === "password_reset" ? "passwordResetCodes" : "verificationCodes"; const list = (output[root] as Array<[string, unknown]> | undefined) || []; list.push([key, value]); output[root] = list; }
|
||||
const mfa = await client.query("SELECT id, subject_id, scope, purpose, expires_at, attempts, pending_secret_encrypted, pending_recovery_code_hashes FROM mfa_challenges"); if (mfa.rows.length) output.mfaChallenges = mfa.rows.map((row) => [String(row.id), { userId: String(row.subject_id), scope: row.scope, purpose: row.purpose || "login", expiresAt: new Date(String(row.expires_at)).getTime(), attempts: Number(row.attempts), pendingSecretEncrypted: row.pending_secret_encrypted || undefined, pendingRecoveryCodeHashes: arrayEntries(parseJson(row.pending_recovery_code_hashes, [])) }]);
|
||||
const captcha = await client.query("SELECT id, action, answer_hash, target_hash, expires_at, attempts, max_attempts, consumed_at, ip, device_id FROM captcha_challenges"); if (captcha.rows.length) output.captchaChallenges = captcha.rows.map((row) => [String(row.id), { id: String(row.id), action: row.action, answerHash: String(row.answer_hash), targetHash: String(row.target_hash || ""), expiresAt: new Date(String(row.expires_at)).getTime(), attempts: Number(row.attempts), maxAttempts: Number(row.max_attempts || 5), consumedAt: row.consumed_at ? new Date(String(row.consumed_at)).getTime() : undefined, ip: String(row.ip || ""), deviceId: row.device_id || undefined }]);
|
||||
const adminAccounts = await client.query("SELECT id, email, password_hash, role, mfa_secret_encrypted, mfa_required, recovery_code_hashes, mfa_last_totp_counter FROM admin_accounts ORDER BY id"); if (adminAccounts.rows.length) output.adminAccounts = adminAccounts.rows.map((row) => [String(row.id), { id: String(row.id), email: String(row.email), password: String(row.password_hash), role: String(row.role), mfaSecretEncrypted: row.mfa_secret_encrypted || undefined, mfaRequired: row.mfa_required === true, recoveryCodeHashes: arrayEntries(parseJson(row.recovery_code_hashes, [])), mfaLastTotpCounter: row.mfa_last_totp_counter === null ? undefined : Number(row.mfa_last_totp_counter) }]);
|
||||
const balances = await client.query("SELECT user_id, available, reserved, version FROM balance_accounts"); if (balances.rows.length) output.balances = balances.rows.map((row) => [String(row.user_id), { available: Number(row.available), reserved: Number(row.reserved), version: Number(row.version || 0) }]);
|
||||
const units = await client.query("SELECT id, version, name, short_code, icon, precision, currency_code, conversion_rate, rounding FROM balance_units ORDER BY version DESC LIMIT 1"); if (units.rows[0]) { const row = units.rows[0]; output.balanceUnit = { name: String(row.name), shortCode: String(row.short_code), icon: String(row.icon), precision: Number(row.precision), version: Number(row.version), currencyCode: String(row.currency_code), conversionRate: Number(row.conversion_rate), rounding: row.rounding }; }
|
||||
const buckets = await client.query("SELECT id, user_id, source, remaining, priority, expires_at FROM balance_buckets"); if (buckets.rows.length) { const grouped = new Map<string, unknown[]>(); for (const row of buckets.rows) { const key = String(row.user_id); const list = grouped.get(key) || []; list.push({ id: String(row.id), userId: key, source: row.source, remaining: Number(row.remaining), priority: Number(row.priority), expiresAt: row.expires_at ? new Date(String(row.expires_at)).toISOString() : undefined }); grouped.set(key, list); } output.buckets = [...grouped.entries()]; }
|
||||
const ledger = await client.query("SELECT id, user_id, type, amount, unit_version, idempotency_key, reference_id, task_id, output_id, bucket_id, expires_at, status, created_at FROM balance_ledger ORDER BY created_at, id"); if (ledger.rows.length) output.ledger = ledger.rows.map((row) => ({ id: String(row.id), userId: String(row.user_id), type: row.type, amount: Number(row.amount), balanceUnitVersion: Number(row.unit_version), idempotencyKey: row.idempotency_key, referenceId: row.reference_id || undefined, taskId: row.task_id || undefined, outputId: row.output_id || undefined, bucketId: row.bucket_id || undefined, expiresAt: row.expires_at ? new Date(String(row.expires_at)).toISOString() : undefined, createdAt: new Date(String(row.created_at)).toISOString(), status: row.status || "posted" }));
|
||||
const products = await client.query("SELECT id, snapshot FROM model_products ORDER BY id"); if (products.rows.length) output.products = products.rows.map((row) => parseJson(row.snapshot, { id: String(row.id) }));
|
||||
const channels = await client.query("SELECT id, status, encrypted_secret_ref, base_url, health, version FROM provider_channels ORDER BY id"); if (channels.rows.length) output.channels = channels.rows.map((row) => { const health = record(parseJson(row.health)); return { id: String(row.id), label: String(health.label || row.id), enabled: health.enabled !== false, priority: Number(health.priority || 0), health: row.status, providerType: health.providerType, providerName: health.providerName, providerModelId: health.providerModelId, enabledModelIds: Array.isArray(health.enabledModelIds) ? health.enabledModelIds : undefined, modelMappings: Array.isArray(health.modelMappings) ? health.modelMappings : undefined, resolutionModelMap: health.resolutionModelMap, baseUrl: row.base_url || undefined, secretRef: row.encrypted_secret_ref || undefined, version: Number(row.version), retryBudget: Number(health.retryBudget || 0), fixtureFailure: health.fixtureFailure, consecutiveFailures: Number(health.consecutiveFailures || 0), circuitOpenedAt: health.circuitOpenedAt, circuitResetAt: health.circuitResetAt }; });
|
||||
const groups = await client.query("SELECT id, version, enabled, channel_order FROM channel_groups ORDER BY id"); if (groups.rows.length) output.channelGroups = groups.rows.map((row) => { const data = record(parseJson(row.channel_order, {})); return { id: String(row.id), version: Number(row.version), name: typeof data.name === "string" && data.name ? data.name : undefined, capabilities: arrayEntries(data.capabilities), enabled: row.enabled !== false, channelIds: arrayEntries(data.channelIds), channelPriorities: record(data.channelPriorities) as Record<string, number>, retryBudget: Number(data.retryBudget || 0) }; });
|
||||
const plans = await client.query("SELECT id, snapshot FROM membership_plans ORDER BY id"); if (plans.rows.length) output.plans = plans.rows.map((row) => parseJson(row.snapshot, { id: String(row.id) }));
|
||||
const tasks = await client.query("SELECT id, user_id, task_type, status, idempotency_key, model_snapshot, route_snapshot, pricing_snapshot, task_snapshot, reserved_amount, state_version, event_sequence, reserve_expires_at, lease_expires_at, lease_token, updated_at, created_at FROM generation_tasks ORDER BY created_at, id"); if (tasks.rows.length) { const attempts = await client.query("SELECT task_id, id, channel_id, sequence, status, provider_request_id, error_code, platform_idempotency_key, reconciliation_status, lease_expires_at, started_at, finished_at FROM generation_attempts ORDER BY task_id, sequence"); const outputs = await client.query("SELECT id, task_id, owner_id, object_key, mime_type, status, expires_at, revoked_at, unit_price_snapshot, charged_amount FROM generation_outputs ORDER BY task_id, created_at, id"); const events = await client.query("SELECT task_id, sequence, event_id, event_type, payload, occurred_at FROM task_event_log ORDER BY task_id, sequence"); const attemptMap = new Map<string, unknown[]>(); for (const row of attempts.rows) { const key = String(row.task_id); const list = attemptMap.get(key) || []; list.push({ id: String(row.id), channelId: row.channel_id, sequence: Number(row.sequence), status: row.status, providerRequestId: row.provider_request_id || undefined, platformIdempotencyKey: row.platform_idempotency_key || undefined, reconciliationStatus: row.reconciliation_status || undefined, leaseExpiresAt: row.lease_expires_at ? new Date(String(row.lease_expires_at)).toISOString() : undefined, errorCode: row.error_code || undefined, startedAt: new Date(String(row.started_at)).toISOString(), finishedAt: row.finished_at ? new Date(String(row.finished_at)).toISOString() : undefined }); attemptMap.set(key, list); } const outputMap = new Map<string, unknown[]>(); for (const row of outputs.rows) { const key = String(row.task_id); const list = outputMap.get(key) || []; list.push({ id: String(row.id), mimeType: row.mime_type, status: row.status, url: row.object_key, unitPriceSnapshot: row.unit_price_snapshot === null ? undefined : Number(row.unit_price_snapshot), chargedAmount: row.charged_amount === null ? undefined : Number(row.charged_amount) }); outputMap.set(key, list); } const eventMap = new Map<string, unknown[]>(); for (const row of events.rows) { const key = String(row.task_id); const list = eventMap.get(key) || []; list.push({ eventId: String(row.event_id), taskId: key, sequence: Number(row.sequence), type: row.event_type, payload: parseJson(row.payload), occurredAt: new Date(String(row.occurred_at)).toISOString() }); eventMap.set(key, list); } output.tasks = tasks.rows.map((row) => { const id = String(row.id); const taskSnapshot = record(parseJson(row.task_snapshot, {})); return [id, { ...taskSnapshot, id, ownerId: String(row.user_id), taskType: row.task_type, status: row.status, idempotencyKey: row.idempotency_key, reservedCost: Number(row.reserved_amount), eventSequence: Number(row.event_sequence ?? row.state_version), modelSnapshot: parseJson(row.model_snapshot), routeSnapshot: parseJson(row.route_snapshot), pricingSnapshot: parseJson(row.pricing_snapshot), reserveExpiresAt: row.reserve_expires_at ? new Date(String(row.reserve_expires_at)).toISOString() : taskSnapshot.reserveExpiresAt, leaseExpiresAt: row.lease_expires_at ? new Date(String(row.lease_expires_at)).toISOString() : undefined, leaseToken: row.lease_token || undefined, attempts: attemptMap.get(id) || [], outputs: outputMap.get(id) || [], createdAt: new Date(String(row.created_at)).toISOString(), updatedAt: new Date(String(row.updated_at || row.created_at)).toISOString() }]; }); output.events = [...eventMap.entries()]; const taskEntries = mapEntries(output.tasks); output.idempotency = taskEntries.map(([, raw]) => { const item = record(raw); return [`${item.ownerId}:${item.idempotencyKey}`, raw]; }); output.idempotencyBodies = taskEntries.flatMap(([, raw]) => { const item = record(raw); return item.idempotencyKey && item.requestFingerprint ? [[`${item.ownerId}:${item.idempotencyKey}`, item.requestFingerprint]] : []; }); }
|
||||
const dispatch = await client.query("SELECT task_id, status, available_at, attempts, lease_owner, lease_expires_at, last_error, dispatched_at, created_at FROM task_dispatch_outbox ORDER BY created_at, task_id"); if (dispatch.rows.length) output.taskDispatchOutbox = dispatch.rows.map((row) => [String(row.task_id), { taskId: String(row.task_id), status: row.status, availableAt: new Date(String(row.available_at)).toISOString(), attempts: Number(row.attempts || 0), leaseOwner: row.lease_owner || undefined, leaseExpiresAt: row.lease_expires_at ? new Date(String(row.lease_expires_at)).toISOString() : undefined, lastError: row.last_error || undefined, dispatchedAt: row.dispatched_at ? new Date(String(row.dispatched_at)).toISOString() : undefined, createdAt: new Date(String(row.created_at)).toISOString() }]);
|
||||
const orders = await client.query("SELECT id, user_id, provider, status, fiat_amount, balance_amount, unit_version, idempotency_key, request_fingerprint, credit_bucket_id, created_at, paid_at FROM recharge_orders ORDER BY created_at, id"); if (orders.rows.length) output.rechargeOrders = orders.rows.map((row) => [String(row.id), { id: String(row.id), userId: String(row.user_id), provider: row.provider, status: row.status, fiatAmount: Number(row.fiat_amount), balanceAmount: Number(row.balance_amount), unitVersion: Number(row.unit_version), idempotencyKey: row.idempotency_key, requestFingerprint: row.request_fingerprint || undefined, creditBucketId: row.credit_bucket_id || undefined, createdAt: new Date(String(row.created_at)).toISOString(), paidAt: row.paid_at ? new Date(String(row.paid_at)).toISOString() : undefined }]);
|
||||
const rechargeEntries = mapEntries(output.rechargeOrders); if (rechargeEntries.length) output.rechargeIdempotency = rechargeEntries.flatMap(([, raw]) => { const item = record(raw); return item.userId && item.idempotencyKey ? [[`${item.userId}:${item.idempotencyKey}`, raw]] : []; });
|
||||
const purchases = await client.query("SELECT id, user_id, plan_id, plan_version, provider, status, purchase_amount, included_balance, idempotency_key, request_fingerprint, plan_snapshot, credit_bucket_id, entitlement_id, created_at, paid_at FROM plan_purchases ORDER BY created_at, id"); if (purchases.rows.length) output.planPurchases = purchases.rows.map((row) => [String(row.id), { id: String(row.id), userId: String(row.user_id), planId: row.plan_id, planVersion: Number(row.plan_version), provider: row.provider === "mock" ? "mock" : "adapter", status: row.status, purchaseAmount: Number(row.purchase_amount), includedBalance: Number(row.included_balance), idempotencyKey: row.idempotency_key, requestFingerprint: row.request_fingerprint || undefined, planSnapshot: parseJson(row.plan_snapshot, undefined), creditBucketId: row.credit_bucket_id || undefined, entitlementId: row.entitlement_id || undefined, createdAt: new Date(String(row.created_at)).toISOString(), paidAt: row.paid_at ? new Date(String(row.paid_at)).toISOString() : undefined }]);
|
||||
const planEvents = await client.query("SELECT id, provider, external_event_id, purchase_id, payload_digest, status, created_at, applied_at FROM plan_payment_events ORDER BY created_at, id"); if (planEvents.rows.length) { const existing = mapEntries(output.paymentEvents); output.paymentEvents = [...existing, ...planEvents.rows.map((row) => [`${row.provider}:plan:${row.external_event_id}`, { id: String(row.id), provider: row.provider === "mock" ? "mock" : "adapter", externalEventId: row.external_event_id, orderId: String(row.purchase_id), status: row.status, payloadDigest: row.payload_digest, createdAt: new Date(String(row.created_at)).toISOString(), appliedAt: row.applied_at ? new Date(String(row.applied_at)).toISOString() : undefined }])]; }
|
||||
const entitlements = await client.query("SELECT user_id, plan_id, version, starts_at, expires_at, snapshot FROM user_plan_entitlements ORDER BY starts_at, user_id"); if (entitlements.rows.length) output.entitlements = entitlements.rows.map((row) => { const fallbackId = stableUuid(`entitlement:${row.user_id}:${row.plan_id}:${row.version}`); const snapshot = record(parseJson(row.snapshot, {})); return [String(snapshot.id || fallbackId), { ...snapshot, id: String(snapshot.id || fallbackId), userId: String(row.user_id), planId: row.plan_id, planVersion: Number(row.version), expiresAt: new Date(String(row.expires_at)).toISOString() }]; });
|
||||
const allocations = await client.query("SELECT user_id, reference_id, bucket_id, amount FROM reserve_allocations ORDER BY user_id, reference_id, bucket_id"); if (allocations.rows.length) { const grouped = new Map<string, unknown[]>(); for (const row of allocations.rows) { const key = `${row.user_id}:${row.reference_id}`; const list = grouped.get(key) || []; list.push({ bucketId: String(row.bucket_id), amount: Number(row.amount) }); grouped.set(key, list); } output.reserveAllocations = [...grouped.entries()]; }
|
||||
const uploads = await client.query("SELECT id, owner_id, object_key, mime_type, byte_size, checksum, scan_status, expires_at, deleted_at, idempotency_key FROM upload_objects ORDER BY id"); if (uploads.rows.length) output.uploads = uploads.rows.map((row) => [String(row.id), { id: String(row.id), ownerId: String(row.owner_id), idempotencyKey: row.idempotency_key || undefined, stagingKey: row.object_key, mimeType: row.mime_type, size: Number(row.byte_size), checksum: row.checksum, status: row.scan_status, expiresAt: new Date(String(row.expires_at)).toISOString(), ...(row.deleted_at ? { deletedAt: new Date(String(row.deleted_at)).toISOString() } : {}) }]);
|
||||
const assets = await client.query("SELECT id, snapshot FROM assets ORDER BY id"); if (assets.rows.length) output.assets = assets.rows.map((row) => [String(row.id), parseJson(row.snapshot, { id: String(row.id) })]);
|
||||
const outbox = await client.query("SELECT id, idempotency_key, target_hash, channel, purpose, payload, status, attempts, next_attempt_at, lease_owner, lease_expires_at, provider_message_id, last_error, created_at, updated_at FROM email_sms_outbox ORDER BY created_at, id"); if (outbox.rows.length) output.messageOutbox = outbox.rows.map((row) => { const payload = record(parseJson(row.payload)); return [String(row.id), { id: String(row.id), idempotencyKey: row.idempotency_key || undefined, targetHash: row.target_hash, channel: row.channel, purpose: row.purpose, targetEncrypted: payload.targetEncrypted, payloadEncrypted: payload.payloadEncrypted, status: row.status, attempts: Number(row.attempts), nextAttemptAt: row.next_attempt_at ? new Date(String(row.next_attempt_at)).toISOString() : undefined, leaseOwner: row.lease_owner || undefined, leaseExpiresAt: row.lease_expires_at ? new Date(String(row.lease_expires_at)).toISOString() : undefined, providerMessageId: row.provider_message_id || undefined, lastError: row.last_error || undefined, createdAt: new Date(String(row.created_at)).toISOString(), updatedAt: new Date(String(row.updated_at || row.created_at)).toISOString() }]; });
|
||||
const webdav = await client.query("SELECT user_id, encrypted_url_ref, encrypted_username_ref, encrypted_password_ref, directory, state, retention_state, last_synced_at, retention_days, manifest_retention_expires_at, manifest_extension_days FROM webdav_configs"); if (webdav.rows.length) output.webdav = webdav.rows.map((row) => [String(row.user_id), { userId: String(row.user_id), configured: true, encryptedUrl: row.encrypted_url_ref, encryptedUsername: row.encrypted_username_ref || undefined, encryptedPassword: row.encrypted_password_ref || undefined, directory: row.directory, state: row.state, retentionState: row.retention_state || "active", lastSyncedAt: row.last_synced_at ? new Date(String(row.last_synced_at)).toISOString() : undefined, retentionDays: Number(row.retention_days), manifestRetentionExpiresAt: row.manifest_retention_expires_at ? new Date(String(row.manifest_retention_expires_at)).toISOString() : undefined, manifestExtensionDays: Number(row.manifest_extension_days || 0) }]);
|
||||
const webdavManifests = await client.query("SELECT DISTINCT ON (user_id) user_id, version, checksum, manifest, state FROM webdav_manifests ORDER BY user_id, version DESC"); for (const row of webdavManifests.rows) { const entries = mapEntries(output.webdav); const current = entries.find(([key]) => String(key) === String(row.user_id)); if (!current) continue; const manifest = record(parseJson(row.manifest, {})); current[1] = { ...record(current[1]), manifestVersion: Number(row.version), manifestChecksum: row.checksum, manifestEtag: manifest.etag }; output.webdav = entries; }
|
||||
const webdavFiles = await client.query("SELECT user_id, path, mime_type, data, checksum, etag, version, updated_at, deleted_at, sync_state FROM webdav_files ORDER BY user_id, path"); if (webdavFiles.rows.length) output.webdavFiles = webdavFiles.rows.map((row) => [`${row.user_id}:${row.path}`, { userId: String(row.user_id), path: String(row.path), mimeType: row.mime_type, data: row.data || "", checksum: row.checksum, etag: row.etag, version: Number(row.version), updatedAt: new Date(String(row.updated_at)).toISOString(), deletedAt: row.deleted_at ? new Date(String(row.deleted_at)).toISOString() : undefined, syncState: row.sync_state || "synced" }]);
|
||||
const webdavJobs = await client.query("SELECT id, user_id, operation, intent, path, data, mime_type, if_match, attempts, status, next_attempt_at, lease_owner, lease_expires_at, last_error, conflict_copy_path, created_at FROM webdav_sync_jobs ORDER BY created_at, id"); if (webdavJobs.rows.length) output.webdavJobs = webdavJobs.rows.map((row) => [String(row.id), { id: String(row.id), userId: String(row.user_id), operation: row.operation, intent: row.intent || undefined, path: row.path, data: row.data || undefined, mimeType: row.mime_type || undefined, ifMatch: row.if_match || undefined, attempts: Number(row.attempts), status: row.status, nextAttemptAt: new Date(String(row.next_attempt_at)).toISOString(), leaseOwner: row.lease_owner || undefined, leaseExpiresAt: row.lease_expires_at ? new Date(String(row.lease_expires_at)).toISOString() : undefined, lastError: row.last_error || undefined, conflictCopyPath: row.conflict_copy_path || undefined, createdAt: new Date(String(row.created_at)).toISOString() }]);
|
||||
const objects = await client.query("SELECT id, owner_id, mime_type, data, staging_key, expires_at, task_id, revoked FROM stored_objects ORDER BY id"); if (objects.rows.length) output.objects = objects.rows.map((row) => [String(row.id), { id: String(row.id), ownerId: String(row.owner_id), mimeType: row.mime_type, data: row.data || undefined, stagingKey: row.staging_key || undefined, expiresAt: new Date(String(row.expires_at)).toISOString(), taskId: row.task_id || undefined, revoked: row.revoked === true }]);
|
||||
const messageProviders = await client.query("SELECT id, snapshot FROM message_providers ORDER BY id"); if (messageProviders.rows.length) output.messageProviders = messageProviders.rows.map((row) => [String(row.id), parseJson(row.snapshot, { id: String(row.id) })]);
|
||||
const messageTemplates = await client.query("SELECT id, snapshot FROM message_templates ORDER BY id"); if (messageTemplates.rows.length) output.messageTemplates = messageTemplates.rows.map((row) => [String(row.id), parseJson(row.snapshot, { id: String(row.id) })]);
|
||||
const paymentProviders = await client.query("SELECT id, snapshot FROM payment_providers ORDER BY id"); if (paymentProviders.rows.length) output.paymentProviders = paymentProviders.rows.map((row) => [String(row.id), parseJson(row.snapshot, { id: String(row.id) })]);
|
||||
const runtime = await client.query("SELECT record_type, record_key, payload FROM runtime_records ORDER BY record_type, record_key"); for (const row of runtime.rows) { const root = String(row.record_type); const value = parseJson(row.payload); const entries = (output[root] as Array<[string, unknown]> | undefined) || []; entries.push([String(row.record_key), value]); output[root] = entries; }
|
||||
const adminIdempotency = await client.query("SELECT actor_id, route, idem_key, request_fingerprint, claim_token, lease_expires_at, state, status_code, response, created_at FROM admin_idempotency"); if (adminIdempotency.rows.length) output.adminIdempotency = adminIdempotency.rows.map((row) => [`${row.actor_id}:${row.route}:${row.idem_key}`, { fingerprint: row.request_fingerprint, claimToken: row.claim_token || undefined, leaseExpiresAt: row.lease_expires_at ? new Date(String(row.lease_expires_at)).toISOString() : undefined, state: row.state, statusCode: row.status_code === null ? undefined : Number(row.status_code), data: row.response === null ? undefined : parseJson(row.response), createdAt: new Date(String(row.created_at)).toISOString() }]);
|
||||
const approvals = await client.query("SELECT id, kind, requester_id, approver_id, status, payload, reason, version, created_at, expires_at, approved_at, executed_at FROM admin_approval_requests ORDER BY created_at, id"); if (approvals.rows.length) output.adminApprovals = approvals.rows.map((row) => [String(row.id), { id: String(row.id), kind: row.kind, requesterId: row.requester_id, approverId: row.approver_id || undefined, status: row.status, payload: parseJson(row.payload, {}), reason: row.reason, version: Number(row.version), createdAt: new Date(String(row.created_at)).toISOString(), expiresAt: new Date(String(row.expires_at)).toISOString(), approvedAt: row.approved_at ? new Date(String(row.approved_at)).toISOString() : undefined, executedAt: row.executed_at ? new Date(String(row.executed_at)).toISOString() : undefined }]);
|
||||
const policies = await client.query("SELECT snapshot FROM storage_policies ORDER BY version DESC LIMIT 1"); if (policies.rows[0]) output.storagePolicy = parseJson(policies.rows[0].snapshot);
|
||||
return Object.keys(output).length ? output : undefined;
|
||||
}
|
||||
|
||||
const mapRoots = new Set(["users", "sessions", "sessionHashes", "sessionFamilies", "refreshReuse", "revokedAccessBefore", "idempotency", "idempotencyBodies", "retentionIdempotency", "tasks", "events", "balances", "buckets", "reserveAllocations", "planPurchases", "entitlements", "verificationCodes", "passwordResetCodes", "passwordResetRequests", "mfaChallenges", "captchaChallenges", "rateLimits", "rechargeOrders", "rechargeIdempotency", "paymentEvents", "adminIdempotency", "adminApprovals", "adminAccounts", "uploads", "assets", "webdav", "webdavFiles", "webdavJobs", "taskDispatchOutbox", "messageOutbox", "messageProviders", "messageTemplates", "paymentProviders", "objects"]);
|
||||
|
||||
function flattenDomainRows(value: unknown): Array<DomainRow & { root: string }> {
|
||||
if (!value || typeof value !== "object") return [];
|
||||
const rows: Array<DomainRow & { root: string }> = [];
|
||||
for (const [root, rootValue] of Object.entries(value as Record<string, unknown>)) {
|
||||
if (root === "snapshotVersion") continue;
|
||||
if (mapRoots.has(root)) {
|
||||
const entries = Array.isArray(rootValue) ? rootValue : [];
|
||||
if (!entries.length) rows.push({ root, entityId: "__empty__", shape: "map", payload: null });
|
||||
for (const entry of entries) if (Array.isArray(entry) && entry.length === 2) rows.push({ root, entityId: String(entry[0]), shape: "map", payload: entry[1] });
|
||||
continue;
|
||||
}
|
||||
if (Array.isArray(rootValue)) {
|
||||
if (!rootValue.length) rows.push({ root, entityId: "__empty__", shape: "array", payload: null });
|
||||
rootValue.forEach((item, index) => rows.push({ root, entityId: String(typeof item === "object" && item && "id" in item ? (item as { id?: unknown }).id || index : index).padStart(24, "0"), shape: "array", payload: item }));
|
||||
continue;
|
||||
}
|
||||
rows.push({ root, entityId: "__value__", shape: "scalar", payload: rootValue });
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
function inflateDomainRows(rows: Array<Record<string, unknown>>): Record<string, unknown> {
|
||||
const output: Record<string, unknown> = {};
|
||||
const grouped = new Map<string, Array<Record<string, unknown>>>();
|
||||
for (const row of rows) { const root = String(row.root_key).split(":").slice(1).join(":"); const list = grouped.get(root) || []; list.push(row); grouped.set(root, list); }
|
||||
for (const [root, values] of grouped) {
|
||||
const shape = values[0]?.shape;
|
||||
if (shape === "map") output[root] = values[0].entity_id === "__empty__" ? [] : values.filter((row) => row.entity_id !== "__empty__").map((row) => [row.entity_id, row.payload]);
|
||||
else if (shape === "array") output[root] = values[0].entity_id === "__empty__" ? [] : values.filter((row) => row.entity_id !== "__empty__").sort((a, b) => String(a.entity_id).localeCompare(String(b.entity_id))).map((row) => row.payload);
|
||||
else output[root] = values.find((row) => row.entity_id === "__value__")?.payload;
|
||||
}
|
||||
output.snapshotVersion = 1;
|
||||
return output;
|
||||
}
|
||||
|
||||
async function enrichCanonicalOutputMetadata(client: PgClient, snapshot: SnapshotRecord) {
|
||||
if (!snapshot.tasks) return snapshot;
|
||||
const result = await client.query("SELECT id, metadata FROM generation_outputs").catch(() => ({ rows: [] as Array<{ id: string; metadata?: unknown }> }));
|
||||
if (!result.rows.length) return snapshot;
|
||||
const metadataById = new Map(result.rows.map((row) => [String(row.id), parseJson(row.metadata, undefined)]));
|
||||
for (const [, raw] of mapEntries(snapshot.tasks)) {
|
||||
const task = record(raw); if (!Array.isArray(task.outputs)) continue;
|
||||
task.outputs = task.outputs.map((rawOutput) => { const output = record(rawOutput); const metadata = metadataById.get(String(output.id)); return metadata === undefined ? rawOutput : { ...output, metadata }; });
|
||||
}
|
||||
return snapshot;
|
||||
}
|
||||
|
||||
export function createPersistenceRepository<T>(options: { adapter?: "memory" | "file" | "postgres"; filePath?: string; databaseUrl?: string; key?: string } = {}): PersistenceRepository<T> {
|
||||
if (options.adapter === "postgres") return options.databaseUrl ? new PostgresRepository<T>(options.databaseUrl, options.key) : new UnavailableRepository<T>("postgres", "PostgreSQL adapter unavailable: DATABASE_URL is not configured");
|
||||
if (options.adapter === "file" && options.filePath) return new FileRepository<T>(options.filePath);
|
||||
return new MemoryRepository<T>();
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
import { mkdir, readdir, rm, stat, writeFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { randomUUID } from "node:crypto";
|
||||
|
||||
export type StagingHealth = { root: string; status: "ready" | "unavailable"; detail?: string };
|
||||
|
||||
export function safeStagingPath(root: string, objectKey: string) {
|
||||
const normalized = objectKey.replaceAll("\\", "/").replace(/^\/+/, "");
|
||||
if (!normalized || normalized.split("/").some((segment) => segment === ".." || segment.includes("\0"))) throw new Error("invalid staging object key");
|
||||
const resolvedRoot = path.resolve(root);
|
||||
const resolved = path.resolve(resolvedRoot, normalized);
|
||||
if (resolved !== resolvedRoot && !resolved.startsWith(`${resolvedRoot}${path.sep}`)) throw new Error("staging path escapes root");
|
||||
return resolved;
|
||||
}
|
||||
|
||||
export async function ensureStagingDir(root: string): Promise<StagingHealth> {
|
||||
try { await mkdir(root, { recursive: true, mode: 0o700 }); return { root: path.resolve(root), status: "ready" }; }
|
||||
catch (error) { return { root: path.resolve(root), status: "unavailable", detail: error instanceof Error ? error.message : String(error) }; }
|
||||
}
|
||||
|
||||
export async function putStagingObject(root: string, ownerId: string, data: Uint8Array, extension = "bin") {
|
||||
const key = `${ownerId}/${randomUUID()}.${extension.replace(/[^a-z0-9]/gi, "") || "bin"}`;
|
||||
const target = safeStagingPath(root, key);
|
||||
await mkdir(path.dirname(target), { recursive: true, mode: 0o700 });
|
||||
await writeFile(target, data, { mode: 0o600, flag: "wx" });
|
||||
return { key, path: target };
|
||||
}
|
||||
|
||||
export async function gcStaging(root: string, ttlMs: number, now = Date.now(), isLive?: (key: string) => boolean) {
|
||||
let removed = 0;
|
||||
const owners = await readdir(root, { withFileTypes: true }).catch(() => []);
|
||||
for (const owner of owners) {
|
||||
if (!owner.isDirectory()) continue;
|
||||
const files = await readdir(path.join(root, owner.name), { withFileTypes: true }).catch(() => []);
|
||||
for (const file of files) {
|
||||
if (!file.isFile()) continue;
|
||||
const key = `${owner.name}/${file.name}`;
|
||||
if (isLive?.(key)) continue;
|
||||
const target = safeStagingPath(root, key);
|
||||
const info = await stat(target).catch(() => undefined);
|
||||
if (info && now - info.mtimeMs > ttlMs) { await rm(target, { force: true }); removed += 1; }
|
||||
}
|
||||
}
|
||||
return removed;
|
||||
}
|
||||
@@ -0,0 +1,563 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { ErrorCodes, type ErrorCode, type GenerationAttempt, type GenerationTask, type TaskType } from "@miragenflow/contracts";
|
||||
import { appendEvent, releaseBalance, settleBalance, transactTask, type Store } from "../store.ts";
|
||||
import { invokeProvider, queryProvider } from "../adapters/provider.ts";
|
||||
import { putStagingObject } from "../infra/staging.ts";
|
||||
|
||||
const DEFAULT_LEASE_MS = 60_000;
|
||||
const workerId = `fixture-worker-${process.pid}-${randomUUID().slice(0, 8)}`;
|
||||
const reconciliationInFlight = new Set<string>();
|
||||
const activeTaskIds = new Set<string>();
|
||||
|
||||
function workerLeaseMs() {
|
||||
return Number(process.env.MIRAGENFLOW_WORKER_LEASE_MS) > 0 ? Number(process.env.MIRAGENFLOW_WORKER_LEASE_MS) : DEFAULT_LEASE_MS;
|
||||
}
|
||||
|
||||
async function taskForWorker(store: Store, taskId: string) {
|
||||
const local = store.tasks.get(taskId);
|
||||
if (local) return local;
|
||||
if (!store.repository.loadTaskById) return undefined;
|
||||
const durable = await store.repository.loadTaskById(taskId);
|
||||
if (durable) store.tasks.set(taskId, durable);
|
||||
return durable;
|
||||
}
|
||||
|
||||
function fixtureFailures() {
|
||||
return new Set((process.env.MIRAGENFLOW_FIXTURE_FAIL_CHANNELS || "").split(",").map((value) => value.trim()).filter(Boolean));
|
||||
}
|
||||
|
||||
function retentionExpiry(store: Store, task: { retentionExpiresAt?: string }) {
|
||||
return task.retentionExpiresAt || new Date(Date.now() + store.storagePolicy.retentionDays * 86_400_000).toISOString();
|
||||
}
|
||||
|
||||
function outputMetadataForTask(task: { taskType: TaskType; inputSnapshot?: { params: Record<string, unknown> } }, metadata?: Record<string, unknown>) {
|
||||
if (task.taskType !== "image") return metadata;
|
||||
const resolution = task.inputSnapshot?.params?.resolution;
|
||||
if (typeof resolution !== "string" || !resolution) return metadata;
|
||||
return { ...(metadata || {}), resolution };
|
||||
}
|
||||
|
||||
async function outputFor(store: Store, ownerId: string, taskId: string, taskType: TaskType, expiresAt: string) {
|
||||
const id = randomUUID();
|
||||
const image = `<svg xmlns="http://www.w3.org/2000/svg" width="1024" height="1024"><rect width="100%" height="100%" fill="#e9e4db"/><circle cx="512" cy="420" r="160" fill="#f4c27a"/><text x="50%" y="85%" text-anchor="middle" font-family="sans-serif" font-size="34" fill="#29251f">MiragenFlow fixture</text></svg>`;
|
||||
const text = "元境幻生任务已完成。";
|
||||
const data = taskType === "audio" ? Buffer.from("ID3\u0004\u0000\u0000\u0000\u0000\u0000\u0000", "binary").toString("base64") : Buffer.from(taskType === "text" || taskType === "reverse-prompt" ? text : image).toString("base64");
|
||||
const mimeType = taskType === "audio" ? "audio/mpeg" : taskType === "text" || taskType === "reverse-prompt" ? "text/plain" : "image/svg+xml";
|
||||
const staged = await putStagingObject(store.stagingDir, ownerId, Buffer.from(data, "base64"), mimeType.split("/")[1]?.replace("svg+xml", "svg") || "bin");
|
||||
store.objects.set(id, { id, ownerId, mimeType, stagingKey: staged.key, taskId, expiresAt });
|
||||
store.assets.set(id, { id, ownerId, type: taskType === "audio" ? "audio" : taskType === "text" || taskType === "reverse-prompt" ? "text" : "image", objectId: id, taskId, name: `${taskType}-${id.slice(0, 8)}`, tags: [], retentionExpiresAt: expiresAt, pinned: false });
|
||||
return { id, mimeType, url: `/api/v1/objects/${id}`, status: "succeeded" as const };
|
||||
}
|
||||
|
||||
async function settleOutputs(store: Store, task: NonNullable<Store["tasks"] extends Map<string, infer T> ? T : never>, attemptId?: string, reconciled = false) {
|
||||
try {
|
||||
return await transactTask(store, task.id, (atomicStore, currentTask) => {
|
||||
const requested = Math.max(1, currentTask.count || 1);
|
||||
// Never bill or expose more outputs than the request asked for. Providers
|
||||
// can accidentally return extra items, and those must not create an
|
||||
// accounting mismatch between the task total and per-output charges.
|
||||
currentTask.outputs = currentTask.outputs.slice(0, requested);
|
||||
const successful = currentTask.outputs.filter((output) => output.status === "succeeded").length;
|
||||
const baseUnit = Math.floor(currentTask.estimatedCost / requested);
|
||||
let remainder = currentTask.estimatedCost - baseUnit * requested;
|
||||
let chargedAmount = 0;
|
||||
currentTask.outputs = currentTask.outputs.map((output) => {
|
||||
if (output.status !== "succeeded") return { ...output, unitPriceSnapshot: 0, chargedAmount: 0 };
|
||||
const charge = baseUnit + (remainder > 0 ? 1 : 0);
|
||||
if (remainder > 0) remainder -= 1;
|
||||
chargedAmount += charge;
|
||||
return { ...output, unitPriceSnapshot: charge, chargedAmount: charge };
|
||||
});
|
||||
if (currentTask.reservedCost > 0) {
|
||||
const settled = settleBalance(atomicStore, currentTask.ownerId, currentTask.reservedCost, chargedAmount, currentTask.id);
|
||||
if (!settled && atomicStore.balances.get(currentTask.ownerId)?.reserved !== undefined) throw new Error("balance settlement failed");
|
||||
}
|
||||
currentTask.reservedCost = 0;
|
||||
const partial = successful > 0 && successful < requested;
|
||||
currentTask.status = partial ? "partial" : "succeeded";
|
||||
currentTask.leaseToken = undefined;
|
||||
currentTask.leaseExpiresAt = undefined;
|
||||
if (reconciled && attemptId) {
|
||||
const reconciliationAttempt = currentTask.attempts.find((attempt) => attempt.id === attemptId);
|
||||
if (reconciliationAttempt) reconciliationAttempt.reconciliationStatus = "confirmed_success";
|
||||
}
|
||||
appendEvent(atomicStore, currentTask.id, { taskId: currentTask.id, attemptId, type: partial ? "task.progress" : "task.succeeded", payload: { reconciled, chargedAmount, outputCount: currentTask.outputs.length, partial } });
|
||||
return { successful, chargedAmount, partial, failed: false };
|
||||
}, { reload: false });
|
||||
} catch {
|
||||
return { successful: 0, chargedAmount: 0, partial: false, failed: true };
|
||||
}
|
||||
}
|
||||
|
||||
function markSettlementFailure(store: Store, task: NonNullable<Store["tasks"] extends Map<string, infer T> ? T : never>, attemptId?: string) {
|
||||
const attempt = attemptId ? task.attempts.find((item) => item.id === attemptId) : undefined;
|
||||
if (attempt) {
|
||||
// Output persistence or balance settlement can fail after the provider has
|
||||
// already succeeded. Keep the attempt explicitly reconcilable so the
|
||||
// reservation cannot become a permanent, unowned hold.
|
||||
attempt.status = "unknown";
|
||||
attempt.reconciliationStatus = "pending";
|
||||
attempt.errorCode = "SETTLEMENT_PENDING_RECONCILIATION";
|
||||
attempt.finishedAt = attempt.finishedAt || new Date().toISOString();
|
||||
}
|
||||
task.status = "unknown";
|
||||
task.publicErrorCode = ErrorCodes.StorageUnavailable;
|
||||
task.leaseExpiresAt = undefined;
|
||||
task.leaseToken = undefined;
|
||||
appendEvent(store, task.id, { taskId: task.id, attemptId, type: "task.unknown", payload: { reconciliationRequired: true, reason: "balance_settlement_failed" } });
|
||||
}
|
||||
|
||||
/** Release a task reservation and finalize its task row in one transaction. */
|
||||
async function releaseTaskReservation(store: Store, taskId: string, amount: number, status: "failed" | "canceled", payload: Record<string, unknown>, attemptId?: string, publicErrorCode?: ErrorCode, reconciliationStatus: "confirmed_failure" | "manual_review" = "confirmed_failure") {
|
||||
try {
|
||||
return await transactTask(store, taskId, (atomicStore, task) => {
|
||||
const releaseAmount = Math.max(0, task.reservedCost);
|
||||
if (releaseAmount > 0 && !releaseBalance(atomicStore, task.ownerId, releaseAmount, task.id)) return false;
|
||||
task.reservedCost = 0;
|
||||
task.status = status;
|
||||
if (publicErrorCode) task.publicErrorCode = publicErrorCode;
|
||||
task.leaseExpiresAt = undefined;
|
||||
task.leaseToken = undefined;
|
||||
if (attemptId) {
|
||||
const reconciliationAttempt = task.attempts.find((attempt) => attempt.id === attemptId);
|
||||
if (reconciliationAttempt) reconciliationAttempt.reconciliationStatus = reconciliationStatus;
|
||||
}
|
||||
appendEvent(atomicStore, task.id, { taskId: task.id, attemptId, type: status === "failed" ? "task.failed" : "task.canceled", payload });
|
||||
return true;
|
||||
});
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export function scheduleTask(store: Store, taskId: string) {
|
||||
if (!store.queue.includes(taskId)) store.queue.push(taskId);
|
||||
const task = store.tasks.get(taskId);
|
||||
if (store.repository.adapter === "postgres" && store.repository.claimTaskDispatchOutbox) { void dispatchTaskOutbox(store); return; }
|
||||
void store.queueAdapter.enqueue(taskId, { maxAttempts: Math.max(1, task?.routeSnapshot?.channelIds.length || 4), priority: task?.planSnapshot?.queuePriority || 0 }).then(async () => {
|
||||
const outbox = store.taskDispatchOutbox.get(taskId);
|
||||
if (outbox) { outbox.status = "sent"; outbox.dispatchedAt = new Date().toISOString(); outbox.leaseOwner = undefined; outbox.leaseExpiresAt = undefined; store.persist(); }
|
||||
const lease = await store.queueAdapter.claim(workerId, workerLeaseMs());
|
||||
if (!lease) return;
|
||||
// A schedule call may race another worker that already has a queue lease.
|
||||
// Never execute or ack an unrelated task under this task's outbox context.
|
||||
if (lease.taskId !== taskId) {
|
||||
if (store.queueAdapter.nack) await store.queueAdapter.nack(lease.taskId, lease.leaseToken, { retry: true, maxAttempts: Math.max(1, store.tasks.get(lease.taskId)?.routeSnapshot?.channelIds.length || 4) });
|
||||
return;
|
||||
}
|
||||
try {
|
||||
await runTask(store, lease.taskId, lease.leaseToken);
|
||||
await store.persistAsync();
|
||||
await store.queueAdapter.ack(lease.taskId, lease.leaseToken);
|
||||
} catch (error) {
|
||||
console.error("task execution failed", { taskId: lease.taskId, error: error instanceof Error ? error.message : String(error) });
|
||||
if (store.queueAdapter.nack) await store.queueAdapter.nack(lease.taskId, lease.leaseToken, { retry: true, maxAttempts: Math.max(1, task?.routeSnapshot?.channelIds.length || 4) });
|
||||
else await store.queueAdapter.ack(lease.taskId, lease.leaseToken);
|
||||
}
|
||||
}).catch((error) => {
|
||||
const outbox = store.taskDispatchOutbox.get(taskId);
|
||||
if (outbox) { outbox.status = "failed"; outbox.attempts += 1; outbox.lastError = error instanceof Error ? error.message : String(error); store.persist(); }
|
||||
console.error("task queue execution failed", error);
|
||||
});
|
||||
}
|
||||
|
||||
/** Claims durable task-dispatch rows with PostgreSQL SKIP LOCKED, enqueues the
|
||||
* task by its unique Redis member. Queue execution is deliberately separate
|
||||
* from the outbox row so a global queue claim cannot mark another task sent.
|
||||
* A crashed process leaves the leased row recoverable on the next scan. */
|
||||
export async function dispatchTaskOutbox(store: Store) {
|
||||
const claim = store.repository.claimTaskDispatchOutbox;
|
||||
const complete = store.repository.completeTaskDispatchOutbox;
|
||||
if (!claim || !complete) return 0;
|
||||
const worker = `dispatch-${process.pid}-${randomUUID().slice(0, 8)}`;
|
||||
const taskIds = await claim.call(store.repository, worker, workerLeaseMs());
|
||||
let processed = 0;
|
||||
for (const taskId of taskIds) {
|
||||
let task;
|
||||
try { task = await taskForWorker(store, taskId); } catch (error) {
|
||||
await complete.call(store.repository, taskId, worker, false, error instanceof Error ? error.message : String(error));
|
||||
continue;
|
||||
}
|
||||
if (!task) {
|
||||
// A PostgreSQL worker can claim an outbox row created by another
|
||||
// instance before this process has rehydrated its local projection.
|
||||
// Never acknowledge that row without publishing the task; leave it
|
||||
// retryable until the task snapshot is available.
|
||||
await complete.call(store.repository, taskId, worker, false, "任务尚未加载，稍后重试");
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
// The outbox dispatcher only publishes a uniquely identified task. It
|
||||
// must not claim an arbitrary queue member here: queue execution is a
|
||||
// separate worker concern and cannot be paired with this outbox row.
|
||||
await store.queueAdapter.enqueue(taskId, { maxAttempts: Math.max(1, task.routeSnapshot?.channelIds.length || 4), priority: task.planSnapshot?.queuePriority || 0 });
|
||||
const completed = await complete.call(store.repository, taskId, worker, true);
|
||||
if (!completed) continue;
|
||||
const outbox = store.taskDispatchOutbox.get(taskId);
|
||||
if (outbox) { outbox.status = "sent"; outbox.dispatchedAt = new Date().toISOString(); outbox.leaseOwner = undefined; outbox.leaseExpiresAt = undefined; }
|
||||
processed += 1;
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
const outbox = store.taskDispatchOutbox.get(taskId);
|
||||
if (outbox) { outbox.status = "pending"; outbox.lastError = message; }
|
||||
await complete.call(store.repository, taskId, worker, false, message);
|
||||
}
|
||||
}
|
||||
// Claim as many independent queue leases as the configured concurrency
|
||||
// allows. Each lease is fenced by its own task id; it is never paired with
|
||||
// an unrelated dispatch-outbox row.
|
||||
const capacity = Math.max(1, Number(store.systemSettings?.queueMaxConcurrent) || 1);
|
||||
const leases = [];
|
||||
for (let index = activeTaskIds.size; index < capacity; index += 1) {
|
||||
const lease = await store.queueAdapter.claim(worker, workerLeaseMs());
|
||||
if (!lease) break;
|
||||
leases.push(lease);
|
||||
}
|
||||
await Promise.allSettled(leases.map(async (lease) => {
|
||||
if (!(await taskForWorker(store, lease.taskId))) {
|
||||
if (store.queueAdapter.nack) await store.queueAdapter.nack(lease.taskId, lease.leaseToken, { retry: true });
|
||||
return;
|
||||
}
|
||||
try {
|
||||
await runTask(store, lease.taskId, lease.leaseToken);
|
||||
await store.queueAdapter.ack(lease.taskId, lease.leaseToken);
|
||||
} catch {
|
||||
if (store.queueAdapter.nack) await store.queueAdapter.nack(lease.taskId, lease.leaseToken, { retry: true, maxAttempts: Math.max(1, store.tasks.get(lease.taskId)?.routeSnapshot?.channelIds.length || 4) });
|
||||
else await store.queueAdapter.ack(lease.taskId, lease.leaseToken);
|
||||
}
|
||||
}));
|
||||
return processed;
|
||||
}
|
||||
|
||||
/** Rehydrate durable queued work after a process restart. */
|
||||
export async function resumeQueuedTasks(store: Store) {
|
||||
if (store.repository.adapter === "postgres" && store.repository.claimTaskDispatchOutbox) { await dispatchTaskOutbox(store); return; }
|
||||
const queued = [...store.tasks.values()].filter((task) => task.status === "queued").map((task) => task.id);
|
||||
for (const taskId of queued) {
|
||||
if (!store.queue.includes(taskId)) store.queue.push(taskId);
|
||||
try {
|
||||
await store.queueAdapter.enqueue(taskId, { maxAttempts: Math.max(1, store.tasks.get(taskId)?.routeSnapshot?.channelIds.length || 4), priority: store.tasks.get(taskId)?.planSnapshot?.queuePriority || 0 });
|
||||
const outbox = store.taskDispatchOutbox.get(taskId);
|
||||
if (outbox) { outbox.status = "sent"; outbox.dispatchedAt = new Date().toISOString(); store.persist(); }
|
||||
const lease = await store.queueAdapter.claim(workerId, workerLeaseMs());
|
||||
if (!lease) continue;
|
||||
if (lease.taskId !== taskId) {
|
||||
if (store.queueAdapter.nack) await store.queueAdapter.nack(lease.taskId, lease.leaseToken, { retry: true, maxAttempts: Math.max(1, store.tasks.get(lease.taskId)?.routeSnapshot?.channelIds.length || 4) });
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
await runTask(store, lease.taskId, lease.leaseToken);
|
||||
await store.persistAsync();
|
||||
await store.queueAdapter.ack(lease.taskId, lease.leaseToken);
|
||||
} catch (error) {
|
||||
console.error("queued task execution failed", { taskId: lease.taskId, error: error instanceof Error ? error.message : String(error) });
|
||||
if (store.queueAdapter.nack) await store.queueAdapter.nack(lease.taskId, lease.leaseToken, { retry: true, maxAttempts: Math.max(1, store.tasks.get(taskId)?.routeSnapshot?.channelIds.length || 4) });
|
||||
else await store.queueAdapter.ack(lease.taskId, lease.leaseToken);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("queued task recovery failed", { taskId, error: error instanceof Error ? error.message : String(error) });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function recoverExpiredTaskLeases(store: Store, now = Date.now()) {
|
||||
let recovered = 0;
|
||||
for (const task of store.tasks.values()) {
|
||||
if (task.status !== "running" || !task.leaseExpiresAt || Date.parse(task.leaseExpiresAt) > now) continue;
|
||||
const attempt = [...task.attempts].reverse().find((item) => item.status === "started");
|
||||
if (attempt) {
|
||||
attempt.status = "unknown";
|
||||
attempt.errorCode = "WORKER_LEASE_EXPIRED";
|
||||
attempt.finishedAt = new Date(now).toISOString();
|
||||
attempt.reconciliationStatus = "pending";
|
||||
attempt.leaseExpiresAt = new Date(now).toISOString();
|
||||
}
|
||||
task.status = "unknown";
|
||||
task.publicErrorCode = ErrorCodes.ModelGroupUnavailable;
|
||||
task.leaseExpiresAt = undefined;
|
||||
task.leaseToken = undefined;
|
||||
appendEvent(store, task.id, { taskId: task.id, attemptId: attempt?.id, type: "task.unknown", payload: { reconciliationRequired: true, reason: "worker_lease_expired", workerId } });
|
||||
recovered += 1;
|
||||
}
|
||||
return recovered;
|
||||
}
|
||||
|
||||
/** PostgreSQL-aware recovery. Each candidate is reloaded and fenced inside
|
||||
* a transaction so a stale process cannot turn a task unknown after another
|
||||
* worker has renewed or completed it. The synchronous helper above remains
|
||||
* the deterministic path for memory/file fixtures. */
|
||||
export async function recoverExpiredTaskLeasesAsync(store: Store, now = Date.now()) {
|
||||
if (store.repository.adapter !== "postgres") return recoverExpiredTaskLeases(store, now);
|
||||
const candidates = [...store.tasks.values()]
|
||||
.filter((task) => task.status === "running" && task.leaseExpiresAt && Date.parse(task.leaseExpiresAt) <= now)
|
||||
.map((task) => task.id);
|
||||
let recovered = 0;
|
||||
for (const taskId of candidates) {
|
||||
try {
|
||||
const changed = await transactTask(store, taskId, (atomicStore, task) => {
|
||||
if (task.status !== "running" || !task.leaseExpiresAt || Date.parse(task.leaseExpiresAt) > now) return false;
|
||||
const attempt = [...task.attempts].reverse().find((item) => item.status === "started");
|
||||
if (attempt) {
|
||||
attempt.status = "unknown";
|
||||
attempt.errorCode = "WORKER_LEASE_EXPIRED";
|
||||
attempt.finishedAt = new Date(now).toISOString();
|
||||
attempt.reconciliationStatus = "pending";
|
||||
attempt.leaseExpiresAt = new Date(now).toISOString();
|
||||
}
|
||||
task.status = "unknown";
|
||||
task.publicErrorCode = ErrorCodes.ModelGroupUnavailable;
|
||||
task.leaseExpiresAt = undefined;
|
||||
task.leaseToken = undefined;
|
||||
appendEvent(atomicStore, task.id, { taskId: task.id, attemptId: attempt?.id, type: "task.unknown", payload: { reconciliationRequired: true, reason: "worker_lease_expired", workerId } });
|
||||
return true;
|
||||
}, { reload: true }, 2);
|
||||
if (changed) recovered += 1;
|
||||
} catch { /* another worker won the row; it will remain authoritative */ }
|
||||
}
|
||||
return recovered;
|
||||
}
|
||||
|
||||
export function recoverExpiredReservations(store: Store, now = Date.now()) {
|
||||
let recovered = 0;
|
||||
for (const task of store.tasks.values()) {
|
||||
if (!task.reserveExpiresAt || Date.parse(task.reserveExpiresAt) > now || task.reservedCost <= 0 || ["succeeded", "partial", "failed", "canceled"].includes(task.status)) continue;
|
||||
const unknownAttempt = task.status === "unknown" ? [...task.attempts].reverse().find((item) => item.status === "unknown") : undefined;
|
||||
// Unknown results are deliberately held for explicit provider or
|
||||
// administrator reconciliation. Releasing them on a timer could refund a
|
||||
// request that the provider accepted and later charges.
|
||||
if (task.status === "unknown") continue;
|
||||
if (!releaseBalance(store, task.ownerId, task.reservedCost, task.id)) continue;
|
||||
task.reservedCost = 0;
|
||||
task.status = "failed";
|
||||
task.publicErrorCode = ErrorCodes.ModelGroupUnavailable;
|
||||
task.reserveExpiresAt = undefined;
|
||||
task.leaseExpiresAt = undefined;
|
||||
task.leaseToken = undefined;
|
||||
if (unknownAttempt) unknownAttempt.reconciliationStatus = "manual_review";
|
||||
appendEvent(store, task.id, { taskId: task.id, attemptId: unknownAttempt?.id, type: "task.failed", payload: { code: unknownAttempt ? "UNKNOWN_RECONCILIATION_EXPIRED" : "RESERVE_EXPIRED", reconciliationRequired: false } });
|
||||
recovered += 1;
|
||||
}
|
||||
return recovered;
|
||||
}
|
||||
|
||||
/** Production reconciler variant: finalize each expired reservation through the
|
||||
* task transaction so the release ledger and task terminal state commit
|
||||
* together. The synchronous helper above remains for the in-memory fixture
|
||||
* compatibility tests and callers that do not await a worker tick. */
|
||||
export async function recoverExpiredReservationsAsync(store: Store, now = Date.now()) {
|
||||
let recovered = 0;
|
||||
const candidates = [...store.tasks.values()].filter((task) => {
|
||||
if (!task.reserveExpiresAt || Date.parse(task.reserveExpiresAt) > now || task.reservedCost <= 0 || ["succeeded", "partial", "failed", "canceled"].includes(task.status)) return false;
|
||||
const attempt = task.status === "unknown" ? [...task.attempts].reverse().find((item) => item.status === "unknown") : undefined;
|
||||
return task.status !== "unknown";
|
||||
});
|
||||
for (const task of candidates) {
|
||||
const unknownAttempt = task.status === "unknown" ? [...task.attempts].reverse().find((item) => item.status === "unknown") : undefined;
|
||||
const released = await releaseTaskReservation(store, task.id, task.reservedCost, "failed", { code: unknownAttempt ? "UNKNOWN_RECONCILIATION_EXPIRED" : "RESERVE_EXPIRED", reconciliationRequired: false }, unknownAttempt?.id, ErrorCodes.ModelGroupUnavailable, unknownAttempt ? "manual_review" : "confirmed_failure");
|
||||
if (!released) continue;
|
||||
recovered += 1;
|
||||
}
|
||||
return recovered;
|
||||
}
|
||||
|
||||
/** Resolve provider-accepted requests without allowing a late response to
|
||||
* overwrite a task that has already been settled or canceled. */
|
||||
export async function reconcileUnknownTasks(store: Store) {
|
||||
let resolved = 0;
|
||||
for (const task of store.tasks.values()) {
|
||||
if (task.status !== "unknown" || reconciliationInFlight.has(task.id)) continue;
|
||||
const attempt = [...task.attempts].reverse().find((item) => item.status === "unknown" && item.providerRequestId);
|
||||
if (!attempt?.providerRequestId) continue;
|
||||
const channel = store.channels.find((item) => item.id === attempt.channelId);
|
||||
if (!channel) continue;
|
||||
reconciliationInFlight.add(task.id);
|
||||
try {
|
||||
const result = await queryProvider(store, channel, attempt.providerRequestId);
|
||||
if (result.status === "unknown") continue;
|
||||
if (result.status === "failed") {
|
||||
if (!(await releaseTaskReservation(store, task.id, task.reservedCost, "failed", { reconciled: true }, attempt.id, ErrorCodes.ModelGroupUnavailable))) continue;
|
||||
} else {
|
||||
attempt.reconciliationStatus = "confirmed_success";
|
||||
if (!task.outputs.length) {
|
||||
task.outputs = [];
|
||||
for (const output of result.outputs || []) {
|
||||
const outputId = randomUUID(); const staged = await putStagingObject(store.stagingDir, task.ownerId, Buffer.from(output.data, "base64"), output.mimeType.split("/")[1] || "bin");
|
||||
store.objects.set(outputId, { id: outputId, ownerId: task.ownerId, mimeType: output.mimeType, stagingKey: staged.key, taskId: task.id, expiresAt: retentionExpiry(store, task) });
|
||||
store.assets.set(outputId, { id: outputId, ownerId: task.ownerId, type: task.taskType === "audio" ? "audio" : task.taskType === "text" || task.taskType === "reverse-prompt" ? "text" : "image", objectId: outputId, taskId: task.id, name: `${task.taskType}-${outputId.slice(0, 8)}`, tags: [], retentionExpiresAt: retentionExpiry(store, task), pinned: false });
|
||||
task.outputs.push({ id: outputId, mimeType: output.mimeType, url: `/api/v1/objects/${outputId}`, status: "succeeded", metadata: outputMetadataForTask(task, output.metadata) as GenerationTask["outputs"][number]["metadata"] });
|
||||
}
|
||||
}
|
||||
if (!task.outputs.length) continue;
|
||||
const settlement = await settleOutputs(store, task, attempt.id, true);
|
||||
if (settlement.failed) continue;
|
||||
}
|
||||
resolved += 1;
|
||||
} finally { reconciliationInFlight.delete(task.id); }
|
||||
}
|
||||
return resolved;
|
||||
}
|
||||
|
||||
/** Executes a claimed task. The leaseLost flag fences all side effects after a failed renewal. */
|
||||
async function runTaskWithLease(store: Store, taskId: string, leaseToken: string, leaseLost: { value: boolean }) {
|
||||
const leaseMs = workerLeaseMs();
|
||||
const started = await store.transact((atomicStore) => {
|
||||
const current = atomicStore.tasks.get(taskId);
|
||||
if (!current || current.status !== "queued") return false;
|
||||
const owner = atomicStore.users.get(current.ownerId);
|
||||
if (!owner || owner.status !== "active") {
|
||||
if (current.reservedCost > 0) releaseBalance(atomicStore, current.ownerId, current.reservedCost, current.id);
|
||||
current.reservedCost = 0;
|
||||
current.status = "canceled";
|
||||
current.publicErrorCode = ErrorCodes.Forbidden;
|
||||
current.reserveExpiresAt = undefined;
|
||||
appendEvent(atomicStore, current.id, { taskId: current.id, type: "task.canceled", payload: { reason: owner?.status === "frozen" ? "USER_FROZEN" : "USER_UNAVAILABLE" } });
|
||||
return false;
|
||||
}
|
||||
atomicStore.queue = atomicStore.queue.filter((id) => id !== taskId);
|
||||
current.status = "running";
|
||||
current.leaseToken = leaseToken;
|
||||
current.leaseExpiresAt = new Date(Date.now() + leaseMs).toISOString();
|
||||
appendEvent(atomicStore, current.id, { taskId: current.id, type: "task.running", payload: { channelGroupId: current.channelGroupId, workerId, leaseExpiresAt: current.leaseExpiresAt } });
|
||||
return true;
|
||||
});
|
||||
if (!started) return;
|
||||
const task = store.tasks.get(taskId);
|
||||
if (!task) return;
|
||||
const group = store.channelGroups.find((item) => item.id === task.channelGroupId);
|
||||
const routeIds = task.routeSnapshot?.channelIds || group?.channelIds || [];
|
||||
const now = Date.now();
|
||||
const channels = routeIds.map((id) => store.channels.find((channel) => channel.id === id)).filter((channel): channel is NonNullable<typeof channel> => {
|
||||
if (!channel?.enabled) return false;
|
||||
if (channel.health !== "open") return true;
|
||||
if (channel.circuitResetAt && channel.circuitResetAt <= now) { channel.health = "degraded"; channel.circuitOpenedAt = undefined; channel.circuitResetAt = undefined; return true; }
|
||||
return false;
|
||||
});
|
||||
const attemptPlan: typeof channels = [];
|
||||
let groupRetries = Math.max(0, task.routeSnapshot?.retryBudget ?? group?.retryBudget ?? 0);
|
||||
for (const channel of channels) {
|
||||
attemptPlan.push(channel);
|
||||
const retries = Math.min(Math.max(0, task.routeSnapshot?.channelRetryBudgets?.[channel.id] ?? channel.retryBudget), groupRetries);
|
||||
for (let retry = 0; retry < retries; retry += 1) attemptPlan.push(channel);
|
||||
groupRetries -= retries;
|
||||
}
|
||||
const maxAttempts = attemptPlan.length;
|
||||
const ownsLease = () => {
|
||||
const current = store.tasks.get(taskId);
|
||||
return !leaseLost.value && current?.status === "running" && current.leaseToken === leaseToken && current.leaseExpiresAt && Date.parse(current.leaseExpiresAt) > Date.now();
|
||||
};
|
||||
const failures = fixtureFailures();
|
||||
for (let index = 0; index < maxAttempts; index += 1) {
|
||||
if (!ownsLease()) return;
|
||||
const channel = attemptPlan[index];
|
||||
if (channel.health === "open" && (!channel.circuitResetAt || channel.circuitResetAt > Date.now())) continue;
|
||||
const attemptStartedAt = new Date().toISOString();
|
||||
if (index > 0 && attemptPlan[index - 1]?.id === channel.id) await new Promise((resolve) => setTimeout(resolve, Math.min(2_000, 100 * 2 ** Math.min(4, index - 1)) + Math.floor(Math.random() * 50)));
|
||||
const attempt: GenerationAttempt = { id: randomUUID(), channelId: channel.id, sequence: index + 1, status: "started", platformIdempotencyKey: `${task.id}:${index + 1}`, startedAt: attemptStartedAt, leaseExpiresAt: new Date(Date.now() + leaseMs).toISOString() };
|
||||
task.attempts.push(attempt);
|
||||
appendEvent(store, task.id, { taskId: task.id, attemptId: attempt.id, type: "task.attempt", payload: { channelId: channel.id, sequence: attempt.sequence, status: attempt.status } });
|
||||
await new Promise((resolve) => setTimeout(resolve, 40));
|
||||
if (!ownsLease()) return;
|
||||
if (channel.baseUrl) {
|
||||
task.providerSubmitted = true;
|
||||
const provider = await invokeProvider(store, channel, { taskId: task.id, platformIdempotencyKey: attempt.platformIdempotencyKey, taskType: task.taskType, publicModelId: task.modelSnapshot?.publicModelId || task.modelProductId, prompt: task.prompt, count: task.count, references: task.inputSnapshot?.references, referenceImages: task.inputSnapshot?.referenceGroup?.images, maskObjectId: task.inputSnapshot?.maskObjectId, params: task.inputSnapshot?.params });
|
||||
if (!ownsLease()) return;
|
||||
if (provider.status === "unknown") {
|
||||
attempt.status = "unknown"; attempt.providerRequestId = provider.providerRequestId; attempt.errorCode = provider.errorCode; attempt.finishedAt = new Date().toISOString(); attempt.reconciliationStatus = "pending"; task.status = "unknown"; task.publicErrorCode = ErrorCodes.ModelGroupUnavailable; task.leaseExpiresAt = undefined; task.leaseToken = undefined; appendEvent(store, task.id, { taskId: task.id, attemptId: attempt.id, type: "task.unknown", payload: { channelId: channel.id, reconciliationRequired: true, providerRequestIdAvailable: Boolean(provider.providerRequestId) } }); return;
|
||||
}
|
||||
if (store.tasks.get(taskId)?.status === "canceled") {
|
||||
attempt.status = "failed"; attempt.errorCode = "CANCELED_AFTER_PROVIDER_SUBMIT"; attempt.finishedAt = new Date().toISOString(); attempt.reconciliationStatus = "confirmed_failure"; task.leaseExpiresAt = undefined; task.leaseToken = undefined; return;
|
||||
}
|
||||
if (provider.status !== "succeeded") {
|
||||
attempt.status = "failed"; attempt.errorCode = provider.errorCode; attempt.retryable = provider.retryable === true; attempt.finishedAt = new Date().toISOString(); channel.consecutiveFailures += 1; if (channel.consecutiveFailures >= 3) { channel.health = "open"; channel.circuitOpenedAt = Date.now(); channel.circuitResetAt = Date.now() + 30_000; } else channel.health = provider.retryable || ["PROVIDER_AUTH", "PROVIDER_SECRET_UNAVAILABLE"].includes(provider.errorCode || "") ? "degraded" : channel.health; appendEvent(store, task.id, { taskId: task.id, attemptId: attempt.id, type: "task.progress", payload: { channelId: channel.id, status: attempt.status, errorCode: attempt.errorCode, nextChannel: attemptPlan[index + 1]?.id } }); if (provider.retryable !== true && !["PROVIDER_AUTH", "PROVIDER_SECRET_UNAVAILABLE"].includes(provider.errorCode || "")) break; continue;
|
||||
}
|
||||
attempt.status = "succeeded"; channel.consecutiveFailures = 0; channel.health = "healthy"; attempt.providerRequestId = provider.providerRequestId; attempt.finishedAt = new Date().toISOString(); task.outputs = []; for (const providerOutput of (provider.outputs || []).slice(0, Math.max(1, task.count || 1))) { const outputId = randomUUID(); const staged = await putStagingObject(store.stagingDir, task.ownerId, Buffer.from(providerOutput.data, "base64"), providerOutput.mimeType.split("/")[1] || "bin"); if (!ownsLease()) return; const expiresAt = retentionExpiry(store, task); store.objects.set(outputId, { id: outputId, ownerId: task.ownerId, mimeType: providerOutput.mimeType, stagingKey: staged.key, taskId: task.id, expiresAt }); store.assets.set(outputId, { id: outputId, ownerId: task.ownerId, type: task.taskType === "audio" ? "audio" : task.taskType === "text" || task.taskType === "reverse-prompt" ? "text" : "image", objectId: outputId, taskId: task.id, name: `${task.taskType}-${outputId.slice(0, 8)}`, tags: [], retentionExpiresAt: expiresAt, pinned: false }); task.outputs.push({ id: outputId, mimeType: providerOutput.mimeType, url: `/api/v1/objects/${outputId}`, status: "succeeded", metadata: outputMetadataForTask(task, providerOutput.metadata) as GenerationTask["outputs"][number]["metadata"] }); } if (!task.outputs.length) { attempt.status = "failed"; attempt.errorCode = "PROVIDER_EMPTY_RESULT"; const released = await releaseTaskReservation(store, task.id, task.reservedCost, "failed", { code: "PROVIDER_EMPTY_RESULT" }, attempt.id); if (!released) markSettlementFailure(store, task, attempt.id); return; } const settlement = await settleOutputs(store, task, attempt.id); if (settlement.failed) { markSettlementFailure(store, task, attempt.id); return; } return;
|
||||
}
|
||||
if (process.env.NODE_ENV !== "test" && process.env.MIRAGENFLOW_TEST_MODE !== "true") {
|
||||
attempt.status = "failed";
|
||||
attempt.errorCode = "CHANNEL_NOT_CONFIGURED";
|
||||
attempt.finishedAt = new Date().toISOString();
|
||||
appendEvent(store, task.id, { taskId: task.id, attemptId: attempt.id, type: "task.progress", payload: { channelId: channel.id, status: attempt.status, errorCode: attempt.errorCode, nextChannel: attemptPlan[index + 1]?.id } });
|
||||
continue;
|
||||
}
|
||||
if (failures.has(channel.id) || channel.fixtureFailure === "retryable") {
|
||||
attempt.status = "failed";
|
||||
attempt.errorCode = "CHANNEL_UNAVAILABLE";
|
||||
attempt.finishedAt = new Date().toISOString();
|
||||
channel.consecutiveFailures += 1;
|
||||
if (channel.consecutiveFailures >= 3) { channel.health = "open"; channel.circuitOpenedAt = Date.now(); channel.circuitResetAt = Date.now() + 30_000; } else channel.health = "degraded";
|
||||
appendEvent(store, task.id, { taskId: task.id, attemptId: attempt.id, type: "task.progress", payload: { channelId: channel.id, status: attempt.status, errorCode: attempt.errorCode, nextChannel: attemptPlan[index + 1]?.id } });
|
||||
continue;
|
||||
}
|
||||
if (channel.fixtureFailure === "unknown") {
|
||||
attempt.status = "unknown";
|
||||
attempt.errorCode = "PROVIDER_TIMEOUT_UNKNOWN";
|
||||
attempt.finishedAt = new Date().toISOString();
|
||||
attempt.reconciliationStatus = "pending";
|
||||
task.status = "unknown";
|
||||
task.publicErrorCode = ErrorCodes.ModelGroupUnavailable;
|
||||
task.leaseExpiresAt = undefined;
|
||||
task.leaseToken = undefined;
|
||||
appendEvent(store, task.id, { taskId: task.id, attemptId: attempt.id, type: "task.unknown", payload: { channelId: channel.id, reconciliationRequired: true } });
|
||||
return;
|
||||
}
|
||||
attempt.status = "succeeded";
|
||||
channel.consecutiveFailures = 0;
|
||||
channel.health = "healthy";
|
||||
channel.circuitOpenedAt = undefined;
|
||||
channel.circuitResetAt = undefined;
|
||||
attempt.providerRequestId = `fixture-${randomUUID()}`;
|
||||
attempt.finishedAt = new Date().toISOString();
|
||||
attempt.leaseExpiresAt = undefined;
|
||||
task.outputs = await Promise.all(Array.from({ length: Math.max(1, task.count || 1) }, () => outputFor(store, task.ownerId, task.id, task.taskType, retentionExpiry(store, task))));
|
||||
if (!ownsLease()) return;
|
||||
const settlement = await settleOutputs(store, task, attempt.id);
|
||||
if (settlement.failed) { markSettlementFailure(store, task, attempt.id); return; }
|
||||
return;
|
||||
}
|
||||
const reservedCost = task.reservedCost;
|
||||
const released = await releaseTaskReservation(store, task.id, reservedCost, "failed", { code: ErrorCodes.ModelGroupUnavailable, suggestion: "请切换到其他同档位模型", reconciliationRequired: false }, undefined, ErrorCodes.ModelGroupUnavailable);
|
||||
if (!released) markSettlementFailure(store, task);
|
||||
}
|
||||
|
||||
/** Runs a task with a periodic queue heartbeat. Timer cleanup is guaranteed for every return path. */
|
||||
export async function runTask(store: Store, taskId: string, leaseToken: string = randomUUID()) {
|
||||
const limit = Math.max(1, Number(store.systemSettings?.queueMaxConcurrent) || 1);
|
||||
if (!activeTaskIds.has(taskId) && activeTaskIds.size >= limit) throw new Error("QUEUE_CONCURRENCY_LIMIT");
|
||||
activeTaskIds.add(taskId);
|
||||
const leaseMs = workerLeaseMs();
|
||||
const leaseLost = { value: false };
|
||||
const queue = store.queueAdapter;
|
||||
// Direct callers historically supplied a random token without a queue lease; retain that API.
|
||||
const renew = queue.renew || queue.heartbeat;
|
||||
const durableRenew = store.repository.adapter === "postgres" ? store.repository.renewTaskLease : undefined;
|
||||
const shouldHeartbeat = Boolean((renew || durableRenew) && leaseToken.includes(":"));
|
||||
let timer: ReturnType<typeof setInterval> | undefined;
|
||||
let active = true;
|
||||
if (shouldHeartbeat) {
|
||||
timer = setInterval(() => {
|
||||
if (!active) return;
|
||||
void (async () => {
|
||||
// Renew the queue lease first. A durable task lease must never be
|
||||
// extended after the queue owner has already been fenced out.
|
||||
if (renew && !(await renew(taskId, leaseToken, leaseMs))) {
|
||||
leaseLost.value = true;
|
||||
return;
|
||||
}
|
||||
const durable = durableRenew ? await durableRenew(taskId, leaseToken, leaseMs) : { renewed: true as const };
|
||||
if (!durable.renewed) {
|
||||
leaseLost.value = true;
|
||||
return;
|
||||
}
|
||||
const task = store.tasks.get(taskId);
|
||||
if (task?.status === "running" && task.leaseToken === leaseToken) {
|
||||
task.leaseExpiresAt = durable.leaseExpiresAt || new Date(Date.now() + leaseMs).toISOString();
|
||||
const attempt = task.attempts.find((item) => item.status === "started");
|
||||
if (attempt) attempt.leaseExpiresAt = task.leaseExpiresAt;
|
||||
}
|
||||
})().catch(() => { leaseLost.value = true; });
|
||||
}, Math.max(10, Math.floor(leaseMs / 3)));
|
||||
timer.unref?.();
|
||||
}
|
||||
try { return await runTaskWithLease(store, taskId, leaseToken, leaseLost); }
|
||||
finally { active = false; if (timer) clearInterval(timer); activeTaskIds.delete(taskId); }
|
||||
}
|
||||
@@ -0,0 +1,182 @@
|
||||
import { createCipheriv, createDecipheriv, createHash, createHmac, randomUUID, scryptSync, timingSafeEqual, randomBytes } from "node:crypto";
|
||||
import type { IncomingHttpHeaders } from "node:http";
|
||||
import type { AuthUser } from "@miragenflow/contracts";
|
||||
import { DomainError } from "./errors.ts";
|
||||
import { ErrorCodes } from "@miragenflow/contracts";
|
||||
|
||||
type TokenPayload = { sub: string; scope: "user" | "admin"; exp: number; iat: number; jti: string; aud: "miragenflow-user" | "miragenflow-admin"; roles?: string[]; scopes?: string[]; sid?: string };
|
||||
const revokedAccessTokens = new Set<string>();
|
||||
|
||||
function encode(value: unknown) {
|
||||
return Buffer.from(JSON.stringify(value)).toString("base64url");
|
||||
}
|
||||
|
||||
function decode<T>(value: string) {
|
||||
try { return JSON.parse(Buffer.from(value, "base64url").toString("utf8")) as T; } catch { throw new DomainError(ErrorCodes.Unauthorized, "登录状态无效", undefined, 401); }
|
||||
}
|
||||
|
||||
export function issueAccessToken(userId: string, scope: TokenPayload["scope"], secret: string, ttlSeconds: number, claims: Pick<TokenPayload, "roles" | "scopes" | "sid"> = {}) {
|
||||
const payload: TokenPayload = { sub: userId, scope, aud: scope === "admin" ? "miragenflow-admin" : "miragenflow-user", jti: randomUUID(), iat: Math.floor(Date.now() / 1000), exp: Math.floor(Date.now() / 1000) + ttlSeconds, ...claims };
|
||||
const body = `${encode({ alg: "HS256", typ: "JWT" })}.${encode(payload)}`;
|
||||
const signature = createHmac("sha256", secret).update(body).digest("base64url");
|
||||
return `${body}.${signature}`;
|
||||
}
|
||||
|
||||
export function verifyAccessToken(token: string, secret: string, expectedScope: TokenPayload["scope"], revokedBefore?: number): TokenPayload {
|
||||
const [header, body, signature] = token.split(".");
|
||||
if (!header || !body || !signature) throw new DomainError(ErrorCodes.Unauthorized, "登录状态无效", undefined, 401);
|
||||
const expected = createHmac("sha256", secret).update(`${header}.${body}`).digest("base64url");
|
||||
const valid = Buffer.byteLength(signature) === Buffer.byteLength(expected) && timingSafeEqual(Buffer.from(signature), Buffer.from(expected));
|
||||
const joseHeader = decode<{ alg?: string; typ?: string }>(header);
|
||||
const payload = decode<TokenPayload>(body);
|
||||
const expectedAudience = expectedScope === "admin" ? "miragenflow-admin" : "miragenflow-user";
|
||||
if (!valid || joseHeader.alg !== "HS256" || joseHeader.typ !== "JWT" || revokedAccessTokens.has(createHash("sha256").update(token).digest("hex")) || (revokedBefore !== undefined && payload.iat <= revokedBefore) || payload.aud !== expectedAudience || payload.scope !== expectedScope || !payload.sub || !Number.isInteger(payload.iat) || !Number.isInteger(payload.exp) || payload.exp <= Math.floor(Date.now() / 1000)) {
|
||||
throw new DomainError(ErrorCodes.Unauthorized, "登录状态已失效", undefined, 401);
|
||||
}
|
||||
return payload;
|
||||
}
|
||||
|
||||
export function revokeAccessToken(token: string | undefined) {
|
||||
if (token) revokedAccessTokens.add(createHash("sha256").update(token).digest("hex"));
|
||||
}
|
||||
|
||||
export function bearer(headers: IncomingHttpHeaders) {
|
||||
const value = headers.authorization;
|
||||
return typeof value === "string" && value.startsWith("Bearer ") ? value.slice(7).trim() : undefined;
|
||||
}
|
||||
|
||||
export function requireUser(headers: IncomingHttpHeaders, secret: string, revokedBefore?: number) {
|
||||
const token = bearer(headers);
|
||||
if (!token) throw new DomainError(ErrorCodes.Unauthorized, "请先登录", undefined, 401);
|
||||
return verifyAccessToken(token, secret, "user", revokedBefore);
|
||||
}
|
||||
|
||||
export function requireAdmin(headers: IncomingHttpHeaders, secret: string, revokedBefore?: number) {
|
||||
const token = bearer(headers);
|
||||
if (!token) throw new DomainError(ErrorCodes.Unauthorized, "请先登录管理员账户", undefined, 401);
|
||||
const payload = verifyAccessToken(token, secret, "admin", revokedBefore);
|
||||
if (!payload.sub) throw new DomainError(ErrorCodes.Forbidden, "管理员身份无效", undefined, 403);
|
||||
return payload;
|
||||
}
|
||||
|
||||
export function publicUser(id: string, details?: { username?: string; email?: string; phone?: string; displayName?: string; status?: AuthUser["status"]; mfaRequired?: boolean; mfaEnabled?: boolean; mfaSetupRequired?: boolean }): AuthUser {
|
||||
const email = details?.email ? `${details.email.slice(0, 2)}***${details.email.slice(details.email.indexOf("@"))}` : undefined;
|
||||
const phone = details?.phone ? `${details.phone.slice(0, 3)}****${details.phone.slice(-2)}` : undefined;
|
||||
return { id, username: details?.username, email, phone, displayName: details?.displayName || "本地用户", status: details?.status || "active", mfaRequired: details?.mfaRequired ?? false, mfaEnabled: details?.mfaEnabled ?? false, mfaSetupRequired: details?.mfaSetupRequired ?? false };
|
||||
}
|
||||
|
||||
export function newSessionId() {
|
||||
return randomUUID();
|
||||
}
|
||||
|
||||
export function hashPassword(password: string) {
|
||||
const salt = randomBytes(16).toString("hex");
|
||||
const digest = scryptSync(password, salt, 64).toString("hex");
|
||||
return `scrypt:v1:${salt}:${digest}`;
|
||||
}
|
||||
|
||||
export function verifyPassword(password: string, encoded: string) {
|
||||
const parts = encoded.split(":");
|
||||
const algorithm = parts[0];
|
||||
const salt = parts.length === 4 ? parts[2] : parts[1];
|
||||
const digest = parts.length === 4 ? parts[3] : parts[2];
|
||||
if (algorithm !== "scrypt" || !salt || !digest) return false;
|
||||
const candidate = scryptSync(password, salt, 64).toString("hex");
|
||||
return candidate.length === digest.length && timingSafeEqual(Buffer.from(candidate), Buffer.from(digest));
|
||||
}
|
||||
|
||||
export function hashRefreshToken(token: string) {
|
||||
return createHash("sha256").update(token).digest("hex");
|
||||
}
|
||||
|
||||
export function hashChallenge(value: string, secret: string) {
|
||||
return createHmac("sha256", secret).update(value).digest("hex");
|
||||
}
|
||||
|
||||
export function encryptSecret(value: string, secret: string) {
|
||||
const key = createHash("sha256").update(secret).digest(); const iv = randomBytes(12); const cipher = createCipheriv("aes-256-gcm", key, iv); const encrypted = Buffer.concat([cipher.update(value, "utf8"), cipher.final()]); return `aesgcm:v1:${iv.toString("base64url")}:${cipher.getAuthTag().toString("base64url")}:${encrypted.toString("base64url")}`;
|
||||
}
|
||||
|
||||
export function decryptSecret(value: string | undefined, secret: string) {
|
||||
if (!value) return undefined; const [algorithm, version, iv, tag, payload] = value.split(":"); if (algorithm !== "aesgcm" || version !== "v1" || !iv || !tag || !payload) return undefined;
|
||||
try { const decipher = createDecipheriv("aes-256-gcm", createHash("sha256").update(secret).digest(), Buffer.from(iv, "base64url")); decipher.setAuthTag(Buffer.from(tag, "base64url")); return Buffer.concat([decipher.update(Buffer.from(payload, "base64url")), decipher.final()]).toString("utf8"); } catch { return undefined; }
|
||||
}
|
||||
|
||||
export function createNumericCode() {
|
||||
return String(100000 + (randomBytes(4).readUInt32BE(0) % 900000));
|
||||
}
|
||||
|
||||
export function createRecoveryCodes(count = 10) {
|
||||
const alphabet = "ABCDEFGHJKLMNPQRSTUVWXYZ23456789";
|
||||
return Array.from({ length: count }, () => {
|
||||
const bytes = randomBytes(10);
|
||||
return Array.from(bytes, (byte) => alphabet[byte % alphabet.length]).join("").match(/.{1,5}/g)!.join("-");
|
||||
});
|
||||
}
|
||||
|
||||
const BASE32_ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567";
|
||||
|
||||
export function createTotpSecret() {
|
||||
const bytes = randomBytes(20);
|
||||
let output = "";
|
||||
let buffer = 0;
|
||||
let bits = 0;
|
||||
for (const byte of bytes) {
|
||||
buffer = (buffer << 8) | byte;
|
||||
bits += 8;
|
||||
while (bits >= 5) {
|
||||
output += BASE32_ALPHABET[(buffer >>> (bits - 5)) & 31];
|
||||
bits -= 5;
|
||||
}
|
||||
}
|
||||
if (bits > 0) output += BASE32_ALPHABET[(buffer << (5 - bits)) & 31];
|
||||
return output;
|
||||
}
|
||||
|
||||
function decodeBase32(value: string) {
|
||||
let buffer = 0;
|
||||
let bits = 0;
|
||||
const bytes: number[] = [];
|
||||
for (const character of value.replace(/=|\s/g, "").toUpperCase()) {
|
||||
const index = BASE32_ALPHABET.indexOf(character);
|
||||
if (index < 0) return undefined;
|
||||
buffer = (buffer << 5) | index;
|
||||
bits += 5;
|
||||
if (bits >= 8) {
|
||||
bytes.push((buffer >>> (bits - 8)) & 255);
|
||||
bits -= 8;
|
||||
}
|
||||
}
|
||||
return Buffer.from(bytes);
|
||||
}
|
||||
|
||||
function totpAt(secret: string, counter: number) {
|
||||
const key = decodeBase32(secret);
|
||||
if (!key) return undefined;
|
||||
const message = Buffer.alloc(8);
|
||||
message.writeBigInt64BE(BigInt(counter));
|
||||
const digest = createHmac("sha1", key).update(message).digest();
|
||||
const offset = digest[digest.length - 1] & 0x0f;
|
||||
const value = ((digest[offset] & 0x7f) << 24) | (digest[offset + 1] << 16) | (digest[offset + 2] << 8) | digest[offset + 3];
|
||||
return String(value % 1_000_000).padStart(6, "0");
|
||||
}
|
||||
|
||||
export function verifyTotpCode(code: unknown, secret: string | undefined, now = Date.now()) {
|
||||
return verifyTotpCodeWithCounter(code, secret, now) !== undefined;
|
||||
}
|
||||
|
||||
export function verifyTotpCodeWithCounter(code: unknown, secret: string | undefined, now = Date.now(), lastAcceptedCounter = -1) {
|
||||
if (!secret || typeof code !== "string" || !/^\d{6}$/.test(code)) return undefined;
|
||||
const counter = Math.floor(now / 30_000);
|
||||
for (const offset of [-1, 0, 1]) {
|
||||
const candidateCounter = counter + offset;
|
||||
if (candidateCounter <= lastAcceptedCounter) continue;
|
||||
const expected = totpAt(secret, counter + offset);
|
||||
if (expected && timingSafeEqual(Buffer.from(expected), Buffer.from(code))) return candidateCounter;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
export function totpProvisioningUri(secret: string, account: string, issuer = "MiragenFlow") {
|
||||
return `otpauth://totp/${encodeURIComponent(issuer)}:${encodeURIComponent(account)}?secret=${secret}&issuer=${encodeURIComponent(issuer)}&algorithm=SHA1&digits=6&period=30`;
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
import { ErrorCodes, type ApiError, type ErrorCode } from "@miragenflow/contracts";
|
||||
|
||||
export class DomainError extends Error {
|
||||
constructor(public readonly code: ErrorCode, message: string, public readonly details?: Record<string, unknown>, public readonly status = 400) {
|
||||
super(message);
|
||||
this.name = "DomainError";
|
||||
}
|
||||
}
|
||||
|
||||
export function apiError(error: unknown, requestId: string): ApiError {
|
||||
if (error instanceof DomainError) return { code: error.code, message: error.message, details: error.details, requestId };
|
||||
return { code: ErrorCodes.Internal, message: "服务器暂时无法处理请求", requestId };
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
|
||||
export function requestIdFrom(request: { headers: Record<string, string | string[] | undefined> }) {
|
||||
const header = request.headers["x-request-id"];
|
||||
return typeof header === "string" && /^[a-zA-Z0-9._:-]{8,128}$/.test(header) ? header : randomUUID();
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
import type { TaskEvent } from "@miragenflow/contracts";
|
||||
|
||||
const payloadKeys: Record<TaskEvent["type"], readonly string[]> = {
|
||||
"task.queued": ["estimatedCost"],
|
||||
"task.running": [],
|
||||
"task.attempt": ["sequence", "status"],
|
||||
"task.progress": ["status", "reconciled", "chargedAmount", "outputCount", "partial", "retentionAction", "retentionState"],
|
||||
"task.succeeded": ["reconciled", "chargedAmount", "outputCount", "partial"],
|
||||
"task.failed": [],
|
||||
"task.canceled": ["released"],
|
||||
"task.unknown": ["reconciliationRequired"],
|
||||
};
|
||||
|
||||
export function publicTaskEvent(event: TaskEvent): TaskEvent {
|
||||
const payload: Record<string, unknown> = {};
|
||||
for (const key of payloadKeys[event.type]) if (event.payload[key] !== undefined) payload[key] = event.payload[key];
|
||||
return { eventId: event.eventId, sequence: event.sequence, taskId: event.taskId, attemptId: event.attemptId, type: event.type, occurredAt: event.occurredAt, payload };
|
||||
}
|
||||
@@ -0,0 +1,626 @@
|
||||
import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
|
||||
import { dirname } from "node:path";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import type { AssetRecord, BalanceBucket, BalanceLedgerEntry, BalanceSummary, BalanceUnitConfig, CapabilitySchema, GenerationTask, InternalModelProduct, MembershipPlan, TaskEvent, UploadObject } from "@miragenflow/contracts";
|
||||
import { openAiImageCapabilityParameters } from "./image-options.ts";
|
||||
import { createPersistenceRepository, type PersistenceRepository, type RepositoryHealth } from "./infra/repository.ts";
|
||||
import { createTaskQueue, type QueueHealth, type TaskQueue } from "./infra/queue.ts";
|
||||
|
||||
export type UserRecord = { id: string; version: number; username?: string; email?: string; phone?: string; displayName?: string; password: string; verified: boolean; status: "pending_verification" | "active" | "frozen" | "closing"; planId?: string; maxConcurrent?: number; dailyTaskLimit?: number; allowedModelProductIds?: string[]; passwordChangedAt?: string; mfaRequired: boolean; mfaEnabled: boolean; mfaSecretEncrypted?: string; mfaRecoveryCodeHashes?: string[]; mfaLastTotpCounter?: number; roles: string[]; failedLoginCount: number; lockedUntil?: number; createdAt: string };
|
||||
export type ProviderModelMapping = { displayModelId: string; requestModelId: string };
|
||||
export type ProviderChannel = { id: string; label: string; enabled: boolean; priority: number; health: "healthy" | "degraded" | "open"; fixtureFailure?: "retryable" | "unknown"; providerType?: "generic" | "openai-images"; providerName?: string; providerModelId?: string; enabledModelIds?: string[]; modelMappings?: ProviderModelMapping[]; resolutionModelMap?: Record<string, string>; baseUrl?: string; secretRef?: string; version: number; retryBudget: number; consecutiveFailures: number; circuitOpenedAt?: number; circuitResetAt?: number; lastTestedAt?: string; lastTestHealthy?: boolean; lastTestRequestPath?: string; lastTestRequestAttempted?: boolean; lastTestError?: string };
|
||||
export type ChannelGroup = {
|
||||
id: string;
|
||||
version: number;
|
||||
name?: string;
|
||||
capabilities?: string[];
|
||||
channelIds: string[];
|
||||
channelPriorities?: Record<string, number>;
|
||||
enabled: boolean;
|
||||
retryBudget: number;
|
||||
};
|
||||
export type SessionRecord = { userId: string; scope: "user" | "admin"; expiresAt: number; absoluteExpiresAt: number; createdAt: number; rotationId: string; familyId: string; revokedAt?: number; replacedBy?: string; deviceId?: string; roles?: string[]; scopes?: string[] };
|
||||
export type VerificationRecord = { codeHash: string; expiresAt: number; attempts: number; maxAttempts: number; consumedAt?: number; targetHash?: string; purpose: "register" | "login" | "reset" | "mfa"; ip?: string; deviceId?: string };
|
||||
export type CaptchaChallengeRecord = { id: string; action: "register" | "login" | "verify" | "reset" | "admin-login"; question: string; answerHash: string; targetHash: string; ip: string; deviceId?: string; expiresAt: number; attempts: number; maxAttempts: number; consumedAt?: number };
|
||||
export type MfaChallengeRecord = { userId: string; scope: "user" | "admin"; purpose: "login" | "enrollment" | "setup"; expiresAt: number; attempts: number; pendingSecretEncrypted?: string; pendingRecoveryCodeHashes?: string[] };
|
||||
export type RechargeOrder = { id: string; userId: string; provider: "mock" | "manual" | "adapter"; status: "pending" | "paid" | "failed" | "refunded"; fiatAmount: number; balanceAmount: number; unitVersion: number; idempotencyKey: string; requestFingerprint?: string; createdAt: string; paidAt?: string; creditBucketId?: string; version?: number };
|
||||
export type WebDavRecord = { userId: string; configured: boolean; directory: string; state: "unconfigured" | "ready" | "syncing" | "error"; retentionState?: "active" | "deleting" | "deleted" | "error"; lastSyncedAt?: string; encryptedUrl?: string; encryptedUsername?: string; encryptedPassword?: string; retentionDays: number; manifestEtag?: string; manifestVersion?: number; manifestChecksum?: string; manifestRetentionExpiresAt?: string; manifestExtensionDays?: number };
|
||||
export type MessageOutboxRecord = { id: string; idempotencyKey?: string; channel: "email" | "sms"; targetHash: string; targetEncrypted?: string; purpose: "register" | "reset" | "notification"; payloadEncrypted?: string; status: "pending" | "queued" | "sending" | "sent" | "failed" | "dead"; attempts: number; providerMessageId?: string; lastError?: string; nextAttemptAt?: string; leaseOwner?: string; leaseExpiresAt?: string; createdAt: string; updatedAt: string };
|
||||
export type AuditRecord = { id: string; actorId?: string; action: string; objectType: string; objectId?: string; before?: unknown; after?: unknown; ip?: string; requestId: string; createdAt: string; previousHash?: string; hash?: string };
|
||||
export type WebDavFileRecord = { userId: string; path: string; mimeType: string; data: string; checksum: string; etag: string; version: number; updatedAt: string; deletedAt?: string; syncState?: "pending" | "synced" };
|
||||
export type WebDavSyncJob = { id: string; userId: string; operation: "put" | "delete"; intent?: "sync-put" | "sync-delete" | "archive-put" | "archive-delete" | "conflict-copy" | "retention-delete"; path: string; data?: string; mimeType?: string; ifMatch?: string; attempts: number; status: "queued" | "running" | "succeeded" | "failed" | "conflict"; nextAttemptAt: string; leaseOwner?: string; leaseExpiresAt?: string; lastError?: string; conflictCopyPath?: string; createdAt: string };
|
||||
export type TaskDispatchOutboxRecord = { taskId: string; status: "pending" | "leased" | "sent" | "failed"; availableAt: string; attempts: number; leaseOwner?: string; leaseExpiresAt?: string; lastError?: string; dispatchedAt?: string; createdAt: string };
|
||||
export type AdminIdempotencyRecord = { fingerprint: string; claimToken?: string; leaseExpiresAt?: string; state: "pending" | "completed"; statusCode?: number; data?: unknown; createdAt: string };
|
||||
export type PaymentEventRecord = { id: string; provider: "mock" | "adapter"; externalEventId: string; orderId: string; status: "received" | "applied" | "rejected"; payloadDigest: string; createdAt: string; appliedAt?: string };
|
||||
export type AdminApprovalRecord = { id: string; kind: "manual_recharge" | "refund" | "high_risk_change"; requesterId: string; approverId?: string; status: "pending" | "approved" | "executed" | "rejected" | "expired"; payload: Record<string, unknown>; reason: string; version: number; createdAt: string; expiresAt: string; approvedAt?: string; executedAt?: string };
|
||||
export type AdminAccountRecord = { id: string; email: string; password: string; role: string; mfaSecretEncrypted?: string; mfaRequired: boolean; recoveryCodeHashes: string[]; mfaLastTotpCounter?: number };
|
||||
export type ReserveAllocation = { bucketId: string; amount: number };
|
||||
export type PlanPurchase = { id: string; userId: string; planId: string; planVersion: number; provider: "mock" | "adapter"; status: "pending" | "paid" | "failed" | "refunded"; purchaseAmount: number; includedBalance: number; idempotencyKey: string; requestFingerprint?: string; createdAt: string; paidAt?: string; creditBucketId?: string; entitlementId?: string; planSnapshot?: MembershipPlan };
|
||||
export type PlanEntitlement = { id: string; userId: string; planId: string; planVersion: number; expiresAt: string; includedBalance: number; remainingBalance: number; creditBucketId?: string; allowedModelProductIds: string[]; allowedChannelGroupIds: string[]; maxConcurrent: number; queuePriority: number; retentionDays: number; maxRetentionExtensionDays: number };
|
||||
export type StoragePolicy = { id: string; version: number; stagingAdapter: "local-private" | "object-storage"; retentionDays: number; maxRetentionExtensionDays: number; archiveMode: "webdav-first" | "disabled"; updatedAt: string };
|
||||
export type SystemSettings = { id: string; version: number; adminMfaRequired: boolean; adminCaptchaRequired: boolean; userMfaRequired: boolean; userVerificationRequired: boolean; emailRegistrationVerificationRequired: boolean; phoneRegistrationVerificationRequired: boolean; captchaRequired: boolean; rateLimitWindowMs: number; rateLimitMax: number; queueMaxConcurrent: number; auditRetentionDays: number; auditMaxEntries: number; updatedAt: string };
|
||||
export type StoreTransactionOptions = { reload?: boolean };
|
||||
|
||||
export type Store = {
|
||||
users: Map<string, UserRecord>;
|
||||
sessions: Map<string, SessionRecord>;
|
||||
sessionHashes: Map<string, string>;
|
||||
sessionFamilies: Map<string, { userId: string; scope: "user" | "admin"; revokedAt?: number }>;
|
||||
refreshReuse: Map<string, { revokedAt: number; userId: string; scope: "user" | "admin"; familyId?: string }>;
|
||||
revokedAccessBefore: Map<string, number>;
|
||||
idempotency: Map<string, GenerationTask>;
|
||||
idempotencyBodies: Map<string, string>;
|
||||
retentionIdempotency: Map<string, { fingerprint: string; taskId: string }>;
|
||||
tasks: Map<string, GenerationTask>;
|
||||
events: Map<string, TaskEvent[]>;
|
||||
balances: Map<string, { available: number; reserved: number; version?: number }>;
|
||||
buckets: Map<string, BalanceBucket[]>;
|
||||
reserveAllocations: Map<string, ReserveAllocation[]>;
|
||||
products: InternalModelProduct[];
|
||||
balanceUnit: BalanceUnitConfig;
|
||||
plans: MembershipPlan[];
|
||||
verificationCodes: Map<string, VerificationRecord>;
|
||||
passwordResetCodes: Map<string, VerificationRecord>;
|
||||
passwordResetRequests: Map<string, { userId: string; expiresAt: number }>;
|
||||
mfaChallenges: Map<string, MfaChallengeRecord>;
|
||||
captchaChallenges: Map<string, CaptchaChallengeRecord>;
|
||||
rateLimits: Map<string, { count: number; resetAt: number }>;
|
||||
ledger: BalanceLedgerEntry[];
|
||||
rechargeOrders: Map<string, RechargeOrder>;
|
||||
rechargeIdempotency: Map<string, RechargeOrder>;
|
||||
paymentEvents: Map<string, PaymentEventRecord>;
|
||||
adminIdempotency: Map<string, AdminIdempotencyRecord>;
|
||||
adminApprovals: Map<string, AdminApprovalRecord>;
|
||||
adminAccounts: Map<string, AdminAccountRecord>;
|
||||
planPurchases: Map<string, PlanPurchase>;
|
||||
entitlements: Map<string, PlanEntitlement>;
|
||||
channels: ProviderChannel[];
|
||||
channelGroups: ChannelGroup[];
|
||||
taskSubscribers: Map<string, Set<(event: TaskEvent) => void>>;
|
||||
audit: AuditRecord[];
|
||||
uploads: Map<string, UploadObject>;
|
||||
assets: Map<string, AssetRecord>;
|
||||
webdav: Map<string, WebDavRecord>;
|
||||
webdavFiles: Map<string, WebDavFileRecord>;
|
||||
webdavJobs: Map<string, WebDavSyncJob>;
|
||||
taskDispatchOutbox: Map<string, TaskDispatchOutboxRecord>;
|
||||
messageOutbox: Map<string, MessageOutboxRecord>;
|
||||
messageProviders: Map<string, Record<string, unknown>>;
|
||||
messageTemplates: Map<string, Record<string, unknown>>;
|
||||
paymentProviders: Map<string, Record<string, unknown>>;
|
||||
storagePolicy: StoragePolicy;
|
||||
systemSettings: SystemSettings;
|
||||
systemSettingsConfigured: boolean;
|
||||
systemSettingsRegistrationConfigured: boolean;
|
||||
objects: Map<string, { id: string; ownerId: string; mimeType: string; data?: string; stagingKey?: string; expiresAt: string; taskId?: string; revoked?: boolean }>;
|
||||
queue: string[];
|
||||
transactionDepth: number;
|
||||
transactionEvents: TaskEvent[];
|
||||
transact: <T>(mutator: (store: Store) => Promise<T> | T, retries?: number, options?: StoreTransactionOptions) => Promise<T>;
|
||||
persistenceFailure?: string;
|
||||
persistencePath?: string;
|
||||
persist: () => void;
|
||||
persistAsync: () => Promise<void>;
|
||||
loadPersisted: () => Promise<void>;
|
||||
repository: PersistenceRepository<unknown>;
|
||||
queueAdapter: TaskQueue;
|
||||
stagingDir: string;
|
||||
health: () => Promise<{ persistence: RepositoryHealth; queue: QueueHealth }>;
|
||||
channelEncryptionKey: string;
|
||||
};
|
||||
|
||||
function defaultUnit(): BalanceUnitConfig { return { name: "金币", shortCode: "COIN", icon: "coins", precision: 0, version: 1, currencyCode: "CNY", conversionRate: 1, rounding: "round" }; }
|
||||
|
||||
export function capabilitySchema(capability: InternalModelProduct["capabilities"][number]): CapabilitySchema {
|
||||
const schema: CapabilitySchema = { capability, inputTypes: capability === "text" ? ["prompt"] : ["prompt", "image"], resultTypes: capability === "audio" ? ["audio"] : capability === "text" || capability === "reverse-prompt" ? ["text"] : ["image"], async: true, maxInputBytes: 50 * 1024 * 1024 };
|
||||
if (capability === "image") schema.parameters = openAiImageCapabilityParameters(true);
|
||||
return schema;
|
||||
}
|
||||
|
||||
function defaultProducts(): InternalModelProduct[] {
|
||||
const imageCapabilities = ["image", "utility", "reverse-prompt", "multi-angle", "matting", "split", "upscale", "expand"] as const;
|
||||
const imageRatios = ["1:1", "16:9", "9:16", "3:4", "4:5", "3:2", "2:3", "4:3", "1.91:1", "2.35:1", "21:9"];
|
||||
return [
|
||||
{ id: "basic-image", publicModelId: "basic-image-v1", name: "基础模型", tier: "basic", capabilities: [...imageCapabilities], basePrice: 10, currencyUnit: "金币", enabled: true, version: 1, maxCount: 8, maxConcurrent: 2, channelGroupId: "image-default", capabilitiesSchema: imageCapabilities.map(capabilitySchema), resolutions: [{ id: "low", label: "低", width: 1024, height: 1024, priceMultiplier: 1, ratios: imageRatios }] },
|
||||
{ id: "advanced-image", publicModelId: "advanced-image-v1", name: "高级模型", tier: "advanced", capabilities: [...imageCapabilities], basePrice: 25, currencyUnit: "金币", enabled: true, version: 1, maxCount: 8, maxConcurrent: 3, channelGroupId: "image-default", capabilitiesSchema: imageCapabilities.map(capabilitySchema), resolutions: [{ id: "medium", label: "中", width: 2048, height: 2048, priceMultiplier: 1.5, ratios: imageRatios }] },
|
||||
{ id: "flagship-image", publicModelId: "flagship-image-v1", name: "旗舰模型", tier: "flagship", capabilities: [...imageCapabilities], basePrice: 60, currencyUnit: "金币", enabled: true, version: 1, maxCount: 4, maxConcurrent: 1, channelGroupId: "image-default", capabilitiesSchema: imageCapabilities.map(capabilitySchema), resolutions: [{ id: "high", label: "高", width: 2880, height: 2880, priceMultiplier: 2, ratios: imageRatios }] },
|
||||
{ id: "basic-text", publicModelId: "basic-text-v1", name: "文字理解", tier: "basic", capabilities: ["text"], basePrice: 2, currencyUnit: "金币", enabled: true, version: 1, maxCount: 1, maxConcurrent: 4, channelGroupId: "image-default", capabilitiesSchema: [capabilitySchema("text")], resolutions: [] },
|
||||
{ id: "basic-audio", publicModelId: "basic-audio-v1", name: "音频生成", tier: "basic", capabilities: ["audio"], basePrice: 8, currencyUnit: "金币", enabled: true, version: 1, maxCount: 1, maxConcurrent: 2, channelGroupId: "image-default", capabilitiesSchema: [capabilitySchema("audio")], resolutions: [] },
|
||||
];
|
||||
}
|
||||
|
||||
type StoreOptions = { persistenceAdapter?: "memory" | "file" | "postgres"; queueAdapter?: "memory" | "redis"; databaseUrl?: string; redisUrl?: string; stagingDir?: string; channelEncryptionKey?: string };
|
||||
|
||||
function baseStore(path: string | undefined, options: StoreOptions): Store {
|
||||
const unit = defaultUnit();
|
||||
const fixtureMode = process.env.NODE_ENV !== "production" || process.env.MIRAGENFLOW_TEST_MODE === "true";
|
||||
const channelFixtureMode = process.env.NODE_ENV === "test" || process.env.MIRAGENFLOW_TEST_MODE === "true";
|
||||
const paymentFixtureMode = process.env.NODE_ENV === "test" || process.env.MIRAGENFLOW_TEST_MODE === "true";
|
||||
const products = fixtureMode ? defaultProducts() : [];
|
||||
const plans = fixtureMode ? [{ id: "starter", code: "starter", name: "基础套餐", icon: "coins", description: "用于开始创作", durationDays: 30, purchaseAmount: 0, includedBalance: 100, published: true, allowedModelProductIds: ["basic-image", "basic-text", "basic-audio"], allowedChannelGroupIds: ["image-default"], maxConcurrent: 2, queuePriority: 0, retentionDays: 30, maxRetentionExtensionDays: 30, version: 1 }] : [];
|
||||
const channels: ProviderChannel[] = channelFixtureMode ? ["channel-a", "channel-b", "channel-c", "channel-d"].map((id, index) => ({ id, label: `渠道 ${index + 1}`, enabled: true, priority: index, health: "healthy" as const, version: 1, retryBudget: 0, consecutiveFailures: 0 })) : [];
|
||||
const channelGroups = channelFixtureMode ? [{ id: "image-default", version: 1, name: "图片生成渠道组", capabilities: ["image"], channelIds: ["channel-a", "channel-b", "channel-c", "channel-d"], channelPriorities: { "channel-a": 0, "channel-b": 1, "channel-c": 2, "channel-d": 3 }, enabled: true, retryBudget: 4 }] : fixtureMode ? [{ id: "image-default", version: 1, name: "图片生成渠道组", capabilities: ["image"], channelIds: [], channelPriorities: {}, enabled: true, retryBudget: 0 }] : [];
|
||||
const paymentProviders = paymentFixtureMode ? new Map([["mock", { id: "mock", name: "模拟支付适配器", enabled: true, mode: "test", secretConfigured: false, version: 1 }]]) : new Map<string, Record<string, unknown>>();
|
||||
const repository = createPersistenceRepository({ adapter: options.persistenceAdapter || (path ? "file" : "memory"), filePath: path, databaseUrl: options.databaseUrl, key: "store" });
|
||||
const queueAdapter = createTaskQueue({ adapter: options.queueAdapter, redisUrl: options.redisUrl });
|
||||
return {
|
||||
users: new Map(), sessions: new Map(), sessionHashes: new Map(), sessionFamilies: new Map(), refreshReuse: new Map(), revokedAccessBefore: new Map(), idempotency: new Map(), idempotencyBodies: new Map(), retentionIdempotency: new Map(), tasks: new Map(), events: new Map(), balances: new Map(), buckets: new Map(), reserveAllocations: new Map(), products, balanceUnit: unit,
|
||||
plans,
|
||||
verificationCodes: new Map(), passwordResetCodes: new Map(), passwordResetRequests: new Map(), mfaChallenges: new Map(), captchaChallenges: new Map(), rateLimits: new Map(), ledger: [], rechargeOrders: new Map(), rechargeIdempotency: new Map(), paymentEvents: new Map(), adminIdempotency: new Map(), adminApprovals: new Map(), adminAccounts: new Map(), planPurchases: new Map(), entitlements: new Map(),
|
||||
channels, channelGroups, taskSubscribers: new Map(), audit: [], uploads: new Map(), assets: new Map(), webdav: new Map(), webdavFiles: new Map(), webdavJobs: new Map(), taskDispatchOutbox: new Map(), messageOutbox: new Map(), messageProviders: new Map(), messageTemplates: new Map(), paymentProviders, storagePolicy: { id: "default", version: 1, stagingAdapter: "local-private", retentionDays: 30, maxRetentionExtensionDays: 30, archiveMode: "webdav-first", updatedAt: new Date().toISOString() }, systemSettings: { id: "default", version: 1, adminMfaRequired: process.env.MIRAGENFLOW_ADMIN_MFA_REQUIRED === "true", adminCaptchaRequired: process.env.MIRAGENFLOW_ADMIN_CAPTCHA_REQUIRED === "true", userMfaRequired: process.env.MIRAGENFLOW_USER_MFA_REQUIRED === "true", userVerificationRequired: process.env.MIRAGENFLOW_USER_VERIFICATION_REQUIRED === "true", emailRegistrationVerificationRequired: process.env.MIRAGENFLOW_EMAIL_REGISTRATION_VERIFICATION_REQUIRED === "true" || process.env.MIRAGENFLOW_USER_VERIFICATION_REQUIRED === "true", phoneRegistrationVerificationRequired: process.env.MIRAGENFLOW_PHONE_REGISTRATION_VERIFICATION_REQUIRED === "true" || process.env.MIRAGENFLOW_USER_VERIFICATION_REQUIRED === "true", captchaRequired: process.env.MIRAGENFLOW_CAPTCHA_REQUIRED === "true", rateLimitWindowMs: Number(process.env.MIRAGENFLOW_RATE_LIMIT_WINDOW_MS) || 60_000, rateLimitMax: Number(process.env.MIRAGENFLOW_RATE_LIMIT_MAX) || 12, queueMaxConcurrent: 4, auditRetentionDays: 365, auditMaxEntries: 100_000, updatedAt: new Date().toISOString() }, systemSettingsConfigured: false, systemSettingsRegistrationConfigured: Boolean(process.env.MIRAGENFLOW_EMAIL_REGISTRATION_VERIFICATION_REQUIRED !== undefined || process.env.MIRAGENFLOW_PHONE_REGISTRATION_VERIFICATION_REQUIRED !== undefined), objects: new Map(), queue: [], transactionDepth: 0, transactionEvents: [], transact: async () => { throw new Error("store transaction not initialized"); }, persistenceFailure: undefined, persistencePath: path, persist: () => undefined, persistAsync: async () => undefined, loadPersisted: async () => undefined, repository, queueAdapter, stagingDir: options.stagingDir || process.env.MIRAGENFLOW_STAGING_DIR?.trim() || "/tmp/miragenflow-staging", health: async () => ({ persistence: await repository.health(), queue: await queueAdapter.health() }), channelEncryptionKey: options.channelEncryptionKey || process.env.MIRAGENFLOW_CHANNEL_ENCRYPTION_KEY || process.env.MIRAGENFLOW_ACCESS_TOKEN_SECRET || "local-development-secret-change-me",
|
||||
};
|
||||
}
|
||||
|
||||
type Persisted = { users: [string, UserRecord][]; sessions: [string, SessionRecord][]; sessionHashes: [string, string][]; sessionFamilies: [string, Store["sessionFamilies"] extends Map<string, infer V> ? V : never][]; refreshReuse: [string, Store["refreshReuse"] extends Map<string, infer V> ? V : never][]; revokedAccessBefore: [string, number][]; idempotency: [string, GenerationTask][]; idempotencyBodies: [string, string][]; retentionIdempotency: [string, { fingerprint: string; taskId: string }][]; tasks: [string, GenerationTask][]; events: [string, TaskEvent[]][]; balances: [string, { available: number; reserved: number; version?: number }][]; buckets: [string, BalanceBucket[]][]; reserveAllocations: [string, ReserveAllocation[]][]; products: InternalModelProduct[]; balanceUnit: BalanceUnitConfig; plans: MembershipPlan[]; planPurchases: [string, PlanPurchase][]; entitlements: [string, PlanEntitlement][]; verificationCodes: [string, VerificationRecord][]; passwordResetCodes: [string, VerificationRecord][]; passwordResetRequests: [string, { userId: string; expiresAt: number }][]; mfaChallenges: [string, MfaChallengeRecord][]; captchaChallenges: [string, CaptchaChallengeRecord][]; rateLimits: [string, { count: number; resetAt: number }][]; ledger: BalanceLedgerEntry[]; rechargeOrders: [string, RechargeOrder][]; rechargeIdempotency: [string, RechargeOrder][]; paymentEvents: [string, PaymentEventRecord][]; adminIdempotency: [string, AdminIdempotencyRecord][]; adminApprovals: [string, AdminApprovalRecord][]; adminAccounts: [string, AdminAccountRecord][]; channels: ProviderChannel[]; channelGroups: ChannelGroup[]; audit: Store["audit"]; uploads: [string, UploadObject][]; assets: [string, AssetRecord][]; webdav: [string, WebDavRecord][]; webdavFiles: [string, WebDavFileRecord][]; webdavJobs: [string, WebDavSyncJob][]; taskDispatchOutbox: [string, TaskDispatchOutboxRecord][]; messageOutbox: [string, MessageOutboxRecord][]; messageProviders: [string, Record<string, unknown>][]; messageTemplates: [string, Record<string, unknown>][]; paymentProviders: [string, Record<string, unknown>][]; storagePolicy: StoragePolicy; systemSettings?: SystemSettings; systemSettingsRegistrationConfigured?: boolean; objects: [string, { id: string; ownerId: string; mimeType: string; data?: string; stagingKey?: string; expiresAt: string; taskId?: string; revoked?: boolean }][]; queue: string[]; snapshotVersion?: number };
|
||||
|
||||
function restore(store: Store, path: string) {
|
||||
if (!existsSync(path)) return;
|
||||
try {
|
||||
const raw = JSON.parse(readFileSync(path, "utf8")) as Partial<Persisted> & { snapshotVersion?: unknown };
|
||||
if (raw.snapshotVersion !== 1) throw new Error("unsupported or missing persistence snapshot version");
|
||||
applySnapshot(store, raw, true);
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === "ENOENT") return;
|
||||
throw new Error(`persistence snapshot rejected: ${error instanceof Error ? error.message : String(error)}`);
|
||||
}
|
||||
}
|
||||
|
||||
function applySnapshot(store: Store, raw: Partial<Persisted>, replace = false) {
|
||||
const entries: Array<[keyof Persisted, keyof Store]> = [["users", "users"], ["sessions", "sessions"], ["sessionHashes", "sessionHashes"], ["sessionFamilies", "sessionFamilies"], ["refreshReuse", "refreshReuse"], ["revokedAccessBefore", "revokedAccessBefore"], ["idempotency", "idempotency"], ["idempotencyBodies", "idempotencyBodies"], ["retentionIdempotency", "retentionIdempotency"], ["tasks", "tasks"], ["events", "events"], ["balances", "balances"], ["buckets", "buckets"], ["reserveAllocations", "reserveAllocations"], ["verificationCodes", "verificationCodes"], ["passwordResetCodes", "passwordResetCodes"], ["passwordResetRequests", "passwordResetRequests"], ["mfaChallenges", "mfaChallenges"], ["captchaChallenges", "captchaChallenges"], ["rateLimits", "rateLimits"], ["rechargeOrders", "rechargeOrders"], ["rechargeIdempotency", "rechargeIdempotency"], ["paymentEvents", "paymentEvents"], ["adminIdempotency", "adminIdempotency"], ["adminApprovals", "adminApprovals"], ["adminAccounts", "adminAccounts"], ["planPurchases", "planPurchases"], ["entitlements", "entitlements"], ["uploads", "uploads"], ["assets", "assets"], ["webdav", "webdav"], ["webdavFiles", "webdavFiles"], ["webdavJobs", "webdavJobs"], ["taskDispatchOutbox", "taskDispatchOutbox"], ["messageOutbox", "messageOutbox"], ["messageProviders", "messageProviders"], ["messageTemplates", "messageTemplates"], ["paymentProviders", "paymentProviders"], ["objects", "objects"]];
|
||||
if (replace) {
|
||||
for (const [, target] of entries) (store[target] as Map<unknown, unknown>).clear();
|
||||
store.ledger = []; store.products = []; store.plans = []; store.channels = []; store.channelGroups = []; store.audit = []; store.queue = [];
|
||||
}
|
||||
for (const [source, target] of entries) {
|
||||
const value = raw[source];
|
||||
if (!Array.isArray(value)) continue;
|
||||
const map = store[target] as Map<unknown, unknown>;
|
||||
for (const entry of value) if (Array.isArray(entry) && entry.length === 2) map.set(entry[0], entry[1]);
|
||||
}
|
||||
for (const user of store.users.values()) {
|
||||
user.version = Number.isSafeInteger(user.version) && user.version > 0 ? user.version : 1;
|
||||
}
|
||||
for (const session of store.sessions.values()) {
|
||||
if (!session.absoluteExpiresAt) session.absoluteExpiresAt = session.expiresAt;
|
||||
if (!session.familyId) session.familyId = `legacy:${session.userId}:${session.rotationId}`;
|
||||
if (!store.sessionFamilies.has(session.familyId)) store.sessionFamilies.set(session.familyId, { userId: session.userId, scope: session.scope });
|
||||
}
|
||||
if (Array.isArray(raw.ledger)) store.ledger = raw.ledger;
|
||||
if (Array.isArray(raw.products)) store.products = raw.products;
|
||||
if (raw.balanceUnit) store.balanceUnit = raw.balanceUnit;
|
||||
if (Array.isArray(raw.plans)) store.plans = raw.plans;
|
||||
if (Array.isArray(raw.channels)) store.channels = raw.channels;
|
||||
if (Array.isArray(raw.channelGroups)) store.channelGroups = raw.channelGroups;
|
||||
if (Array.isArray(raw.audit)) store.audit = raw.audit;
|
||||
if (raw.storagePolicy) store.storagePolicy = raw.storagePolicy;
|
||||
if (raw.systemSettings && typeof raw.systemSettings === "object") {
|
||||
store.systemSettings = raw.systemSettings;
|
||||
store.systemSettingsConfigured = true;
|
||||
store.systemSettingsRegistrationConfigured = raw.systemSettingsRegistrationConfigured === true;
|
||||
}
|
||||
if (Array.isArray(raw.queue)) store.queue = raw.queue;
|
||||
}
|
||||
|
||||
function snapshot(store: Store): Persisted {
|
||||
return { users: [...store.users], sessions: [...store.sessions], sessionHashes: [...store.sessionHashes], sessionFamilies: [...store.sessionFamilies], refreshReuse: [...store.refreshReuse], revokedAccessBefore: [...store.revokedAccessBefore], idempotency: [...store.idempotency], idempotencyBodies: [...store.idempotencyBodies], retentionIdempotency: [...store.retentionIdempotency], tasks: [...store.tasks], events: [...store.events], balances: [...store.balances], buckets: [...store.buckets], reserveAllocations: [...store.reserveAllocations], products: store.products, balanceUnit: store.balanceUnit, plans: store.plans, planPurchases: [...store.planPurchases], entitlements: [...store.entitlements], verificationCodes: [...store.verificationCodes], passwordResetCodes: [...store.passwordResetCodes], passwordResetRequests: [...store.passwordResetRequests], mfaChallenges: [...store.mfaChallenges], captchaChallenges: [...store.captchaChallenges], rateLimits: [...store.rateLimits], ledger: store.ledger, rechargeOrders: [...store.rechargeOrders], rechargeIdempotency: [...store.rechargeIdempotency], paymentEvents: [...store.paymentEvents], adminIdempotency: [...store.adminIdempotency], adminApprovals: [...store.adminApprovals], adminAccounts: [...store.adminAccounts], channels: store.channels, channelGroups: store.channelGroups, audit: store.audit, uploads: [...store.uploads], assets: [...store.assets], webdav: [...store.webdav], webdavJobs: [...store.webdavJobs], webdavFiles: [...store.webdavFiles], taskDispatchOutbox: [...store.taskDispatchOutbox], messageOutbox: [...store.messageOutbox], messageProviders: [...store.messageProviders], messageTemplates: [...store.messageTemplates], paymentProviders: [...store.paymentProviders], storagePolicy: store.storagePolicy, systemSettings: store.systemSettings, systemSettingsRegistrationConfigured: store.systemSettingsRegistrationConfigured, objects: [...store.objects], queue: store.queue, snapshotVersion: 1 };
|
||||
}
|
||||
|
||||
function frozenSnapshot(store: Store) { return JSON.parse(JSON.stringify(snapshot(store))) as Persisted; }
|
||||
|
||||
function writeFileSnapshot(path: string, value: Persisted) {
|
||||
mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
|
||||
const temporary = `${path}.${process.pid}.${randomUUID()}.tmp`;
|
||||
writeFileSync(temporary, JSON.stringify(value), { encoding: "utf8", mode: 0o600 });
|
||||
renameSync(temporary, path);
|
||||
}
|
||||
|
||||
export function createStore(path = process.env.MIRAGENFLOW_STORE_FILE?.trim() || undefined, options: StoreOptions = {}): Store {
|
||||
const persistenceAdapter = options.persistenceAdapter || (process.env.MIRAGENFLOW_PERSISTENCE_ADAPTER as StoreOptions["persistenceAdapter"]) || (path ? "file" : "memory");
|
||||
const effectivePath = persistenceAdapter === "file" ? path : undefined;
|
||||
const store = baseStore(effectivePath, { ...options, persistenceAdapter, queueAdapter: options.queueAdapter || (process.env.MIRAGENFLOW_QUEUE_ADAPTER as StoreOptions["queueAdapter"]) || "memory", databaseUrl: options.databaseUrl || process.env.DATABASE_URL || process.env.MIRAGENFLOW_DATABASE_URL, redisUrl: options.redisUrl || process.env.REDIS_URL || process.env.MIRAGENFLOW_REDIS_URL });
|
||||
if (effectivePath) restore(store, effectivePath);
|
||||
let persistChain: Promise<void> = Promise.resolve();
|
||||
let loadChain: Promise<void> = Promise.resolve();
|
||||
let persistenceEpoch = 0;
|
||||
let transactionChain: Promise<unknown> = Promise.resolve();
|
||||
const enqueuePersist = (value: Persisted, expectedRevision = store.repository.getRevision?.()) => {
|
||||
// Recover from a failed write so one transient database error does not
|
||||
// permanently poison every subsequent persistence attempt.
|
||||
const operation = persistChain.catch(() => undefined).then(async () => {
|
||||
await store.repository.save(value, expectedRevision);
|
||||
store.persistenceFailure = undefined;
|
||||
});
|
||||
persistChain = operation.catch((error) => {
|
||||
store.persistenceFailure = error instanceof Error ? error.message : String(error);
|
||||
});
|
||||
return operation;
|
||||
};
|
||||
store.persist = () => {
|
||||
if (store.transactionDepth > 0) return;
|
||||
persistenceEpoch += 1;
|
||||
const value = frozenSnapshot(store);
|
||||
if (store.persistencePath) {
|
||||
writeFileSnapshot(store.persistencePath, value);
|
||||
store.persistenceFailure = undefined;
|
||||
return;
|
||||
}
|
||||
void enqueuePersist(value, store.repository.getRevision?.()).catch(() => undefined);
|
||||
};
|
||||
store.persistAsync = async () => {
|
||||
if (store.transactionDepth > 0) return;
|
||||
persistenceEpoch += 1;
|
||||
const value = frozenSnapshot(store);
|
||||
if (store.persistencePath) {
|
||||
writeFileSnapshot(store.persistencePath, value);
|
||||
store.persistenceFailure = undefined;
|
||||
return;
|
||||
}
|
||||
await persistChain.catch(() => undefined);
|
||||
await enqueuePersist(value, store.repository.getRevision?.());
|
||||
};
|
||||
store.loadPersisted = () => {
|
||||
const operation = loadChain.catch(() => undefined).then(async () => {
|
||||
// Never apply a durable snapshot while an earlier local write is still
|
||||
// in flight. If another write starts while the database read is in
|
||||
// flight, discard that read and try again so an older snapshot cannot
|
||||
// overwrite the newer in-memory graph.
|
||||
for (let attempt = 0; attempt < 4; attempt += 1) {
|
||||
await persistChain.catch(() => undefined);
|
||||
const epoch = persistenceEpoch;
|
||||
const raw = await store.repository.load();
|
||||
if (epoch !== persistenceEpoch) continue;
|
||||
if (raw && typeof raw === "object") applySnapshot(store, raw as Partial<Persisted>, true);
|
||||
store.persistenceFailure = undefined;
|
||||
return;
|
||||
}
|
||||
throw new Error("persistence snapshot changed while loading");
|
||||
});
|
||||
loadChain = operation.catch(() => undefined);
|
||||
return operation;
|
||||
};
|
||||
store.transact = async <T>(mutator: (value: Store) => Promise<T> | T, retries = 3, options: StoreTransactionOptions = {}) => {
|
||||
const run = transactionChain.catch(() => undefined).then(async () => {
|
||||
for (let attempt = 0; attempt < Math.max(1, retries); attempt += 1) {
|
||||
// Most transactions reload the durable snapshot before applying a
|
||||
// mutation so independent processes are fenced by the repository
|
||||
// revision. Ledger/task transitions may be called after provider
|
||||
// output has been staged in memory; those callers opt out of the
|
||||
// reload and commit the already prepared task graph atomically.
|
||||
if (store.repository.adapter === "postgres" && options.reload !== false) { await persistChain.catch(() => undefined); await store.loadPersisted(); }
|
||||
const before = JSON.parse(JSON.stringify(snapshot(store))) as Persisted;
|
||||
store.transactionDepth += 1; store.transactionEvents = [];
|
||||
try {
|
||||
const result = await mutator(store);
|
||||
store.transactionDepth -= 1;
|
||||
await store.persistAsync();
|
||||
const events = store.transactionEvents.splice(0);
|
||||
for (const event of events) for (const subscriber of store.taskSubscribers.get(event.taskId) || []) subscriber(event);
|
||||
return result;
|
||||
} catch (error) {
|
||||
store.transactionDepth = Math.max(0, store.transactionDepth - 1);
|
||||
store.transactionEvents = [];
|
||||
// Validation can fail before the mutator changes anything. Avoid
|
||||
// rebuilding the graph in that case so callers holding a reference
|
||||
// to an untouched record keep observing the same object.
|
||||
if (JSON.stringify(snapshot(store)) !== JSON.stringify(before)) applySnapshot(store, before, true);
|
||||
const conflict = /persistence conflict|revision changed/i.test(error instanceof Error ? error.message : String(error));
|
||||
if (conflict && store.repository.adapter === "postgres" && options.reload !== false && attempt + 1 < Math.max(1, retries)) continue;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
throw new Error("store transaction retries exhausted");
|
||||
});
|
||||
transactionChain = run.catch(() => undefined);
|
||||
return run;
|
||||
};
|
||||
store.health = async () => { const persistence = await store.repository.health(); return { persistence: store.persistenceFailure ? { ...persistence, status: "unavailable" as const, detail: store.persistenceFailure } : persistence, queue: await store.queueAdapter.health() }; };
|
||||
return store;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a task state/ledger transition under one Store transaction. The task
|
||||
* row, balance account, ledger rows and reserve allocations are persisted by
|
||||
* the same repository.save call, so a thrown error rolls every mutation back.
|
||||
* Callers that prepare provider outputs in the current process must explicitly
|
||||
* pass `{ reload: false }`; all other transitions reload the durable task
|
||||
* projection first so an HTTP/admin writer does not mutate a stale snapshot.
|
||||
*/
|
||||
export async function transactTask<T>(
|
||||
store: Store,
|
||||
taskId: string,
|
||||
mutator: (store: Store, task: NonNullable<Store["tasks"] extends Map<string, infer TTask> ? TTask : never>) => Promise<T> | T,
|
||||
options: StoreTransactionOptions = { reload: true },
|
||||
retries = 1,
|
||||
) {
|
||||
return store.transact(async (atomicStore) => {
|
||||
const task = atomicStore.tasks.get(taskId);
|
||||
if (!task) throw new Error(`task ${taskId} not found`);
|
||||
return mutator(atomicStore, task);
|
||||
}, retries, options);
|
||||
}
|
||||
|
||||
/**
|
||||
* Synchronous counterpart used by watchdog callbacks that intentionally keep a
|
||||
* synchronous public API. It still snapshots and rolls back the complete
|
||||
* Store graph; persistence is queued as one snapshot (and PostgreSQL's
|
||||
* repository.save commits that snapshot in one SQL transaction).
|
||||
*/
|
||||
export function transactTaskSync<T>(
|
||||
store: Store,
|
||||
taskId: string,
|
||||
mutator: (store: Store, task: NonNullable<Store["tasks"] extends Map<string, infer TTask> ? TTask : never>) => T,
|
||||
) {
|
||||
const before = JSON.parse(JSON.stringify(snapshot(store))) as Persisted;
|
||||
store.transactionDepth += 1;
|
||||
store.transactionEvents = [];
|
||||
try {
|
||||
const task = store.tasks.get(taskId);
|
||||
if (!task) throw new Error(`task ${taskId} not found`);
|
||||
const result = mutator(store, task);
|
||||
store.transactionDepth = Math.max(0, store.transactionDepth - 1);
|
||||
store.persist();
|
||||
const events = store.transactionEvents.splice(0);
|
||||
for (const event of events) for (const subscriber of store.taskSubscribers.get(event.taskId) || []) subscriber(event);
|
||||
return result;
|
||||
} catch (error) {
|
||||
store.transactionDepth = Math.max(0, store.transactionDepth - 1);
|
||||
store.transactionEvents = [];
|
||||
applySnapshot(store, before, true);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export function balanceSummary(store: Store, userId: string): BalanceSummary { const balance = store.balances.get(userId) || { available: 0, reserved: 0 }; const { name, shortCode, icon, precision, version } = store.balanceUnit; return { available: balance.available, reserved: balance.reserved, unit: { name, shortCode, icon, precision, version } }; }
|
||||
|
||||
/** Increment the durable balance-account version after a successful mutation. */
|
||||
export function bumpBalanceVersion(balance: { version?: number }) {
|
||||
balance.version = (Number.isSafeInteger(balance.version) ? Number(balance.version) : 0) + 1;
|
||||
return balance.version;
|
||||
}
|
||||
|
||||
export function appendEvent(store: Store, taskId: string, event: Omit<TaskEvent, "eventId" | "sequence" | "occurredAt">) {
|
||||
const events = store.events.get(taskId) || [];
|
||||
const next: TaskEvent = { ...event, eventId: randomUUID(), sequence: events.length + 1, occurredAt: new Date().toISOString() };
|
||||
events.push(next); store.events.set(taskId, events);
|
||||
if (store.transactionDepth > 0) store.transactionEvents.push(next);
|
||||
else for (const subscriber of store.taskSubscribers.get(taskId) || []) subscriber(next);
|
||||
const task = store.tasks.get(taskId); if (task) { task.eventSequence = next.sequence; task.updatedAt = next.occurredAt; } store.persist(); return next;
|
||||
}
|
||||
|
||||
export function appendLedger(store: Store, entry: BalanceLedgerEntry) {
|
||||
if (!entry.idempotencyKey || !Number.isSafeInteger(entry.amount) || (entry.amount === 0 && entry.type !== "settle") || store.ledger.some((item) => item.idempotencyKey === entry.idempotencyKey)) return false;
|
||||
store.ledger.push({ ...entry, status: entry.status || "posted" });
|
||||
return true;
|
||||
}
|
||||
|
||||
function expireBuckets(store: Store, userId: string, now = Date.now()) {
|
||||
const buckets = store.buckets.get(userId) || [];
|
||||
const balance = store.balances.get(userId) || { available: 0, reserved: 0 };
|
||||
let expired = 0;
|
||||
let changed = false;
|
||||
for (const bucket of buckets) {
|
||||
if (bucket.remaining <= 0 || !bucket.expiresAt || Date.parse(bucket.expiresAt) > now) continue;
|
||||
const amount = bucket.remaining;
|
||||
const effective = Math.min(amount, Math.max(0, balance.available - expired));
|
||||
const key = `expire:${userId}:${bucket.id}`;
|
||||
const existing = store.ledger.some((entry) => entry.idempotencyKey === key);
|
||||
const posted = existing || effective <= 0 || appendLedger(store, { id: randomUUID(), userId, type: "adjustment", amount: -effective, balanceUnitVersion: store.balanceUnit.version, idempotencyKey: key, createdAt: new Date(now).toISOString(), referenceId: bucket.id, bucketId: bucket.id });
|
||||
if (!posted) continue;
|
||||
expired += effective;
|
||||
bucket.remaining = 0;
|
||||
changed = true;
|
||||
}
|
||||
if (changed) {
|
||||
balance.available = Math.max(0, balance.available - expired);
|
||||
bumpBalanceVersion(balance);
|
||||
store.balances.set(userId, balance);
|
||||
store.persist();
|
||||
}
|
||||
return buckets;
|
||||
}
|
||||
|
||||
function allocationKey(userId: string, referenceId: string) { return `${userId}:${referenceId}`; }
|
||||
|
||||
export function ensureBalanceBuckets(store: Store, userId: string) {
|
||||
const buckets = expireBuckets(store, userId);
|
||||
const balance = store.balances.get(userId) || { available: 0, reserved: 0 };
|
||||
const tracked = buckets.reduce((total, bucket) => total + Math.max(0, bucket.remaining), 0);
|
||||
if (tracked < balance.available) {
|
||||
buckets.push({ id: randomUUID(), userId, source: "manual", remaining: balance.available - tracked, priority: buckets.length });
|
||||
bumpBalanceVersion(balance);
|
||||
store.balances.set(userId, balance);
|
||||
store.buckets.set(userId, buckets);
|
||||
store.persist();
|
||||
}
|
||||
return buckets;
|
||||
}
|
||||
|
||||
function balanceBucketsNeedRefresh(store: Store, userId: string, now = Date.now()) {
|
||||
const balance = store.balances.get(userId) || { available: 0, reserved: 0 };
|
||||
let available = balance.available;
|
||||
let tracked = 0;
|
||||
let expired = false;
|
||||
for (const bucket of store.buckets.get(userId) || []) {
|
||||
if (bucket.remaining > 0 && bucket.expiresAt && !(Date.parse(bucket.expiresAt) > now)) {
|
||||
available -= Math.min(bucket.remaining, Math.max(0, available));
|
||||
expired = true;
|
||||
} else tracked += Math.max(0, bucket.remaining);
|
||||
}
|
||||
return expired || tracked < Math.max(0, available);
|
||||
}
|
||||
|
||||
export async function refreshBalances(store: Store, userIds: Iterable<string>) {
|
||||
const ids = [...new Set(userIds)];
|
||||
if (!ids.some((userId) => balanceBucketsNeedRefresh(store, userId))) return;
|
||||
await store.transact((tx) => { for (const userId of ids) ensureBalanceBuckets(tx, userId); });
|
||||
}
|
||||
|
||||
type AllocationSlice = { allocation: ReserveAllocation; bucket: BalanceBucket; amount: number };
|
||||
|
||||
function planReleaseAllocations(store: Store, userId: string, referenceId: string, amount: number) {
|
||||
const allocations = store.reserveAllocations.get(allocationKey(userId, referenceId)) || [];
|
||||
const buckets = store.buckets.get(userId) || [];
|
||||
const slices: AllocationSlice[] = [];
|
||||
let remaining = amount;
|
||||
for (let index = allocations.length - 1; index >= 0 && remaining > 0; index -= 1) {
|
||||
const allocation = allocations[index];
|
||||
const bucket = buckets.find((item) => item.id === allocation.bucketId);
|
||||
if (!bucket) return undefined;
|
||||
const released = Math.min(allocation.amount, remaining);
|
||||
if (released > 0) slices.push({ allocation, bucket, amount: released });
|
||||
remaining -= released;
|
||||
}
|
||||
return remaining === 0 ? slices : undefined;
|
||||
}
|
||||
|
||||
export function reserveBalance(store: Store, userId: string, amount: number, referenceId: string, options: { bucketFilter?: (bucket: BalanceBucket) => boolean; preferredBucketIds?: Set<string> } = {}) {
|
||||
if (!Number.isSafeInteger(amount) || amount <= 0) return false;
|
||||
const idempotencyKey = `reserve:${userId}:${referenceId}`;
|
||||
if (store.ledger.some((item) => item.idempotencyKey === idempotencyKey || item.idempotencyKey.startsWith(`${idempotencyKey}:`))) return true;
|
||||
const balance = store.balances.get(userId) || { available: 0, reserved: 0 };
|
||||
const buckets = ensureBalanceBuckets(store, userId).filter((bucket) => bucket.remaining > 0 && (!options.bucketFilter || options.bucketFilter(bucket))).sort((a, b) => Number(Boolean(options.preferredBucketIds?.has(b.id))) - Number(Boolean(options.preferredBucketIds?.has(a.id))) || a.priority - b.priority || (a.expiresAt || "9999").localeCompare(b.expiresAt || "9999"));
|
||||
if (balance.available < amount || buckets.reduce((total, bucket) => total + bucket.remaining, 0) < amount) return false;
|
||||
let remaining = amount;
|
||||
const allocations: Array<{ bucketId: string; amount: number }> = [];
|
||||
for (const bucket of buckets) {
|
||||
if (remaining <= 0) break;
|
||||
const allocated = Math.min(bucket.remaining, remaining);
|
||||
remaining -= allocated;
|
||||
allocations.push({ bucketId: bucket.id, amount: allocated });
|
||||
}
|
||||
const createdAt = new Date().toISOString();
|
||||
const ledgerStart = store.ledger.length;
|
||||
for (const allocation of allocations) {
|
||||
if (!appendLedger(store, { id: randomUUID(), userId, type: "reserve", amount: allocation.amount, balanceUnitVersion: store.balanceUnit.version, idempotencyKey: `${idempotencyKey}:${allocation.bucketId}`, createdAt, referenceId, bucketId: allocation.bucketId })) {
|
||||
store.ledger.splice(ledgerStart);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
for (const allocation of allocations) (store.buckets.get(userId) || []).find((bucket) => bucket.id === allocation.bucketId)!.remaining -= allocation.amount;
|
||||
balance.available -= amount;
|
||||
balance.reserved += amount;
|
||||
bumpBalanceVersion(balance);
|
||||
store.balances.set(userId, balance);
|
||||
store.reserveAllocations.set(allocationKey(userId, referenceId), allocations.map((allocation) => ({ ...allocation })));
|
||||
store.persist();
|
||||
return true;
|
||||
}
|
||||
|
||||
export function releaseBalance(store: Store, userId: string, amount: number, referenceId: string) {
|
||||
const idempotencyKey = `release:${userId}:${referenceId}:${amount}`;
|
||||
if (amount <= 0 || store.ledger.some((item) => item.idempotencyKey === idempotencyKey || item.idempotencyKey.startsWith(`${idempotencyKey}:`))) return false;
|
||||
const balance = store.balances.get(userId) || { available: 0, reserved: 0 };
|
||||
if (!Number.isSafeInteger(amount) || amount > balance.reserved) return false;
|
||||
const slices = planReleaseAllocations(store, userId, referenceId, amount);
|
||||
if (!slices) return false;
|
||||
const ledgerStart = store.ledger.length;
|
||||
const createdAt = new Date().toISOString();
|
||||
const expiredBucketIds = new Set(slices.filter((slice) => Boolean(slice.bucket.expiresAt) && Date.parse(slice.bucket.expiresAt!) <= Date.parse(createdAt)).map((slice) => slice.bucket.id));
|
||||
for (const slice of slices) {
|
||||
const expired = expiredBucketIds.has(slice.bucket.id);
|
||||
if (!appendLedger(store, { id: randomUUID(), userId, type: expired ? "adjustment" : "release", amount: expired ? -slice.amount : slice.amount, balanceUnitVersion: store.balanceUnit.version, idempotencyKey: expired ? `${idempotencyKey}:expire:${slice.bucket.id}` : `${idempotencyKey}:${slice.bucket.id}`, createdAt, referenceId, bucketId: slice.bucket.id })) {
|
||||
store.ledger.splice(ledgerStart);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
let restoredAmount = 0;
|
||||
for (const slice of slices) {
|
||||
slice.allocation.amount -= slice.amount;
|
||||
if (!expiredBucketIds.has(slice.bucket.id)) { slice.bucket.remaining += slice.amount; restoredAmount += slice.amount; }
|
||||
}
|
||||
const key = allocationKey(userId, referenceId);
|
||||
const remainingAllocations = (store.reserveAllocations.get(key) || []).filter((allocation) => allocation.amount > 0);
|
||||
if (remainingAllocations.length) store.reserveAllocations.set(key, remainingAllocations); else store.reserveAllocations.delete(key);
|
||||
balance.reserved -= amount;
|
||||
balance.available += restoredAmount;
|
||||
bumpBalanceVersion(balance);
|
||||
store.balances.set(userId, balance);
|
||||
store.persist();
|
||||
return true;
|
||||
}
|
||||
|
||||
export function refundBalance(store: Store, userId: string, amount: number, referenceId: string) {
|
||||
const idempotencyKey = `refund:${userId}:${referenceId}`;
|
||||
if (!Number.isSafeInteger(amount) || amount <= 0 || store.ledger.some((item) => item.idempotencyKey === idempotencyKey)) return false;
|
||||
const balance = store.balances.get(userId) || { available: 0, reserved: 0 };
|
||||
const buckets = ensureBalanceBuckets(store, userId);
|
||||
const bucket = { id: randomUUID(), userId, source: "manual" as const, remaining: amount, priority: buckets.length };
|
||||
if (!appendLedger(store, { id: randomUUID(), userId, type: "refund", amount, balanceUnitVersion: store.balanceUnit.version, idempotencyKey, createdAt: new Date().toISOString(), referenceId, bucketId: bucket.id })) return false;
|
||||
balance.available += amount; buckets.push(bucket); bumpBalanceVersion(balance); store.balances.set(userId, balance); store.buckets.set(userId, buckets); store.persist(); return true;
|
||||
}
|
||||
|
||||
export function adjustBalance(store: Store, userId: string, amount: number, referenceId: string) {
|
||||
const idempotencyKey = `admin-adjustment:${referenceId}`;
|
||||
if (!Number.isSafeInteger(amount) || amount === 0 || store.ledger.some((item) => item.idempotencyKey === idempotencyKey || item.idempotencyKey.startsWith(`${idempotencyKey}:`))) return false;
|
||||
const buckets = ensureBalanceBuckets(store, userId);
|
||||
const balance = store.balances.get(userId) || { available: 0, reserved: 0 };
|
||||
if (balance.available + amount < 0) return false;
|
||||
const createdAt = new Date().toISOString();
|
||||
const availableBuckets = buckets.filter((bucket) => bucket.remaining > 0).sort((a, b) => a.priority - b.priority || (a.expiresAt || "9999").localeCompare(b.expiresAt || "9999"));
|
||||
if (amount > 0) {
|
||||
const bucket = { id: randomUUID(), userId, source: "manual" as const, remaining: amount, priority: buckets.length };
|
||||
if (!appendLedger(store, { id: randomUUID(), userId, type: "adjustment", amount, balanceUnitVersion: store.balanceUnit.version, idempotencyKey, createdAt, referenceId, bucketId: bucket.id })) return false;
|
||||
balance.available += amount; buckets.push(bucket); bumpBalanceVersion(balance); store.balances.set(userId, balance); store.buckets.set(userId, buckets); store.persist(); return true;
|
||||
}
|
||||
let remaining = -amount;
|
||||
const allocations: Array<{ bucket: BalanceBucket; amount: number }> = [];
|
||||
for (const bucket of availableBuckets) {
|
||||
if (remaining <= 0) break;
|
||||
const allocated = Math.min(bucket.remaining, remaining);
|
||||
if (allocated > 0) allocations.push({ bucket, amount: allocated });
|
||||
remaining -= allocated;
|
||||
}
|
||||
if (remaining > 0) return false;
|
||||
const ledgerStart = store.ledger.length;
|
||||
for (const allocation of allocations) {
|
||||
if (!appendLedger(store, { id: randomUUID(), userId, type: "adjustment", amount: -allocation.amount, balanceUnitVersion: store.balanceUnit.version, idempotencyKey: `${idempotencyKey}:${allocation.bucket.id}`, createdAt, referenceId, bucketId: allocation.bucket.id })) { store.ledger.splice(ledgerStart); return false; }
|
||||
}
|
||||
for (const allocation of allocations) allocation.bucket.remaining -= allocation.amount;
|
||||
balance.available += amount; bumpBalanceVersion(balance); store.balances.set(userId, balance); store.buckets.set(userId, buckets); store.persist(); return true;
|
||||
}
|
||||
|
||||
export function settleBalance(store: Store, userId: string, reservedAmount: number, chargedAmount: number, referenceId: string) {
|
||||
const idempotencyKey = `settle:${userId}:${referenceId}`;
|
||||
if (reservedAmount <= 0 || chargedAmount < 0 || chargedAmount > reservedAmount || store.ledger.some((item) => item.idempotencyKey === idempotencyKey || item.idempotencyKey.startsWith(`${idempotencyKey}:`))) return false;
|
||||
const balance = store.balances.get(userId) || { available: 0, reserved: 0 };
|
||||
if (!Number.isSafeInteger(reservedAmount) || !Number.isSafeInteger(chargedAmount) || reservedAmount > balance.reserved) return false;
|
||||
const key = allocationKey(userId, referenceId);
|
||||
const allocations = store.reserveAllocations.get(key) || [];
|
||||
if (allocations.reduce((total, allocation) => total + allocation.amount, 0) !== reservedAmount) return false;
|
||||
const buckets = store.buckets.get(userId) || [];
|
||||
const settled: AllocationSlice[] = [];
|
||||
const released: AllocationSlice[] = [];
|
||||
let remainingCharge = chargedAmount;
|
||||
for (const allocation of allocations) {
|
||||
const bucket = buckets.find((item) => item.id === allocation.bucketId);
|
||||
if (!bucket) return false;
|
||||
const charged = Math.min(allocation.amount, remainingCharge);
|
||||
if (charged > 0) settled.push({ allocation, bucket, amount: charged });
|
||||
if (allocation.amount > charged) released.push({ allocation, bucket, amount: allocation.amount - charged });
|
||||
remainingCharge -= charged;
|
||||
}
|
||||
if (remainingCharge !== 0) return false;
|
||||
const ledgerStart = store.ledger.length;
|
||||
const now = new Date().toISOString();
|
||||
const expiredBucketIds = new Set(released.filter((slice) => Boolean(slice.bucket.expiresAt) && Date.parse(slice.bucket.expiresAt!) <= Date.parse(now)).map((slice) => slice.bucket.id));
|
||||
for (const slice of released) {
|
||||
const expired = expiredBucketIds.has(slice.bucket.id);
|
||||
if (!appendLedger(store, { id: randomUUID(), userId, type: expired ? "adjustment" : "release", amount: expired ? -slice.amount : slice.amount, balanceUnitVersion: store.balanceUnit.version, idempotencyKey: expired ? `${idempotencyKey}:expire:${slice.bucket.id}` : `release:settle:${userId}:${referenceId}:${slice.bucket.id}`, createdAt: now, referenceId, bucketId: slice.bucket.id })) { store.ledger.splice(ledgerStart); return false; }
|
||||
}
|
||||
if (!settled.length) {
|
||||
if (!appendLedger(store, { id: randomUUID(), userId, type: "settle", amount: 0, balanceUnitVersion: store.balanceUnit.version, idempotencyKey, createdAt: now, referenceId })) { store.ledger.splice(ledgerStart); return false; }
|
||||
} else {
|
||||
for (const slice of settled) {
|
||||
if (!appendLedger(store, { id: randomUUID(), userId, type: "settle", amount: -slice.amount, balanceUnitVersion: store.balanceUnit.version, idempotencyKey: `${idempotencyKey}:${slice.bucket.id}`, createdAt: now, referenceId, bucketId: slice.bucket.id })) { store.ledger.splice(ledgerStart); return false; }
|
||||
}
|
||||
}
|
||||
let restoredAmount = 0;
|
||||
for (const slice of released) if (!expiredBucketIds.has(slice.bucket.id)) { slice.bucket.remaining += slice.amount; restoredAmount += slice.amount; }
|
||||
store.reserveAllocations.delete(key);
|
||||
balance.reserved -= reservedAmount;
|
||||
balance.available += restoredAmount;
|
||||
bumpBalanceVersion(balance);
|
||||
store.balances.set(userId, balance);
|
||||
store.persist();
|
||||
return true;
|
||||
}
|
||||
Reference in New Issue
Block a user