feat(canvas): remove Volcengine Ark protocol support and streamline video generation references
This commit is contained in:
HouYunFei
+10
-149
@@ -3,31 +3,20 @@ import { nanoid } from "nanoid";
|
||||
|
||||
import i18n from "@/i18n";
|
||||
import { dataUrlToFile } from "@/lib/image-utils";
|
||||
import { getMediaBlob, uploadMediaFile, type UploadedFile } from "@/services/file-storage";
|
||||
import { uploadMediaFile, type UploadedFile } from "@/services/file-storage";
|
||||
import { imageToDataUrl } from "@/services/image-storage";
|
||||
import { boolConfig, buildSeedancePromptText, isSeedanceVideoConfig, normalizeSeedanceDuration, normalizeSeedanceRatio, normalizeSeedanceResolution, seedanceVideoReferenceError, SEEDANCE_REFERENCE_LIMITS } from "@/lib/seedance-video";
|
||||
import { buildApiUrl, modelOptionName, resolveModelRequestConfig, resolveModelScript, type AiConfig } from "@/stores/use-config-store";
|
||||
import { boolConfig, buildApiUrl, modelOptionName, resolveModelRequestConfig, resolveModelScript, type AiConfig } from "@/stores/use-config-store";
|
||||
import { runModelPlugin } from "./model-plugin";
|
||||
import type { ReferenceImage } from "@/types/image";
|
||||
import type { ReferenceAudio, ReferenceVideo } from "@/types/media";
|
||||
|
||||
type VideoResponse = { id: string; status?: string; error?: { message?: string }; url?: string; result_url?: string; video_url?: string; content?: { video_url?: string; url?: string } | null };
|
||||
type ApiVideoResponse = VideoResponse | { code?: number | string; data?: VideoResponse | null; msg?: string; message?: string; error?: { message?: string } };
|
||||
type SeedanceTask = {
|
||||
id: string;
|
||||
status?: "queued" | "running" | "succeeded" | "completed" | "failed" | "cancelled" | "expired";
|
||||
error?: { code?: string; message?: string } | null;
|
||||
content?: { video_url?: string; url?: string; last_frame_url?: string } | null;
|
||||
url?: string;
|
||||
result_url?: string;
|
||||
video_url?: string;
|
||||
};
|
||||
type ApiEnvelope<T> = T | { code?: number | string; data?: T | null; msg?: string; message?: string; error?: { message?: string } };
|
||||
type RequestOptions = { signal?: AbortSignal };
|
||||
const apiText = (key: string, options?: Record<string, unknown>) => i18n.t(`apiErrors.${key}`, options);
|
||||
|
||||
export type VideoGenerationResult = { blob?: Blob; url?: string; mimeType?: string };
|
||||
export type VideoGenerationTask = { id: string; provider: "openai" | "seedance" | "plugin"; model: string };
|
||||
export type VideoGenerationTask = { id: string; provider: "openai" | "plugin"; model: string };
|
||||
export type VideoGenerationTaskState = { status: "pending" } | { status: "completed"; result: VideoGenerationResult } | { status: "failed"; error: string };
|
||||
|
||||
/** Results for scripted (plugin) video models, which run their own create+poll in one shot at task creation. */
|
||||
@@ -44,32 +33,25 @@ function aiHeaders(config: AiConfig, contentType?: string) {
|
||||
};
|
||||
}
|
||||
|
||||
export async function requestVideoGeneration(config: AiConfig, prompt: string, references: ReferenceImage[] = [], videoReferences: ReferenceVideo[] = [], audioReferences: ReferenceAudio[] = [], options?: RequestOptions): Promise<VideoGenerationResult> {
|
||||
const task = await createVideoGenerationTask(config, prompt, references, videoReferences, audioReferences, options);
|
||||
const delayMs = task.provider === "seedance" ? 5000 : 2500;
|
||||
export async function requestVideoGeneration(config: AiConfig, prompt: string, references: ReferenceImage[] = [], options?: RequestOptions): Promise<VideoGenerationResult> {
|
||||
const task = await createVideoGenerationTask(config, prompt, references, options);
|
||||
for (let attempt = 0; attempt < 120; attempt += 1) {
|
||||
if (options?.signal?.aborted) throw new DOMException("Aborted", "AbortError");
|
||||
const state = await pollVideoGenerationTask(config, task, options);
|
||||
if (state.status === "completed") return state.result;
|
||||
if (state.status === "failed") throw new Error(state.error);
|
||||
if (attempt === 119) throw new Error(apiText("videoTimeout", { provider: task.provider === "seedance" ? "Seedance " : "" }));
|
||||
await delay(delayMs, options?.signal);
|
||||
if (attempt === 119) throw new Error(apiText("videoTimeout", { provider: "" }));
|
||||
await delay(2500, options?.signal);
|
||||
}
|
||||
throw new Error(apiText("videoTimeout", { provider: "" }));
|
||||
}
|
||||
|
||||
export async function createVideoGenerationTask(config: AiConfig, prompt: string, references: ReferenceImage[] = [], videoReferences: ReferenceVideo[] = [], audioReferences: ReferenceAudio[] = [], options?: RequestOptions): Promise<VideoGenerationTask> {
|
||||
export async function createVideoGenerationTask(config: AiConfig, prompt: string, references: ReferenceImage[] = [], options?: RequestOptions): Promise<VideoGenerationTask> {
|
||||
const selectedModel = (config.model || config.videoModel).trim();
|
||||
const requestConfig = resolveModelRequestConfig(config, selectedModel);
|
||||
const script = resolveModelScript(config, selectedModel);
|
||||
if (script) return createPluginVideoTask(requestConfig, selectedModel, script, prompt, references, options);
|
||||
assertVideoConfig(requestConfig, requestConfig.model);
|
||||
if (isSeedanceVideoConfig(requestConfig)) {
|
||||
return createSeedanceTask(requestConfig, selectedModel, prompt, references, videoReferences, audioReferences, options);
|
||||
}
|
||||
if (videoReferences.length || audioReferences.length) {
|
||||
throw new Error(apiText("videoReferencesUnsupported"));
|
||||
}
|
||||
return createOpenAIVideoTask(requestConfig, selectedModel, prompt, references, options);
|
||||
}
|
||||
|
||||
@@ -80,7 +62,7 @@ export async function pollVideoGenerationTask(config: AiConfig, task: VideoGener
|
||||
}
|
||||
const requestConfig = resolveModelRequestConfig(config, task.model);
|
||||
assertVideoConfig(requestConfig, requestConfig.model);
|
||||
return task.provider === "seedance" ? pollSeedanceTask(requestConfig, task, options) : pollOpenAIVideoTask(requestConfig, task, options);
|
||||
return pollOpenAIVideoTask(requestConfig, task, options);
|
||||
}
|
||||
|
||||
async function createPluginVideoTask(config: AiConfig, model: string, script: string, prompt: string, references: ReferenceImage[], options?: RequestOptions): Promise<VideoGenerationTask> {
|
||||
@@ -170,114 +152,6 @@ async function pollOpenAIVideoTask(config: AiConfig, task: VideoGenerationTask,
|
||||
}
|
||||
}
|
||||
|
||||
async function createSeedanceTask(config: AiConfig, model: string, prompt: string, references: ReferenceImage[], videoReferences: ReferenceVideo[], audioReferences: ReferenceAudio[], options?: RequestOptions): Promise<VideoGenerationTask> {
|
||||
if (audioReferences.length && !references.length && !videoReferences.length) {
|
||||
throw new Error(apiText("seedanceAudioRequiresVisual"));
|
||||
}
|
||||
assertSeedanceVideoReferences(videoReferences);
|
||||
assertSeedanceAudioReferences(audioReferences);
|
||||
const content = await buildSeedanceContent(config, prompt, references, videoReferences, audioReferences);
|
||||
if (!content.length) throw new Error(apiText("videoPromptRequired"));
|
||||
const payload = {
|
||||
model: modelOptionName(model),
|
||||
content,
|
||||
ratio: normalizeSeedanceRatio(config.size),
|
||||
resolution: normalizeSeedanceResolution(config.vquality),
|
||||
duration: normalizeSeedanceDuration(config.videoSeconds),
|
||||
generate_audio: boolConfig(config.videoGenerateAudio, true),
|
||||
watermark: boolConfig(config.videoWatermark, false),
|
||||
};
|
||||
|
||||
try {
|
||||
const created = unwrapSeedanceTask((await axios.post<ApiEnvelope<SeedanceTask>>(seedanceApiUrl(config), payload, { headers: aiHeaders(config, "application/json"), signal: options?.signal })).data);
|
||||
if (!created.id) throw new Error(apiText("seedanceNoTaskId"));
|
||||
return { id: created.id, provider: "seedance", model };
|
||||
} catch (error) {
|
||||
throw new Error(readAxiosError(error, apiText("seedanceTaskCreateFailed")));
|
||||
}
|
||||
}
|
||||
|
||||
async function pollSeedanceTask(config: AiConfig, task: VideoGenerationTask, options?: RequestOptions): Promise<VideoGenerationTaskState> {
|
||||
try {
|
||||
const state = unwrapSeedanceTask((await axios.get<ApiEnvelope<SeedanceTask>>(seedanceApiUrl(config, task.id), { headers: aiHeaders(config), signal: options?.signal })).data);
|
||||
const url = videoResultUrl(state);
|
||||
if (url) return { status: "completed", result: await videoResultFromUrl(url, options) };
|
||||
if (state.status === "succeeded" || state.status === "completed") return { status: "failed", error: apiText("seedanceNoVideoUrl") };
|
||||
if (state.status === "failed" || state.status === "cancelled" || state.status === "expired") return { status: "failed", error: readApiErrorMessage(state.error?.message) || apiText(state.status === "expired" ? "seedanceVideoTimeout" : "seedanceVideoFailed") };
|
||||
return { status: "pending" };
|
||||
} catch (error) {
|
||||
throw new Error(readAxiosError(error, apiText("seedanceTaskQueryFailed")));
|
||||
}
|
||||
}
|
||||
|
||||
function assertSeedanceVideoReferences(videoReferences: ReferenceVideo[]) {
|
||||
const error = seedanceVideoReferenceError(videoReferences);
|
||||
if (error) throw new Error(error);
|
||||
let total = 0;
|
||||
for (const video of videoReferences) {
|
||||
if (!video.durationMs) continue;
|
||||
if (video.durationMs < 2000 || video.durationMs > 15000) throw new Error(apiText("seedanceVideoDuration"));
|
||||
total += video.durationMs;
|
||||
}
|
||||
if (total > 15000) throw new Error(apiText("seedanceVideoTotalDuration"));
|
||||
}
|
||||
|
||||
function assertSeedanceAudioReferences(audioReferences: ReferenceAudio[]) {
|
||||
let total = 0;
|
||||
for (const audio of audioReferences) {
|
||||
if (!audio.durationMs) continue;
|
||||
if (audio.durationMs < 2000 || audio.durationMs > 15000) throw new Error(apiText("seedanceAudioDuration"));
|
||||
total += audio.durationMs;
|
||||
}
|
||||
if (total > 15000) throw new Error(apiText("seedanceAudioTotalDuration"));
|
||||
}
|
||||
|
||||
function seedanceApiUrl(config: AiConfig, taskId?: string) {
|
||||
return buildApiUrl(config.baseUrl, `/contents/generations/tasks${taskId ? `/${encodeURIComponent(taskId)}` : ""}`);
|
||||
}
|
||||
|
||||
async function buildSeedanceContent(config: AiConfig, prompt: string, references: ReferenceImage[], videoReferences: ReferenceVideo[], audioReferences: ReferenceAudio[]) {
|
||||
const content: Array<Record<string, unknown>> = [];
|
||||
const text = buildSeedancePromptText(prompt, references, videoReferences, audioReferences);
|
||||
if (text) content.push({ type: "text", text });
|
||||
for (const image of references.slice(0, SEEDANCE_REFERENCE_LIMITS.images)) {
|
||||
content.push({ type: "image_url", image_url: { url: await resolveSeedanceImageUrl(config, image) }, role: "reference_image" });
|
||||
}
|
||||
for (const video of videoReferences.slice(0, SEEDANCE_REFERENCE_LIMITS.videos)) {
|
||||
content.push({ type: "video_url", video_url: { url: await resolveSeedanceVideoUrl(video) }, role: "reference_video" });
|
||||
}
|
||||
for (const audio of audioReferences.slice(0, SEEDANCE_REFERENCE_LIMITS.audios)) {
|
||||
content.push({ type: "audio_url", audio_url: { url: await resolveSeedanceAudioUrl(audio) }, role: "reference_audio" });
|
||||
}
|
||||
return content;
|
||||
}
|
||||
|
||||
async function resolveSeedanceImageUrl(config: AiConfig, image: ReferenceImage) {
|
||||
const directUrl = image.url || image.dataUrl;
|
||||
if (isPublicMediaUrl(directUrl) || directUrl.startsWith("asset://")) return directUrl;
|
||||
const dataUrl = await imageToDataUrl(image);
|
||||
if (!dataUrl) throw new Error(apiText("referenceImageReadFailed"));
|
||||
return dataUrl;
|
||||
}
|
||||
|
||||
async function resolveSeedanceVideoUrl(video: ReferenceVideo) {
|
||||
if (isPublicMediaUrl(video.url) || video.url.startsWith("asset://")) return video.url;
|
||||
let blob: Blob | null = null;
|
||||
if (video.storageKey) blob = await getMediaBlob(video.storageKey);
|
||||
if (!blob && video.url?.startsWith("blob:")) blob = await (await fetch(video.url)).blob();
|
||||
if (!blob) throw new Error(apiText("invalidReferenceVideo"));
|
||||
return blobToDataUrl(blob);
|
||||
}
|
||||
|
||||
async function resolveSeedanceAudioUrl(audio: ReferenceAudio) {
|
||||
if (isPublicMediaUrl(audio.url) || audio.url.startsWith("asset://")) return audio.url;
|
||||
let blob: Blob | null = null;
|
||||
if (audio.storageKey) blob = await getMediaBlob(audio.storageKey);
|
||||
if (!blob && audio.url?.startsWith("blob:")) blob = await (await fetch(audio.url)).blob();
|
||||
if (!blob) throw new Error(apiText("invalidReferenceAudio"));
|
||||
return blobToDataUrl(blob);
|
||||
}
|
||||
|
||||
async function videoResultFromUrl(url: string, options?: RequestOptions): Promise<VideoGenerationResult> {
|
||||
try {
|
||||
const response = await axios.get<Blob>(url, { responseType: "blob", signal: options?.signal });
|
||||
@@ -319,10 +193,6 @@ function unwrapVideoResponse(payload: ApiVideoResponse) {
|
||||
return unwrapEnvelope(payload, apiText("noVideoTask"));
|
||||
}
|
||||
|
||||
function unwrapSeedanceTask(payload: ApiEnvelope<SeedanceTask>) {
|
||||
return unwrapEnvelope(payload, apiText("seedanceNoTask"));
|
||||
}
|
||||
|
||||
function unwrapEnvelope<T>(payload: ApiEnvelope<T>, emptyMessage: string): T {
|
||||
if (!payload) throw new Error(emptyMessage);
|
||||
if (typeof payload === "object" && "code" in payload && payload.code !== undefined) {
|
||||
@@ -333,7 +203,7 @@ function unwrapEnvelope<T>(payload: ApiEnvelope<T>, emptyMessage: string): T {
|
||||
return payload as T;
|
||||
}
|
||||
|
||||
function videoResultUrl(payload: VideoResponse | SeedanceTask) {
|
||||
function videoResultUrl(payload: VideoResponse) {
|
||||
return [payload.video_url, payload.result_url, payload.url, payload.content?.video_url, payload.content?.url].find((url) => typeof url === "string" && (isPublicMediaUrl(url) || /\.mp4(\?|#|$)/i.test(url)));
|
||||
}
|
||||
|
||||
@@ -415,12 +285,3 @@ function delay(ms: number, signal?: AbortSignal) {
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
function blobToDataUrl(blob: Blob) {
|
||||
return new Promise<string>((resolve, reject) => {
|
||||
const reader = new FileReader();
|
||||
reader.onload = () => resolve(String(reader.result || ""));
|
||||
reader.onerror = () => reject(new Error(apiText("localAssetReadFailed")));
|
||||
reader.readAsDataURL(blob);
|
||||
});
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user