import type { ChatCompletionMessage } from "@/services/api/image"; import type { ReferenceImage } from "@/types/image"; import type { ReferenceAudio, ReferenceVideo } from "@/types/media"; import { CanvasNodeType, type CanvasConnection, type CanvasNodeData } from "../types"; export type NodeGenerationContext = { prompt: string; referenceImages: ReferenceImage[]; referenceVideos: ReferenceVideo[]; referenceAudios: ReferenceAudio[]; textCount: number; imageCount: number; videoCount: number; audioCount: number; }; export type NodeGenerationInput = { nodeId: string; type: "text" | "image" | "video" | "audio"; title: string; text?: string; image?: ReferenceImage; video?: ReferenceVideo; audio?: ReferenceAudio; }; export function buildNodeGenerationContext(nodeId: string, nodes: CanvasNodeData[], connections: CanvasConnection[], prompt: string): NodeGenerationContext { const inputs = buildNodeGenerationInputs(nodeId, nodes, connections); const upstreamText = inputs .map((input) => input.text) .filter(Boolean) .join("\n\n"); const referenceImages = inputs.map((input) => input.image).filter((image): image is ReferenceImage => Boolean(image)); const referenceVideos = inputs.map((input) => input.video).filter((video): video is ReferenceVideo => Boolean(video)); const referenceAudios = inputs.map((input) => input.audio).filter((audio): audio is ReferenceAudio => Boolean(audio)); return { prompt: upstreamText ? `${prompt}\n\n${upstreamText}` : prompt, referenceImages, referenceVideos, referenceAudios, textCount: inputs.filter((input) => input.type === "text").length, imageCount: referenceImages.length, videoCount: referenceVideos.length, audioCount: referenceAudios.length, }; } export function buildNodeGenerationInputs(nodeId: string, nodes: CanvasNodeData[], connections: CanvasConnection[]): NodeGenerationInput[] { return getOrderedUpstreamNodes(nodeId, nodes, connections).flatMap((node): NodeGenerationInput[] => { const image = readReferenceImage(node); if (image) return [{ nodeId: node.id, type: "image" as const, title: node.title, image }]; const video = readReferenceVideo(node); if (video) return [{ nodeId: node.id, type: "video" as const, title: node.title, video }]; const audio = readReferenceAudio(node); if (audio) return [{ nodeId: node.id, type: "audio" as const, title: node.title, audio }]; const text = readNodeTextInput(node); if (text) return [{ nodeId: node.id, type: "text" as const, title: node.title, text }]; return []; }); } export function buildNodeChatMessages(context: NodeGenerationContext): ChatCompletionMessage[] { if (!context.referenceImages.length) { return [{ role: "user", content: context.prompt }]; } return [ { role: "user", content: [{ type: "text" as const, text: context.prompt }, ...context.referenceImages.map((image) => ({ type: "image_url" as const, image_url: { url: image.dataUrl } }))], }, ]; } export async function hydrateNodeGenerationContext(context: NodeGenerationContext) { const { imageToDataUrl } = await import("@/services/image-storage"); return { ...context, referenceImages: await Promise.all(context.referenceImages.map(async (image) => ({ ...image, dataUrl: await imageToDataUrl(image) }))) }; } function readNodeTextInput(node: CanvasNodeData) { if (node.type === CanvasNodeType.Text) return node.metadata?.content || node.metadata?.prompt || ""; return node.metadata?.prompt || ""; } function readReferenceImage(node: CanvasNodeData): ReferenceImage | null { if (node.type !== CanvasNodeType.Image || !node.metadata?.content) return null; return { id: node.id, name: `${node.title || node.id}.png`, type: node.metadata.mimeType || "image/png", dataUrl: node.metadata.content, storageKey: node.metadata.storageKey, }; } function readReferenceVideo(node: CanvasNodeData): ReferenceVideo | null { if (node.type !== CanvasNodeType.Video || !node.metadata?.content) return null; return { id: node.id, name: `${node.title || node.id}.mp4`, type: node.metadata.mimeType || "video/mp4", url: node.metadata.content, storageKey: node.metadata.storageKey, bytes: node.metadata.bytes, width: node.metadata.naturalWidth, height: node.metadata.naturalHeight, durationMs: node.metadata.durationMs, }; } function readReferenceAudio(node: CanvasNodeData): ReferenceAudio | null { if (node.type !== CanvasNodeType.Audio || !node.metadata?.content) return null; return { id: node.id, name: `${node.title || node.id}.mp3`, type: node.metadata.mimeType || "audio/mpeg", url: node.metadata.content, storageKey: node.metadata.storageKey, durationMs: node.metadata.durationMs, }; } function getOrderedUpstreamNodes(nodeId: string, nodes: CanvasNodeData[], connections: CanvasConnection[]) { const target = nodes.find((node) => node.id === nodeId); const upstreamNodes = connections .filter((connection) => connection.toNodeId === nodeId) .map((connection) => nodes.find((node) => node.id === connection.fromNodeId)) .filter((node): node is CanvasNodeData => Boolean(node)); const order = target?.metadata?.inputOrder || []; return [...order.map((id) => upstreamNodes.find((node) => node.id === id)).filter((node): node is CanvasNodeData => Boolean(node)), ...upstreamNodes.filter((node) => !order.includes(node.id))]; }