AI Gateway Reference-to-Video Generation

import { experimental_generateVideo as generateVideo } from 'ai';
import fs from 'node:fs';
 
const result = await generateVideo({
  model: 'google/veo-3.1-generate-001',
  prompt:
    'The video opens with a medium shot of a woman in a high-fashion flamingo dress walking through a lagoon',
  inputReferences: [
    fs.readFileSync('dress.png'),
    fs.readFileSync('glasses.png'),
    fs.readFileSync('woman.png'),
  ],
  duration: 8,
  aspectRatio: '16:9',
  generateAudio: true,
});
 
fs.writeFileSync('output.mp4', result.videos[0].uint8Array);

import { experimental_generateVideo as generateVideo } from 'ai';
import fs from 'node:fs';
 
const result = await generateVideo({
  model: 'klingai/kling-v3.0-i2v',
  prompt: 'The two characters meet and walk together through a sunny park',
  inputReferences: [
    'https://example.com/character-1.png',
    'https://example.com/character-2.png',
  ],
  aspectRatio: '16:9',
  duration: 5,
  providerOptions: {
    klingai: {
      mode: 'std',
    },
  },
});
 
fs.writeFileSync('output.mp4', result.videos[0].uint8Array);

import { experimental_generateVideo as generateVideo } from 'ai';
import fs from 'node:fs';
 
const result = await generateVideo({
  model: 'alibaba/wan-v2.6-r2v',
  prompt: 'character1 and character2 have a friendly conversation in a cozy cafe',
  resolution: '1920x1080',
  duration: 4,
  // References can be images or videos
  inputReferences: [
    'https://example.com/cat.png',
    'https://example.com/dog.png',
  ],
  providerOptions: {
    alibaba: {
      shotType: 'single',
    },
  },
});
 
fs.writeFileSync('output.mp4', result.videos[0].uint8Array);

import { experimental_generateVideo as generateVideo } from 'ai';
import { put } from '@vercel/blob';
import fs from 'node:fs';
 
const catImage = fs.readFileSync('./cat.png');
const { url: catUrl } = await put('cat.png', catImage, { access: 'public' });
 
const dogImage = fs.readFileSync('./dog.png');
const { url: dogUrl } = await put('dog.png', dogImage, { access: 'public' });
 
const result = await generateVideo({
  model: 'alibaba/wan-v2.6-r2v',
  prompt: 'character1 and character2 play together in a sunny garden',
  resolution: '1280x720',
  duration: 4,
  inputReferences: [catUrl, dogUrl],
  providerOptions: {
    alibaba: {
      shotType: 'single',
    },
  },
});
 
fs.writeFileSync('output.mp4', result.videos[0].uint8Array);

import { experimental_generateVideo as generateVideo } from 'ai';
import fs from 'node:fs';
 
const result = await generateVideo({
  model: 'spacexai/grok-imagine-video',
  prompt:
    'The comic cat from <IMAGE_1> and the comic dog from <IMAGE_2> ' +
    'are having a playful chase through a sunlit park. ' +
    'Cinematic slow-motion, warm afternoon light.',
  inputReferences: [
    'https://example.com/comic-cat.png',
    'https://example.com/comic-dog.png',
  ],
  duration: 8,
  aspectRatio: '16:9',
  providerOptions: {
    spacexai: {
      pollTimeoutMs: 600000,
    },
  },
});
 
fs.writeFileSync('output.mp4', result.videos[0].uint8Array);

import { experimental_generateVideo as generateVideo } from 'ai';
import fs from 'node:fs';
 
const result = await generateVideo({
  model: 'bytedance/seedance-2.5',
  prompt:
    'Replace the cat in [Video 1] with the lion from [Image 1]. The lion lies down and gently interacts with the girl in a warm and tender way.',
  aspectRatio: '16:9',
  duration: 12,
  inputReferences: [
    { data: 'https://example.com/lion.jpg', mediaType: 'image/jpeg' },
    { data: 'https://example.com/cat-video.mp4', mediaType: 'video/mp4' },
  ],
  generateAudio: true,
  providerOptions: {
    bytedance: {
      pollTimeoutMs: 600000,
    },
  },
});
 
fs.writeFileSync('output.mp4', result.videos[0].uint8Array);

const result = await generateVideo({
  model: gateway.videoModel('alibaba/wan-v2.6-r2v'),
  prompt: 'character1 and character2 have a friendly conversation in a cozy cafe',
  duration: 4,
  // Hosted URLs, not inline bytes: the start request is capped at 300 KiB.
  inputReferences: [
    'https://example.com/cat.png',
    'https://example.com/dog.png',
  ],
  webhook: async () => ({
    url: callbackUrl,
    received: waitForDelivery(token),
  }),
  poll: { timeoutMs: 10 * 60 * 1000 },
});