Build Lip-Syncing AI Avatars for Customer Service in 2026

npm install @heygen/streaming-avatar

import StreamingAvatar, { AvatarQuality, StreamingEvents } from "@heygen/streaming-avatar";

const avatar = new StreamingAvatar({
  token: process.env.HEYGEN_API_KEY,
});

// Start a session — this reserves compute on HeyGen's end
const sessionData = await avatar.createStartAvatar({
  quality: AvatarQuality.High,
  avatarName: "Anna_public_3_20240108",  // Use HeyGen's public avatars to start
  voice: {
    voiceId: "en-US-AriaNeural",         // Azure voice ID passed through
    rate: 1.0,
    emotion: "Friendly",
  },
  language: "en",
  disableIdleTimeout: false,
});

console.log("Session ID:", sessionData.session_id);

// frontend/AvatarPlayer.tsx
import { useEffect, useRef } from "react";

export function AvatarPlayer({ sessionData }: { sessionData: SessionData }) {
  const videoRef = useRef<HTMLVideoElement>(null);

  useEffect(() => {
    const peerConnection = new RTCPeerConnection({
      iceServers: sessionData.ice_servers,  // From HeyGen session response
    });

    // Attach incoming video track to the <video> element
    peerConnection.ontrack = (event) => {
      if (videoRef.current && event.streams[0]) {
        videoRef.current.srcObject = event.streams[0];
      }
    };

    // Set remote description from HeyGen's SDP offer
    peerConnection.setRemoteDescription(
      new RTCSessionDescription(sessionData.sdp)
    );

    // Generate and send answer back to HeyGen
    peerConnection.createAnswer().then((answer) => {
      peerConnection.setLocalDescription(answer);
      // Send answer to your backend → HeyGen
      sendAnswerToBackend(answer);
    });

    return () => peerConnection.close();
  }, [sessionData]);

  return (
    <video
      ref={videoRef}
      autoPlay
      playsInline
      className="rounded-xl w-full max-w-md"
    />
  );
}

// backend/avatarController.ts
import { OpenAI } from "openai";

const openai = new OpenAI();

export async function handleCustomerMessage(
  sessionId: string,
  customerMessage: string,
  avatarClient: StreamingAvatar
) {
  // Generate support response
  const completion = await openai.chat.completions.create({
    model: "gpt-4o",
    messages: [
      { role: "system", content: CUSTOMER_SERVICE_PROMPT },
      { role: "user", content: customerMessage },
    ],
    max_tokens: 150,  // Keep responses short — long clips add latency
  });

  const agentResponse = completion.choices[0].message.content;

  // Send text to avatar — HeyGen handles TTS + lip sync internally
  await avatarClient.speak({
    text: agentResponse,
    taskType: "repeat",   // "repeat" = speak exactly this text
    taskMode: "sync",     // Wait for speech to finish before resolving
  });

  return agentResponse;
}

// Detect customer speech and interrupt avatar
const recognition = new webkitSpeechRecognition();
recognition.continuous = true;

recognition.onstart = async () => {
  // Stop avatar mid-sentence when customer starts talking
  await avatarClient.interrupt();
};

recognition.onresult = async (event) => {
  const transcript = event.results[event.results.length - 1][0].transcript;
  if (event.results[event.results.length - 1].isFinal) {
    await handleCustomerMessage(sessionId, transcript, avatarClient);
  }
};

recognition.start();

# Install dependencies (GPU required for real-time)
pip install wav2lip-hq torch torchaudio

# Download pretrained model
wget https://github.com/Rudrabha/Wav2Lip/releases/download/v1.0/wav2lip_gan.pth

# local_avatar.py
import subprocess
import tempfile
from pathlib import Path

def generate_avatar_clip(text: str, base_image: str) -> str:
    """
    Generates a lip-synced video from text + static avatar image.
    Returns path to output MP4.
    """
    # Step 1: Generate audio from text
    audio_path = text_to_speech(text)  # Use your TTS of choice

    # Step 2: Run Wav2Lip inference
    output_path = tempfile.mktemp(suffix=".mp4")
    subprocess.run([
        "python", "inference.py",
        "--checkpoint_path", "wav2lip_gan.pth",
        "--face", base_image,           # Your avatar photo or video loop
        "--audio", audio_path,
        "--outfile", output_path,
        "--resize_factor", "1",
        "--nosmooth",                   # Sharper but faster
    ], check=True)

    return output_path

# Test HeyGen session creation
curl -X POST https://api.heygen.com/v1/streaming.new \
  -H "x-api-key: $HEYGEN_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{"quality": "high", "avatar_name": "Anna_public_3_20240108"}'